├── .coveragerc ├── .gitignore ├── .gitreview ├── .mailmap ├── .pre-commit-config.yaml ├── .pylintrc ├── .stestr.conf ├── .zuul.yaml ├── CONTRIBUTING.rst ├── LICENSE ├── README.rst ├── bindep.txt ├── doc ├── diagrams │ ├── area_of_influence.graffle.tgz │ ├── core.graffle.tgz │ ├── jobboard.graffle.tgz │ ├── tasks.graffle.tgz │ └── worker-engine.graffle.tgz ├── requirements.txt └── source │ ├── conf.py │ ├── index.rst │ ├── templates │ └── layout.html │ └── user │ ├── arguments_and_results.rst │ ├── atoms.rst │ ├── conductors.rst │ ├── engines.rst │ ├── examples.rst │ ├── exceptions.rst │ ├── history.rst │ ├── img │ ├── area_of_influence.svg │ ├── conductor.png │ ├── conductor_cycle.png │ ├── distributed_flow_rpc.png │ ├── engine_states.svg │ ├── flow_states.svg │ ├── job_states.svg │ ├── jobboard.png │ ├── mandelbrot.png │ ├── retry_states.svg │ ├── task_states.svg │ ├── tasks.png │ ├── wbe_request_states.svg │ └── worker-engine.svg │ ├── index.rst │ ├── inputs_and_outputs.rst │ ├── jobs.rst │ ├── notifications.rst │ ├── patterns.rst │ ├── persistence.rst │ ├── resumption.rst │ ├── shelf.rst │ ├── states.rst │ ├── types.rst │ ├── utils.rst │ └── workers.rst ├── playbooks └── tests │ └── functional │ ├── Debian.yaml │ ├── RedHat.yaml │ └── pre.yml ├── pyproject.toml ├── releasenotes ├── notes │ ├── .placeholder │ ├── add-sentinel-redis-support-9fd16e2a5dd5c0c9.yaml │ ├── bug-2056656-871b67ddbc8cfc92.yaml │ ├── deprecate-eventlet-df4a34a7d56acc47.yaml │ ├── disable-process_executor-python-312-d1074c816bc8303e.yaml │ ├── drop-python-2-7-73d3113c69d724d6.yaml │ ├── etcd-jobboard-backend-8a9fea2238fb0f12.yaml │ ├── fix-endless-loop-on-storage-error-dd4467f0bbc66abf.yaml │ ├── fix-endless-loop-on-storage-failures-b98b30f0c34d25e1.yaml │ ├── fix-revert-all-revert-a0310cd7beaa7409.yaml │ ├── fix-storage-failure-handling-5c115d92daa0eb82.yaml │ ├── fix-zookeeper-option-parsing-f9d37fbc39af47f4.yaml │ ├── mask-keys-74b9bb5c420d8091.yaml │ ├── redis-username-df0eb33869db09a2.yaml │ ├── remove-process_executor-f59d40a5dd287cd7.yaml │ ├── remove-py38-15af791146f479e1.yaml │ ├── remove-strict-redis-f2a5a924b314de41.yaml │ ├── sentinel-fallbacks-6fe2ab0d68959cdf.yaml │ ├── sentinel-ssl-399c56ed7067d282.yaml │ ├── sentinel-use-redis-creds-63f58b12ad46a2b5.yaml │ └── zookeeper-ssl-support-b9abf24a39096b62.yaml └── source │ ├── 2023.1.rst │ ├── 2023.2.rst │ ├── 2024.1.rst │ ├── 2024.2.rst │ ├── 2025.1.rst │ ├── _static │ └── .placeholder │ ├── _templates │ └── .placeholder │ ├── conf.py │ ├── index.rst │ ├── ocata.rst │ ├── pike.rst │ ├── queens.rst │ ├── rocky.rst │ ├── stein.rst │ ├── train.rst │ ├── unreleased.rst │ ├── ussuri.rst │ └── victoria.rst ├── requirements.txt ├── run_tests.sh ├── setup-etcd-env.sh ├── setup.cfg ├── setup.py ├── taskflow ├── __init__.py ├── atom.py ├── conductors │ ├── __init__.py │ ├── backends │ │ ├── __init__.py │ │ ├── impl_blocking.py │ │ ├── impl_executor.py │ │ └── impl_nonblocking.py │ └── base.py ├── contrib │ └── __init__.py ├── deciders.py ├── engines │ ├── __init__.py │ ├── action_engine │ │ ├── __init__.py │ │ ├── actions │ │ │ ├── __init__.py │ │ │ ├── base.py │ │ │ ├── retry.py │ │ │ └── task.py │ │ ├── builder.py │ │ ├── compiler.py │ │ ├── completer.py │ │ ├── deciders.py │ │ ├── engine.py │ │ ├── executor.py │ │ ├── runtime.py │ │ ├── scheduler.py │ │ ├── scopes.py │ │ ├── selector.py │ │ └── traversal.py │ ├── base.py │ ├── helpers.py │ └── worker_based │ │ ├── __init__.py │ │ ├── dispatcher.py │ │ ├── endpoint.py │ │ ├── engine.py │ │ ├── executor.py │ │ ├── protocol.py │ │ ├── proxy.py │ │ ├── server.py │ │ ├── types.py │ │ └── worker.py ├── examples │ ├── 99_bottles.py │ ├── alphabet_soup.py │ ├── build_a_car.py │ ├── buildsystem.py │ ├── calculate_in_parallel.py │ ├── calculate_linear.py │ ├── create_parallel_volume.py │ ├── delayed_return.py │ ├── distance_calculator.py │ ├── dump_memory_backend.py │ ├── echo_listener.py │ ├── example_utils.py │ ├── fake_billing.py │ ├── graph_flow.py │ ├── hello_world.py │ ├── jobboard_produce_consume_colors.py │ ├── parallel_table_multiply.py │ ├── persistence_example.py │ ├── pseudo_scoping.out.txt │ ├── pseudo_scoping.py │ ├── resume_from_backend.out.txt │ ├── resume_from_backend.py │ ├── resume_many_flows.out.txt │ ├── resume_many_flows.py │ ├── resume_many_flows │ │ ├── my_flows.py │ │ ├── resume_all.py │ │ └── run_flow.py │ ├── resume_vm_boot.py │ ├── resume_volume_create.py │ ├── retry_flow.out.txt │ ├── retry_flow.py │ ├── reverting_linear.out.txt │ ├── reverting_linear.py │ ├── run_by_iter.out.txt │ ├── run_by_iter.py │ ├── run_by_iter_enumerate.out.txt │ ├── run_by_iter_enumerate.py │ ├── share_engine_thread.py │ ├── simple_linear.out.txt │ ├── simple_linear.py │ ├── simple_linear_listening.out.txt │ ├── simple_linear_listening.py │ ├── simple_linear_pass.out.txt │ ├── simple_linear_pass.py │ ├── simple_map_reduce.py │ ├── switch_graph_flow.py │ ├── timing_listener.py │ ├── tox_conductor.py │ ├── wbe_event_sender.py │ ├── wbe_mandelbrot.out.txt │ ├── wbe_mandelbrot.py │ ├── wbe_simple_linear.out.txt │ ├── wbe_simple_linear.py │ └── wrapped_exception.py ├── exceptions.py ├── flow.py ├── formatters.py ├── jobs │ ├── __init__.py │ ├── backends │ │ ├── __init__.py │ │ ├── impl_etcd.py │ │ ├── impl_redis.py │ │ └── impl_zookeeper.py │ └── base.py ├── listeners │ ├── __init__.py │ ├── base.py │ ├── capturing.py │ ├── claims.py │ ├── logging.py │ ├── printing.py │ └── timing.py ├── logging.py ├── patterns │ ├── __init__.py │ ├── graph_flow.py │ ├── linear_flow.py │ └── unordered_flow.py ├── persistence │ ├── __init__.py │ ├── backends │ │ ├── __init__.py │ │ ├── impl_dir.py │ │ ├── impl_memory.py │ │ ├── impl_sqlalchemy.py │ │ ├── impl_zookeeper.py │ │ └── sqlalchemy │ │ │ ├── __init__.py │ │ │ ├── alembic │ │ │ ├── README │ │ │ ├── alembic.ini │ │ │ ├── env.py │ │ │ ├── script.py.mako │ │ │ └── versions │ │ │ │ ├── 00af93df9d77_add_unique_into_all_indexes.py │ │ │ │ ├── 0bc3e1a3c135_set_result_meduimtext_type.py │ │ │ │ ├── 14b227d79a87_add_intention_column.py │ │ │ │ ├── 1c783c0c2875_replace_exception_an.py │ │ │ │ ├── 1cea328f0f65_initial_logbook_deta.py │ │ │ │ ├── 2ad4984f2864_switch_postgres_to_json_native.py │ │ │ │ ├── 3162c0f3f8e4_add_revert_results_and_revert_failure_.py │ │ │ │ ├── 40fc8c914bd2_fix_atomdetails_failure_size.py │ │ │ │ ├── 589dccdf2b6e_rename_taskdetails_to_atomdetails.py │ │ │ │ ├── 6df9422fcb43_fix_flowdetails_meta_size.py │ │ │ │ ├── 84d6e888850_add_task_detail_type.py │ │ │ │ └── README │ │ │ ├── migration.py │ │ │ └── tables.py │ ├── base.py │ ├── models.py │ └── path_based.py ├── retry.py ├── states.py ├── storage.py ├── task.py ├── test.py ├── tests │ ├── __init__.py │ ├── fixtures.py │ ├── test_examples.py │ ├── unit │ │ ├── __init__.py │ │ ├── action_engine │ │ │ ├── __init__.py │ │ │ ├── test_builder.py │ │ │ ├── test_compile.py │ │ │ ├── test_creation.py │ │ │ └── test_scoping.py │ │ ├── jobs │ │ │ ├── __init__.py │ │ │ ├── base.py │ │ │ ├── test_entrypoint.py │ │ │ ├── test_etcd_job.py │ │ │ ├── test_redis_job.py │ │ │ └── test_zk_job.py │ │ ├── patterns │ │ │ ├── __init__.py │ │ │ ├── test_graph_flow.py │ │ │ ├── test_linear_flow.py │ │ │ └── test_unordered_flow.py │ │ ├── persistence │ │ │ ├── __init__.py │ │ │ ├── base.py │ │ │ ├── test_dir_persistence.py │ │ │ ├── test_memory_persistence.py │ │ │ ├── test_sql_persistence.py │ │ │ └── test_zk_persistence.py │ │ ├── test_arguments_passing.py │ │ ├── test_check_transition.py │ │ ├── test_conductors.py │ │ ├── test_deciders.py │ │ ├── test_engine_helpers.py │ │ ├── test_engines.py │ │ ├── test_exceptions.py │ │ ├── test_failure.py │ │ ├── test_flow_dependencies.py │ │ ├── test_formatters.py │ │ ├── test_functor_task.py │ │ ├── test_listeners.py │ │ ├── test_mapfunctor_task.py │ │ ├── test_notifier.py │ │ ├── test_progress.py │ │ ├── test_reducefunctor_task.py │ │ ├── test_retries.py │ │ ├── test_states.py │ │ ├── test_storage.py │ │ ├── test_suspend.py │ │ ├── test_task.py │ │ ├── test_types.py │ │ ├── test_utils.py │ │ ├── test_utils_async_utils.py │ │ ├── test_utils_binary.py │ │ ├── test_utils_iter_utils.py │ │ ├── test_utils_kazoo_utils.py │ │ ├── test_utils_threading_utils.py │ │ └── worker_based │ │ │ ├── __init__.py │ │ │ ├── test_creation.py │ │ │ ├── test_dispatcher.py │ │ │ ├── test_endpoint.py │ │ │ ├── test_executor.py │ │ │ ├── test_message_pump.py │ │ │ ├── test_pipeline.py │ │ │ ├── test_protocol.py │ │ │ ├── test_proxy.py │ │ │ ├── test_server.py │ │ │ ├── test_types.py │ │ │ └── test_worker.py │ └── utils.py ├── types │ ├── __init__.py │ ├── entity.py │ ├── failure.py │ ├── graph.py │ ├── latch.py │ ├── notifier.py │ ├── sets.py │ ├── timing.py │ └── tree.py ├── utils │ ├── __init__.py │ ├── async_utils.py │ ├── banner.py │ ├── eventlet_utils.py │ ├── iter_utils.py │ ├── kazoo_utils.py │ ├── kombu_utils.py │ ├── misc.py │ ├── persistence_utils.py │ ├── redis_utils.py │ ├── schema_utils.py │ └── threading_utils.py └── version.py ├── test-requirements.txt ├── tools ├── clear_zk.sh ├── env_builder.sh ├── pretty_tox.sh ├── schema_generator.py ├── speed_test.py ├── state_graph.py ├── subunit_trace.py ├── test-setup.sh └── update_states.sh └── tox.ini /.coveragerc: -------------------------------------------------------------------------------- 1 | [run] 2 | branch = True 3 | source = taskflow 4 | omit = taskflow/tests/*,taskflow/test.py 5 | 6 | [report] 7 | ignore_errors = True 8 | 9 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.py[cod] 2 | 3 | # C extensions 4 | *.so 5 | 6 | # Packages 7 | *.egg 8 | *.eggs 9 | *.egg-info 10 | dist 11 | build 12 | eggs 13 | parts 14 | bin 15 | var 16 | sdist 17 | develop-eggs 18 | .installed.cfg 19 | lib 20 | lib64 21 | 22 | # Installer logs 23 | pip-log.txt 24 | 25 | # Unit test / coverage reports 26 | .coverage* 27 | .diagram-tools/* 28 | .tox 29 | nosetests.xml 30 | .venv 31 | cover 32 | .stestr/ 33 | htmlcov 34 | 35 | # Translations 36 | *.mo 37 | 38 | # Mr Developer 39 | .mr.developer.cfg 40 | .project 41 | .pydevproject 42 | .settings 43 | 44 | # DS_STORE 45 | .DS_Store 46 | 47 | # Sqlite databases 48 | *.sqlite 49 | 50 | # Modified Files 51 | *.swp 52 | 53 | # PBR 54 | build 55 | AUTHORS 56 | ChangeLog 57 | 58 | # doc 59 | doc/build/ 60 | .diagram-tools/ 61 | 62 | .idea 63 | env 64 | 65 | # files created by releasenotes build 66 | RELEASENOTES.rst 67 | releasenotes/notes/reno.cache 68 | releasenotes/build 69 | 70 | # Generated by etcd 71 | etcd-v* 72 | default.etcd 73 | -------------------------------------------------------------------------------- /.gitreview: -------------------------------------------------------------------------------- 1 | [gerrit] 2 | host=review.opendev.org 3 | port=29418 4 | project=openstack/taskflow.git 5 | -------------------------------------------------------------------------------- /.mailmap: -------------------------------------------------------------------------------- 1 | Anastasia Karpinska 2 | Angus Salkeld 3 | Changbin Liu 4 | Changbin Liu 5 | Ivan A. Melnikov 6 | Jessica Lucci 7 | Jessica Lucci 8 | Joshua Harlow 9 | Joshua Harlow 10 | Kevin Chen 11 | Kevin Chen 12 | Kevin Chen 13 | -------------------------------------------------------------------------------- /.pre-commit-config.yaml: -------------------------------------------------------------------------------- 1 | repos: 2 | - repo: https://github.com/pre-commit/pre-commit-hooks 3 | rev: v5.0.0 4 | hooks: 5 | - id: trailing-whitespace 6 | # Replaces or checks mixed line ending 7 | - id: mixed-line-ending 8 | args: ['--fix', 'lf'] 9 | exclude: '.*\.(svg)$' 10 | # Forbid files which have a UTF-8 byte-order marker 11 | - id: check-byte-order-marker 12 | # Checks that non-binary executables have a proper shebang 13 | - id: check-executables-have-shebangs 14 | # Check for files that contain merge conflict strings. 15 | - id: check-merge-conflict 16 | # Check for debugger imports and py37+ breakpoint() 17 | # calls in python source 18 | - id: debug-statements 19 | - id: check-yaml 20 | files: .*\.(yaml|yml)$ 21 | - repo: https://opendev.org/openstack/hacking 22 | rev: 7.0.0 23 | hooks: 24 | - id: hacking 25 | additional_dependencies: [] 26 | exclude: '^(doc|releasenotes|tools)/.*$' 27 | - repo: https://github.com/asottile/pyupgrade 28 | rev: v3.18.0 29 | hooks: 30 | - id: pyupgrade 31 | args: [--py3-only] 32 | -------------------------------------------------------------------------------- /.pylintrc: -------------------------------------------------------------------------------- 1 | [MESSAGES CONTROL] 2 | 3 | # Disable the message(s) with the given id(s). 4 | disable=C0111,I0011,R0201,R0922,W0142,W0511,W0613,W0622,W0703 5 | 6 | [BASIC] 7 | 8 | # Variable names can be 1 to 31 characters long, with lowercase and underscores 9 | variable-rgx=[a-z_][a-z0-9_]{0,30}$ 10 | 11 | # Argument names can be 2 to 31 characters long, with lowercase and underscores 12 | argument-rgx=[a-z_][a-z0-9_]{1,30}$ 13 | 14 | # Method names should be at least 3 characters long 15 | # and be lowercased with underscores 16 | method-rgx=[a-z_][a-z0-9_]{2,50}$ 17 | 18 | # Don't require docstrings on tests. 19 | no-docstring-rgx=((__.*__)|([tT]est.*)|setUp|tearDown)$ 20 | 21 | [DESIGN] 22 | max-args=10 23 | max-attributes=20 24 | max-branchs=30 25 | max-public-methods=100 26 | max-statements=60 27 | min-public-methods=0 28 | 29 | [REPORTS] 30 | output-format=parseable 31 | include-ids=yes 32 | 33 | [VARIABLES] 34 | additional-builtins=_ 35 | -------------------------------------------------------------------------------- /.stestr.conf: -------------------------------------------------------------------------------- 1 | [DEFAULT] 2 | test_path=./taskflow/tests/unit 3 | top_dir=. 4 | -------------------------------------------------------------------------------- /.zuul.yaml: -------------------------------------------------------------------------------- 1 | - job: 2 | name: taskflow-functional 3 | parent: openstack-tox 4 | abstract: true 5 | pre-run: playbooks/tests/functional/pre.yml 6 | vars: 7 | tox_envlist: functional 8 | irrelevant-files: 9 | - ^\.gitreview$ 10 | - ^.*\.rst$ 11 | - ^doc/.*$ 12 | - ^LICENSE$ 13 | - ^releasenotes/.*$ 14 | - ^\.pre-commit-config\.yaml$ 15 | 16 | - job: 17 | name: taskflow-functional-redis 18 | parent: taskflow-functional 19 | vars: 20 | tox_environment: 21 | PIFPAF_DAEMON: redis 22 | 23 | - job: 24 | name: taskflow-functional-etcd 25 | parent: taskflow-functional 26 | vars: 27 | tox_environment: 28 | PIFPAF_DAEMON: etcd 29 | SETUP_ENV_SCRIPT: ./setup-etcd-env.sh 30 | 31 | - project: 32 | templates: 33 | - check-requirements 34 | - lib-forward-testing-python3 35 | - openstack-cover-jobs 36 | - openstack-python3-jobs 37 | - periodic-stable-jobs 38 | - publish-openstack-docs-pti 39 | - release-notes-jobs-python3 40 | check: 41 | jobs: 42 | - taskflow-functional-redis 43 | - taskflow-functional-etcd 44 | gate: 45 | jobs: 46 | - taskflow-functional-redis 47 | - taskflow-functional-etcd 48 | -------------------------------------------------------------------------------- /CONTRIBUTING.rst: -------------------------------------------------------------------------------- 1 | If you would like to contribute to the development of OpenStack, 2 | you must follow the steps documented at: 3 | 4 | https://docs.openstack.org/infra/manual/developers.html#development-workflow 5 | 6 | Once those steps have been completed, changes to OpenStack 7 | should be submitted for review via the Gerrit tool, following 8 | the workflow documented at: 9 | 10 | https://docs.openstack.org/infra/manual/developers.html#development-workflow 11 | 12 | Pull requests submitted through GitHub will be ignored. 13 | 14 | Bugs should be filed on Launchpad, not GitHub: 15 | 16 | https://bugs.launchpad.net/taskflow 17 | 18 | The mailing list is (prefix subjects with "[Oslo][TaskFlow]"): 19 | 20 | https://lists.openstack.org/cgi-bin/mailman/listinfo/openstack-discuss 21 | 22 | Questions and discussions take place in #openstack-oslo on irc.OFTC.net. 23 | -------------------------------------------------------------------------------- /README.rst: -------------------------------------------------------------------------------- 1 | ======================== 2 | Team and repository tags 3 | ======================== 4 | 5 | .. image:: https://governance.openstack.org/tc/badges/taskflow.svg 6 | :target: https://governance.openstack.org/tc/reference/tags/index.html 7 | 8 | .. Change things from this point on 9 | 10 | TaskFlow 11 | ======== 12 | 13 | .. image:: https://img.shields.io/pypi/v/taskflow.svg 14 | :target: https://pypi.org/project/taskflow/ 15 | :alt: Latest Version 16 | 17 | A library to do [jobs, tasks, flows] in a highly available, easy to understand 18 | and declarative manner (and more!) to be used with OpenStack and other 19 | projects. 20 | 21 | * Free software: Apache license 22 | * Documentation: https://docs.openstack.org/taskflow/latest/ 23 | * Source: https://opendev.org/openstack/taskflow 24 | * Bugs: https://bugs.launchpad.net/taskflow/ 25 | * Release notes: https://docs.openstack.org/releasenotes/taskflow/ 26 | 27 | Join us 28 | ------- 29 | 30 | - https://launchpad.net/taskflow 31 | 32 | Testing and requirements 33 | ------------------------ 34 | 35 | Requirements 36 | ~~~~~~~~~~~~ 37 | 38 | Because this project has many optional (pluggable) parts like persistence 39 | backends and engines, we decided to split our requirements into two 40 | parts: - things that are absolutely required (you can't use the project 41 | without them) are put into ``requirements.txt``. The requirements 42 | that are required by some optional part of this project (you can use the 43 | project without them) are put into our ``test-requirements.txt`` file (so 44 | that we can still test the optional functionality works as expected). If 45 | you want to use the feature in question (`eventlet`_ or the worker based engine 46 | that uses `kombu`_ or the `sqlalchemy`_ persistence backend or jobboards which 47 | have an implementation built using `kazoo`_ ...), you should add 48 | that requirement(s) to your project or environment. 49 | 50 | Tox.ini 51 | ~~~~~~~ 52 | 53 | Our ``tox.ini`` file describes several test environments that allow to test 54 | TaskFlow with different python versions and sets of requirements installed. 55 | Please refer to the `tox`_ documentation to understand how to make these test 56 | environments work for you. 57 | 58 | Developer documentation 59 | ----------------------- 60 | 61 | We also have sphinx documentation in ``docs/source``. 62 | 63 | *To build it, run:* 64 | 65 | :: 66 | 67 | $ python setup.py build_sphinx 68 | 69 | .. _kazoo: https://kazoo.readthedocs.io/en/latest/ 70 | .. _sqlalchemy: https://www.sqlalchemy.org/ 71 | .. _kombu: https://kombu.readthedocs.io/en/latest/ 72 | .. _eventlet: http://eventlet.net/ 73 | .. _tox: https://tox.testrun.org/ 74 | -------------------------------------------------------------------------------- /bindep.txt: -------------------------------------------------------------------------------- 1 | # This is a cross-platform list tracking distribution packages needed for install and tests; 2 | # see https://docs.openstack.org/infra/bindep/ for additional information. 3 | 4 | graphviz [!platform:gentoo] 5 | media-gfx/graphviz [platform:gentoo] 6 | 7 | mariadb [platform:rpm] 8 | mariadb-server [platform:redhat platform:debian] 9 | mariadb-devel [platform:redhat] 10 | libmariadb-dev-compat [platform:debian] 11 | libmysqlclient-dev [platform:ubuntu] 12 | libmysqlclient-devel [platform:suse] 13 | mysql-client [platform:dpkg !platform:debian] 14 | mysql-server [platform:dpkg !platform:debian] 15 | postgresql 16 | postgresql-client [platform:dpkg] 17 | libpq-dev [platform:dpkg] 18 | 19 | redis [platform:rpm tests-functional-redis] 20 | redis-server [platform:dpkg tests-functional-redis] 21 | redis-sentinel [platform:dpkg tests-functional-redis] 22 | -------------------------------------------------------------------------------- /doc/diagrams/area_of_influence.graffle.tgz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openstack/taskflow/d01920ef1c0e3432c5d3a1e7708a34ff834bf339/doc/diagrams/area_of_influence.graffle.tgz -------------------------------------------------------------------------------- /doc/diagrams/core.graffle.tgz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openstack/taskflow/d01920ef1c0e3432c5d3a1e7708a34ff834bf339/doc/diagrams/core.graffle.tgz -------------------------------------------------------------------------------- /doc/diagrams/jobboard.graffle.tgz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openstack/taskflow/d01920ef1c0e3432c5d3a1e7708a34ff834bf339/doc/diagrams/jobboard.graffle.tgz -------------------------------------------------------------------------------- /doc/diagrams/tasks.graffle.tgz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openstack/taskflow/d01920ef1c0e3432c5d3a1e7708a34ff834bf339/doc/diagrams/tasks.graffle.tgz -------------------------------------------------------------------------------- /doc/diagrams/worker-engine.graffle.tgz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openstack/taskflow/d01920ef1c0e3432c5d3a1e7708a34ff834bf339/doc/diagrams/worker-engine.graffle.tgz -------------------------------------------------------------------------------- /doc/requirements.txt: -------------------------------------------------------------------------------- 1 | sphinx>=2.0.0 # BSD 2 | openstackdocstheme>=2.2.1 # Apache-2.0 3 | reno>=3.1.0 # Apache-2.0 4 | doc8>=0.6.0 # Apache-2.0 5 | -------------------------------------------------------------------------------- /doc/source/conf.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2020 Red Hat, Inc. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 12 | # implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | import datetime 17 | 18 | # -- General configuration ---------------------------------------------------- 19 | 20 | # Add any Sphinx extension module names here, as strings. They can be 21 | # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom ones. 22 | extensions = [ 23 | 'sphinx.ext.autodoc', 24 | 'sphinx.ext.doctest', 25 | 'sphinx.ext.extlinks', 26 | 'sphinx.ext.inheritance_diagram', 27 | 'sphinx.ext.viewcode', 28 | 'openstackdocstheme' 29 | ] 30 | 31 | # openstackdocstheme options 32 | openstackdocs_repo_name = 'openstack/taskflow' 33 | openstackdocs_auto_name = False 34 | openstackdocs_bug_project = 'taskflow' 35 | openstackdocs_bug_tag = '' 36 | 37 | # Add any paths that contain templates here, relative to this directory. 38 | templates_path = ['templates'] 39 | 40 | # The suffix of source filenames. 41 | source_suffix = '.rst' 42 | 43 | # The master toctree document. 44 | master_doc = 'index' 45 | 46 | # List of patterns, relative to source directory, that match files and 47 | # directories to ignore when looking for source files. 48 | exclude_patterns = ['_build'] 49 | 50 | # General information about the project. 51 | project = 'TaskFlow' 52 | copyright = '%s, OpenStack Foundation' % datetime.date.today().year 53 | 54 | # If true, '()' will be appended to :func: etc. cross-reference text. 55 | add_function_parentheses = True 56 | 57 | # If true, the current module name will be prepended to all description 58 | # unit titles (such as .. function::). 59 | add_module_names = True 60 | 61 | # The name of the Pygments (syntax highlighting) style to use. 62 | pygments_style = 'native' 63 | 64 | # Prefixes that are ignored for sorting the Python module index 65 | modindex_common_prefix = ['taskflow.'] 66 | 67 | # Shortened external links. 68 | source_tree = 'https://opendev.org/openstack/taskflow/src/branch/master/' 69 | extlinks = { 70 | 'example': (source_tree + '/taskflow/examples/%s.py', '%s'), 71 | 'pybug': ('http://bugs.python.org/issue%s', '%s'), 72 | } 73 | 74 | 75 | # -- Options for HTML output -------------------------------------------------- 76 | 77 | # The theme to use for HTML and HTML Help pages. Major themes that come with 78 | # Sphinx are currently 'default' and 'sphinxdoc'. 79 | # html_theme_path = ["."] 80 | html_theme = 'openstackdocs' 81 | 82 | 83 | # -- Options for autoddoc ---------------------------------------------------- 84 | 85 | # Keep source order 86 | autodoc_member_order = 'bysource' 87 | 88 | # Always include members 89 | autodoc_default_options = {'members': None, 'show-inheritance': None} 90 | -------------------------------------------------------------------------------- /doc/source/index.rst: -------------------------------------------------------------------------------- 1 | ========== 2 | TaskFlow 3 | ========== 4 | 5 | *TaskFlow is a Python library that helps to make task execution easy, 6 | consistent and reliable.* [#f1]_ 7 | 8 | .. note:: 9 | 10 | If you are just getting started or looking for an overview please 11 | visit: https://wiki.openstack.org/wiki/TaskFlow which provides better 12 | introductory material, description of high level goals and related content. 13 | 14 | .. toctree:: 15 | :maxdepth: 2 16 | 17 | user/index 18 | 19 | Release Notes 20 | ============= 21 | 22 | Read also the `taskflow Release Notes 23 | `_. 24 | 25 | Indices and tables 26 | ================== 27 | 28 | * :ref:`genindex` 29 | * :ref:`modindex` 30 | * :ref:`search` 31 | 32 | .. [#f1] It should be noted that even though it is designed with OpenStack 33 | integration in mind, and that is where most of its *current* 34 | integration is it aims to be generally usable and useful in any 35 | project. 36 | -------------------------------------------------------------------------------- /doc/source/templates/layout.html: -------------------------------------------------------------------------------- 1 | {% extends "!layout.html" %} 2 | {% block sidebarrel %} 3 |

{{ _('Navigation')}}

4 | 15 | {% endblock %} 16 | -------------------------------------------------------------------------------- /doc/source/user/conductors.rst: -------------------------------------------------------------------------------- 1 | ---------- 2 | Conductors 3 | ---------- 4 | 5 | .. image:: img/conductor.png 6 | :width: 97px 7 | :alt: Conductor 8 | 9 | Overview 10 | ======== 11 | 12 | Conductors provide a mechanism that unifies the various 13 | concepts under a single easy to use (as plug-and-play as we can make it) 14 | construct. 15 | 16 | They are responsible for the following: 17 | 18 | * Interacting with :doc:`jobboards ` (examining and claiming 19 | :doc:`jobs `). 20 | * Creating :doc:`engines ` from the claimed jobs (using 21 | :ref:`factories ` to reconstruct the contained 22 | tasks and flows to be executed). 23 | * Dispatching the engine using the provided :doc:`persistence ` 24 | layer and engine configuration. 25 | * Completing or abandoning the claimed :doc:`job ` (depending on 26 | dispatching and execution outcome). 27 | * *Rinse and repeat*. 28 | 29 | .. note:: 30 | 31 | They are inspired by and have similar responsibilities 32 | as `railroad conductors`_ or `musical conductors`_. 33 | 34 | Considerations 35 | ============== 36 | 37 | Some usage considerations should be used when using a conductor to make sure 38 | it's used in a safe and reliable manner. Eventually we hope to make these 39 | non-issues but for now they are worth mentioning. 40 | 41 | Endless cycling 42 | --------------- 43 | 44 | **What:** Jobs that fail (due to some type of internal error) on one conductor 45 | will be abandoned by that conductor and then another conductor may experience 46 | those same errors and abandon it (and repeat). This will create a job 47 | abandonment cycle that will continue for as long as the job exists in an 48 | claimable state. 49 | 50 | **Example:** 51 | 52 | .. image:: img/conductor_cycle.png 53 | :scale: 70% 54 | :alt: Conductor cycling 55 | 56 | **Alleviate by:** 57 | 58 | #. Forcefully delete jobs that have been failing continuously after a given 59 | number of conductor attempts. This can be either done manually or 60 | automatically via scripts (or other associated monitoring) or via 61 | the jobboards :py:func:`~taskflow.jobs.base.JobBoard.trash` method. 62 | #. Resolve the internal error's cause (storage backend failure, other...). 63 | 64 | Interfaces 65 | ========== 66 | 67 | .. automodule:: taskflow.conductors.base 68 | .. automodule:: taskflow.conductors.backends 69 | .. automodule:: taskflow.conductors.backends.impl_executor 70 | 71 | Implementations 72 | =============== 73 | 74 | Blocking 75 | -------- 76 | 77 | .. automodule:: taskflow.conductors.backends.impl_blocking 78 | 79 | Non-blocking 80 | ------------ 81 | 82 | .. automodule:: taskflow.conductors.backends.impl_nonblocking 83 | 84 | Hierarchy 85 | ========= 86 | 87 | .. inheritance-diagram:: 88 | taskflow.conductors.base 89 | taskflow.conductors.backends.impl_blocking 90 | taskflow.conductors.backends.impl_nonblocking 91 | taskflow.conductors.backends.impl_executor 92 | :parts: 1 93 | 94 | .. _musical conductors: http://en.wikipedia.org/wiki/Conducting 95 | .. _railroad conductors: http://en.wikipedia.org/wiki/Conductor_%28transportation%29 96 | -------------------------------------------------------------------------------- /doc/source/user/exceptions.rst: -------------------------------------------------------------------------------- 1 | ---------- 2 | Exceptions 3 | ---------- 4 | 5 | .. inheritance-diagram:: 6 | taskflow.exceptions 7 | :parts: 1 8 | 9 | .. automodule:: taskflow.exceptions 10 | -------------------------------------------------------------------------------- /doc/source/user/history.rst: -------------------------------------------------------------------------------- 1 | .. include:: ../../../ChangeLog 2 | 3 | -------------------------------------------------------------------------------- /doc/source/user/img/conductor.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openstack/taskflow/d01920ef1c0e3432c5d3a1e7708a34ff834bf339/doc/source/user/img/conductor.png -------------------------------------------------------------------------------- /doc/source/user/img/conductor_cycle.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openstack/taskflow/d01920ef1c0e3432c5d3a1e7708a34ff834bf339/doc/source/user/img/conductor_cycle.png -------------------------------------------------------------------------------- /doc/source/user/img/distributed_flow_rpc.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openstack/taskflow/d01920ef1c0e3432c5d3a1e7708a34ff834bf339/doc/source/user/img/distributed_flow_rpc.png -------------------------------------------------------------------------------- /doc/source/user/img/jobboard.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openstack/taskflow/d01920ef1c0e3432c5d3a1e7708a34ff834bf339/doc/source/user/img/jobboard.png -------------------------------------------------------------------------------- /doc/source/user/img/mandelbrot.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openstack/taskflow/d01920ef1c0e3432c5d3a1e7708a34ff834bf339/doc/source/user/img/mandelbrot.png -------------------------------------------------------------------------------- /doc/source/user/img/tasks.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openstack/taskflow/d01920ef1c0e3432c5d3a1e7708a34ff834bf339/doc/source/user/img/tasks.png -------------------------------------------------------------------------------- /doc/source/user/index.rst: -------------------------------------------------------------------------------- 1 | ================ 2 | Using TaskFlow 3 | ================ 4 | 5 | Considerations 6 | ============== 7 | 8 | Things to consider before (and during) development and integration with 9 | TaskFlow into your project: 10 | 11 | * Read over the `paradigm shifts`_ and engage the team in `IRC`_ (or via the 12 | `openstack-dev`_ mailing list) if these need more explanation (prefix 13 | ``[Oslo][TaskFlow]`` to your emails subject to get an even faster 14 | response). 15 | * Follow (or at least attempt to follow) some of the established 16 | `best practices`_ (feel free to add your own suggested best practices). 17 | * Keep in touch with the team (see above); we are all friendly and enjoy 18 | knowing your use cases and learning how we can help make your lives easier 19 | by adding or adjusting functionality in this library. 20 | 21 | .. _IRC: irc://irc.oftc.net/openstack-oslo 22 | .. _best practices: https://wiki.openstack.org/wiki/TaskFlow/Best_practices 23 | .. _paradigm shifts: https://wiki.openstack.org/wiki/TaskFlow/Paradigm_shifts 24 | .. _openstack-dev: mailto:openstack-dev@lists.openstack.org 25 | 26 | User Guide 27 | ========== 28 | 29 | .. toctree:: 30 | :maxdepth: 2 31 | 32 | atoms 33 | arguments_and_results 34 | inputs_and_outputs 35 | 36 | patterns 37 | engines 38 | workers 39 | notifications 40 | persistence 41 | resumption 42 | 43 | jobs 44 | conductors 45 | 46 | examples 47 | 48 | Miscellaneous 49 | ============= 50 | 51 | .. toctree:: 52 | :maxdepth: 2 53 | 54 | exceptions 55 | states 56 | types 57 | utils 58 | 59 | Bookshelf 60 | ========= 61 | 62 | A useful collection of links, documents, papers, similar 63 | projects, frameworks and libraries. 64 | 65 | .. note:: 66 | 67 | Please feel free to submit your own additions and/or changes. 68 | 69 | .. toctree:: 70 | :maxdepth: 1 71 | 72 | shelf 73 | 74 | Release notes 75 | ============= 76 | 77 | .. toctree:: 78 | :maxdepth: 2 79 | 80 | history 81 | -------------------------------------------------------------------------------- /doc/source/user/patterns.rst: -------------------------------------------------------------------------------- 1 | -------- 2 | Patterns 3 | -------- 4 | 5 | .. automodule:: taskflow.flow 6 | 7 | 8 | Linear flow 9 | ~~~~~~~~~~~ 10 | 11 | .. automodule:: taskflow.patterns.linear_flow 12 | 13 | 14 | Unordered flow 15 | ~~~~~~~~~~~~~~ 16 | 17 | .. automodule:: taskflow.patterns.unordered_flow 18 | 19 | 20 | Graph flow 21 | ~~~~~~~~~~ 22 | 23 | .. automodule:: taskflow.patterns.graph_flow 24 | .. automodule:: taskflow.deciders 25 | 26 | Hierarchy 27 | ~~~~~~~~~ 28 | 29 | .. inheritance-diagram:: 30 | taskflow.flow 31 | taskflow.patterns.linear_flow 32 | taskflow.patterns.unordered_flow 33 | taskflow.patterns.graph_flow 34 | :parts: 2 35 | -------------------------------------------------------------------------------- /doc/source/user/shelf.rst: -------------------------------------------------------------------------------- 1 | Libraries & frameworks 2 | ---------------------- 3 | 4 | * `APScheduler`_ (Python) 5 | * `Async`_ (Python) 6 | * `Celery`_ (Python) 7 | * `Graffiti`_ (Python) 8 | * `JobLib`_ (Python) 9 | * `Luigi`_ (Python) 10 | * `Mesos`_ (C/C++) 11 | * `Papy`_ (Python) 12 | * `Parallel Python`_ (Python) 13 | * `RQ`_ (Python) 14 | * `Spiff`_ (Python) 15 | * `TBB Flow`_ (C/C++) 16 | 17 | Languages 18 | --------- 19 | 20 | * `Ani`_ 21 | * `Make`_ 22 | * `Plaid`_ 23 | 24 | Services 25 | -------- 26 | 27 | * `Cloud Dataflow`_ 28 | * `Mistral`_ 29 | 30 | Papers 31 | ------ 32 | 33 | * `Advances in Dataflow Programming Languages`_ 34 | 35 | Related paradigms 36 | ----------------- 37 | 38 | * `Dataflow programming`_ 39 | * `Programming paradigm(s)`_ 40 | 41 | .. _APScheduler: http://pythonhosted.org/APScheduler/ 42 | .. _Async: http://pypi.python.org/pypi/async 43 | .. _Celery: http://www.celeryproject.org/ 44 | .. _Graffiti: http://github.com/SegFaultAX/graffiti 45 | .. _JobLib: http://pythonhosted.org/joblib/index.html 46 | .. _Luigi: http://github.com/spotify/luigi 47 | .. _RQ: http://python-rq.org/ 48 | .. _Mistral: http://wiki.openstack.org/wiki/Mistral 49 | .. _Mesos: http://mesos.apache.org/ 50 | .. _Parallel Python: http://www.parallelpython.com/ 51 | .. _Spiff: http://github.com/knipknap/SpiffWorkflow 52 | .. _Papy: http://code.google.com/p/papy/ 53 | .. _Make: http://www.gnu.org/software/make/ 54 | .. _Ani: http://code.google.com/p/anic/ 55 | .. _Programming paradigm(s): http://en.wikipedia.org/wiki/Programming_paradigm 56 | .. _Plaid: http://www.cs.cmu.edu/~aldrich/plaid/ 57 | .. _Advances in Dataflow Programming Languages: http://www.cs.ucf.edu/~dcm/Teaching/COT4810-Spring2011/Literature/DataFlowProgrammingLanguages.pdf 58 | .. _Cloud Dataflow: https://cloud.google.com/dataflow/ 59 | .. _TBB Flow: https://www.threadingbuildingblocks.org/tutorial-intel-tbb-flow-graph 60 | .. _Dataflow programming: http://en.wikipedia.org/wiki/Dataflow_programming 61 | -------------------------------------------------------------------------------- /doc/source/user/types.rst: -------------------------------------------------------------------------------- 1 | ----- 2 | Types 3 | ----- 4 | 5 | .. note:: 6 | 7 | Even though these types **are** made for public consumption and usage 8 | should be encouraged/easily possible it should be noted that these may be 9 | moved out to new libraries at various points in the future. If you are 10 | using these types **without** using the rest of this library it is 11 | **strongly** encouraged that you be a vocal proponent of getting these made 12 | into *isolated* libraries (as using these types in this manner is not 13 | the expected and/or desired usage). 14 | 15 | Entity 16 | ====== 17 | 18 | .. automodule:: taskflow.types.entity 19 | 20 | Failure 21 | ======= 22 | 23 | .. automodule:: taskflow.types.failure 24 | 25 | Graph 26 | ===== 27 | 28 | .. automodule:: taskflow.types.graph 29 | 30 | Notifier 31 | ======== 32 | 33 | .. automodule:: taskflow.types.notifier 34 | :special-members: __call__ 35 | 36 | Sets 37 | ==== 38 | 39 | .. automodule:: taskflow.types.sets 40 | 41 | Timing 42 | ====== 43 | 44 | .. automodule:: taskflow.types.timing 45 | 46 | Tree 47 | ==== 48 | 49 | .. automodule:: taskflow.types.tree 50 | 51 | -------------------------------------------------------------------------------- /doc/source/user/utils.rst: -------------------------------------------------------------------------------- 1 | --------- 2 | Utilities 3 | --------- 4 | 5 | .. warning:: 6 | 7 | External usage of internal utility functions and modules should be kept 8 | to a **minimum** as they may be altered, refactored or moved to other 9 | locations **without** notice (and without the typical deprecation cycle). 10 | 11 | Async 12 | ~~~~~ 13 | 14 | .. automodule:: taskflow.utils.async_utils 15 | 16 | Banner 17 | ~~~~~~ 18 | 19 | .. automodule:: taskflow.utils.banner 20 | 21 | Eventlet 22 | ~~~~~~~~ 23 | 24 | .. automodule:: taskflow.utils.eventlet_utils 25 | 26 | Iterators 27 | ~~~~~~~~~ 28 | 29 | .. automodule:: taskflow.utils.iter_utils 30 | 31 | Kazoo 32 | ~~~~~ 33 | 34 | .. automodule:: taskflow.utils.kazoo_utils 35 | 36 | Kombu 37 | ~~~~~ 38 | 39 | .. automodule:: taskflow.utils.kombu_utils 40 | 41 | Miscellaneous 42 | ~~~~~~~~~~~~~ 43 | 44 | .. automodule:: taskflow.utils.misc 45 | 46 | Persistence 47 | ~~~~~~~~~~~ 48 | 49 | .. automodule:: taskflow.utils.persistence_utils 50 | 51 | Redis 52 | ~~~~~ 53 | 54 | .. automodule:: taskflow.utils.redis_utils 55 | 56 | Schema 57 | ~~~~~~ 58 | 59 | .. automodule:: taskflow.utils.schema_utils 60 | 61 | Threading 62 | ~~~~~~~~~ 63 | 64 | .. automodule:: taskflow.utils.threading_utils 65 | -------------------------------------------------------------------------------- /playbooks/tests/functional/Debian.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | backend_services_map: 3 | redis: 4 | - redis-server 5 | - redis-sentinel 6 | etcd: [] 7 | -------------------------------------------------------------------------------- /playbooks/tests/functional/RedHat.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | backend_services_map: 3 | redis: 4 | - redis 5 | - redis-sentinel 6 | etcd: [] 7 | -------------------------------------------------------------------------------- /playbooks/tests/functional/pre.yml: -------------------------------------------------------------------------------- 1 | - hosts: all 2 | vars: 3 | taskflow_backend_daemon: "{{ tox_environment.PIFPAF_DAEMON }}" 4 | roles: 5 | - role: bindep 6 | bindep_profile: "tests-functional-{{ taskflow_backend_daemon }}" 7 | tasks: 8 | - name: Include OS-specific variables 9 | include_vars: "{{ ansible_os_family }}.yaml" 10 | # NOTE(yoctozepto): Debian and Ubuntu have this nasty policy of starting 11 | # installed services for us. We don't rely on system-wide service and use 12 | # pifpaf. Unfortunately, default port may conflict with system-wide service. 13 | # So, for sanity and resource conservation, let's stop it before tests run. 14 | - name: "Stop backend services" 15 | service: 16 | name: "{{ item }}" 17 | state: stopped 18 | enabled: no 19 | become: yes 20 | loop: "{{ backend_services_map[taskflow_backend_daemon] }}" 21 | 22 | -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [build-system] 2 | requires = ["pbr>=6.1.1"] 3 | build-backend = "pbr.build" 4 | -------------------------------------------------------------------------------- /releasenotes/notes/.placeholder: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openstack/taskflow/d01920ef1c0e3432c5d3a1e7708a34ff834bf339/releasenotes/notes/.placeholder -------------------------------------------------------------------------------- /releasenotes/notes/add-sentinel-redis-support-9fd16e2a5dd5c0c9.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | features: 3 | - | 4 | Allow to use Sentinel for Redis connections. New variable *sentinel* can be 5 | passed to Redis jobboard. It is None by default, Sentinel name should be passed 6 | to enable this functionality. 7 | -------------------------------------------------------------------------------- /releasenotes/notes/bug-2056656-871b67ddbc8cfc92.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | fixes: 3 | - | 4 | Redis job board driver no longer uses ``username`` and ``password`` for 5 | its connections to Redis Sentinel, to restore the previous behavior which 6 | was already used by some deployment tools. Add credential to 7 | ``sentinel_kwargs`` to enable authentication for Redis Sentinel. 8 | -------------------------------------------------------------------------------- /releasenotes/notes/deprecate-eventlet-df4a34a7d56acc47.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | deprecations: 3 | - | 4 | Eventlet usages are deprecated and the removal of Eventlet from 5 | OpenStack `is planned `_, 6 | for this reason the eventlet_utils module is now deprecated. 7 | The support of Eventlet will be soon removed from taskflow. 8 | 9 | Please start considering removing your internal Eventlet usages and 10 | start migrating your stack. 11 | -------------------------------------------------------------------------------- /releasenotes/notes/disable-process_executor-python-312-d1074c816bc8303e.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | deprecations: 3 | - | 4 | The process_executor module has been deprecated, starting with Python 3.12. 5 | It is still available in older versions of Python. There is no replacement 6 | for it. 7 | -------------------------------------------------------------------------------- /releasenotes/notes/drop-python-2-7-73d3113c69d724d6.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | upgrade: 3 | - | 4 | Python 2.7 support has been dropped. The minimum version of Python now 5 | supported by taskflow is Python 3.6. 6 | -------------------------------------------------------------------------------- /releasenotes/notes/etcd-jobboard-backend-8a9fea2238fb0f12.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | features: 3 | - | 4 | Added an Etcd-based backend for jobboard. This backend is similar to the 5 | Redis backend, it requires that the consumer extends the expiry of the job 6 | that is being running. 7 | -------------------------------------------------------------------------------- /releasenotes/notes/fix-endless-loop-on-storage-error-dd4467f0bbc66abf.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | fixes: 3 | - | 4 | Limit retries for storage failures on saving flow/task state in the storage. 5 | Previously on StorageFailure exception may cause an endless loop during 6 | execution of flows throwing errors and retrying to save details. 7 | 8 | -------------------------------------------------------------------------------- /releasenotes/notes/fix-endless-loop-on-storage-failures-b98b30f0c34d25e1.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | fixes: 3 | - | 4 | Fixed potential endless loop of exceptions when the storage is down and 5 | Taskflow loads a logbook. 6 | -------------------------------------------------------------------------------- /releasenotes/notes/fix-revert-all-revert-a0310cd7beaa7409.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | fixes: 3 | - | 4 | Fixed a bug when using retries with unordered flows, a REVERT_ALL triggered 5 | by one of the subflow was overriden by an other subflow running in parallel, 6 | leading to an incomplete revert of the flow. 7 | -------------------------------------------------------------------------------- /releasenotes/notes/fix-storage-failure-handling-5c115d92daa0eb82.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | fixes: 3 | - | 4 | Fixed an issue with the handling of exceptions when running a flow, some 5 | jobs may have been incorrectly consumed (and not rescheduled) during 6 | outages. 7 | -------------------------------------------------------------------------------- /releasenotes/notes/fix-zookeeper-option-parsing-f9d37fbc39af47f4.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | fixes: 3 | - | 4 | Fixed an issue when the configuration options of the zookeeper jobboard 5 | backend were passed as strings, the string ''"False"'' was wrongly 6 | interpreted as ''True''. Now the string ''"False"'' is interpreted as the 7 | ''False'' boolean. 8 | -------------------------------------------------------------------------------- /releasenotes/notes/mask-keys-74b9bb5c420d8091.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | features: 3 | - | 4 | Added ``mask_inputs_keys`` and ``mask_outputs_keys`` parameters to the 5 | constructors for ``FailureFormatter`` and ``DynamicLoggingListener`` 6 | that can be used to mask sensitive information from the ``requires`` 7 | and ``provides`` fields respectively when logging a atom. 8 | -------------------------------------------------------------------------------- /releasenotes/notes/redis-username-df0eb33869db09a2.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | features: 3 | - | 4 | The redis jobboard driver now supports the username option, which is 5 | required in authentication request to Redis with ACL enabled. 6 | -------------------------------------------------------------------------------- /releasenotes/notes/remove-process_executor-f59d40a5dd287cd7.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | upgrade: 3 | - | 4 | Process executor was removed. 5 | -------------------------------------------------------------------------------- /releasenotes/notes/remove-py38-15af791146f479e1.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | upgrade: 3 | - | 4 | Support for Python 3.8 has been removed. Now the minimum python version 5 | supported is 3.9 . 6 | -------------------------------------------------------------------------------- /releasenotes/notes/remove-strict-redis-f2a5a924b314de41.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | upgrade: 3 | - | 4 | The minimum redis-py version required is now >= 3.0.0 5 | -------------------------------------------------------------------------------- /releasenotes/notes/sentinel-fallbacks-6fe2ab0d68959cdf.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | features: 3 | - | 4 | The redis driver now supports ``sentinel_fallbacks`` option. This allows 5 | using additional sentinel servers as fallbacks. 6 | -------------------------------------------------------------------------------- /releasenotes/notes/sentinel-ssl-399c56ed7067d282.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | features: 3 | - | 4 | Redis jobboard driver now enables SSL for connections to Redis Sentinel 5 | when SSL is enabled for connections to Redis. 6 | -------------------------------------------------------------------------------- /releasenotes/notes/sentinel-use-redis-creds-63f58b12ad46a2b5.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | features: 3 | - | 4 | Now the redis driver uses the credential for redis servers in connections 5 | to Redis Sentinel servers. 6 | 7 | upgrade: 8 | - | 9 | Now the redis driver uses the same credentials as redis by default. If 10 | a different credentials need to be used, override these via 11 | ``sentinel_kwargs``. 12 | -------------------------------------------------------------------------------- /releasenotes/notes/zookeeper-ssl-support-b9abf24a39096b62.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | features: 3 | - | 4 | SSL support for zookeeper backend (kazoo client). Now the following options 5 | can be passed to zookeeper config: *keyfile*, *keyfile_password*, 6 | *certfile*, *use_ssl*, *verify_certs*. -------------------------------------------------------------------------------- /releasenotes/source/2023.1.rst: -------------------------------------------------------------------------------- 1 | =========================== 2 | 2023.1 Series Release Notes 3 | =========================== 4 | 5 | .. release-notes:: 6 | :branch: unmaintained/2023.1 7 | -------------------------------------------------------------------------------- /releasenotes/source/2023.2.rst: -------------------------------------------------------------------------------- 1 | =========================== 2 | 2023.2 Series Release Notes 3 | =========================== 4 | 5 | .. release-notes:: 6 | :branch: stable/2023.2 7 | -------------------------------------------------------------------------------- /releasenotes/source/2024.1.rst: -------------------------------------------------------------------------------- 1 | =========================== 2 | 2024.1 Series Release Notes 3 | =========================== 4 | 5 | .. release-notes:: 6 | :branch: stable/2024.1 7 | -------------------------------------------------------------------------------- /releasenotes/source/2024.2.rst: -------------------------------------------------------------------------------- 1 | =========================== 2 | 2024.2 Series Release Notes 3 | =========================== 4 | 5 | .. release-notes:: 6 | :branch: stable/2024.2 7 | -------------------------------------------------------------------------------- /releasenotes/source/2025.1.rst: -------------------------------------------------------------------------------- 1 | =========================== 2 | 2025.1 Series Release Notes 3 | =========================== 4 | 5 | .. release-notes:: 6 | :branch: stable/2025.1 7 | -------------------------------------------------------------------------------- /releasenotes/source/_static/.placeholder: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openstack/taskflow/d01920ef1c0e3432c5d3a1e7708a34ff834bf339/releasenotes/source/_static/.placeholder -------------------------------------------------------------------------------- /releasenotes/source/_templates/.placeholder: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openstack/taskflow/d01920ef1c0e3432c5d3a1e7708a34ff834bf339/releasenotes/source/_templates/.placeholder -------------------------------------------------------------------------------- /releasenotes/source/index.rst: -------------------------------------------------------------------------------- 1 | =========================== 2 | taskflow Release Notes 3 | =========================== 4 | 5 | .. toctree:: 6 | :maxdepth: 1 7 | 8 | unreleased 9 | 2025.1 10 | 2024.2 11 | 2024.1 12 | 2023.2 13 | 2023.1 14 | victoria 15 | ussuri 16 | train 17 | stein 18 | rocky 19 | queens 20 | pike 21 | ocata 22 | -------------------------------------------------------------------------------- /releasenotes/source/ocata.rst: -------------------------------------------------------------------------------- 1 | =================================== 2 | Ocata Series Release Notes 3 | =================================== 4 | 5 | .. release-notes:: 6 | :branch: origin/stable/ocata 7 | -------------------------------------------------------------------------------- /releasenotes/source/pike.rst: -------------------------------------------------------------------------------- 1 | =================================== 2 | Pike Series Release Notes 3 | =================================== 4 | 5 | .. release-notes:: 6 | :branch: stable/pike 7 | -------------------------------------------------------------------------------- /releasenotes/source/queens.rst: -------------------------------------------------------------------------------- 1 | =================================== 2 | Queens Series Release Notes 3 | =================================== 4 | 5 | .. release-notes:: 6 | :branch: stable/queens 7 | -------------------------------------------------------------------------------- /releasenotes/source/rocky.rst: -------------------------------------------------------------------------------- 1 | =================================== 2 | Rocky Series Release Notes 3 | =================================== 4 | 5 | .. release-notes:: 6 | :branch: stable/rocky 7 | -------------------------------------------------------------------------------- /releasenotes/source/stein.rst: -------------------------------------------------------------------------------- 1 | =================================== 2 | Stein Series Release Notes 3 | =================================== 4 | 5 | .. release-notes:: 6 | :branch: stable/stein 7 | -------------------------------------------------------------------------------- /releasenotes/source/train.rst: -------------------------------------------------------------------------------- 1 | ========================== 2 | Train Series Release Notes 3 | ========================== 4 | 5 | .. release-notes:: 6 | :branch: stable/train 7 | -------------------------------------------------------------------------------- /releasenotes/source/unreleased.rst: -------------------------------------------------------------------------------- 1 | ========================== 2 | Unreleased Release Notes 3 | ========================== 4 | 5 | .. release-notes:: 6 | -------------------------------------------------------------------------------- /releasenotes/source/ussuri.rst: -------------------------------------------------------------------------------- 1 | =========================== 2 | Ussuri Series Release Notes 3 | =========================== 4 | 5 | .. release-notes:: 6 | :branch: stable/ussuri 7 | -------------------------------------------------------------------------------- /releasenotes/source/victoria.rst: -------------------------------------------------------------------------------- 1 | ============================= 2 | Victoria Series Release Notes 3 | ============================= 4 | 5 | .. release-notes:: 6 | :branch: unmaintained/victoria 7 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | # Requirements lower bounds listed here are our best effort to keep them up to 2 | # date but we do not test them so no guarantee of having them all correct. If 3 | # you find any incorrect lower bounds, let us know or propose a fix. 4 | 5 | # See: https://bugs.launchpad.net/pbr/+bug/1384919 for why this is here... 6 | pbr>=2.0.0 # Apache-2.0 7 | 8 | # Packages needed for using this library. 9 | 10 | # For deprecation warning 11 | debtcollector>=1.2.0 # Apache-2.0 12 | 13 | # For async and/or periodic work 14 | futurist>=1.2.0 # Apache-2.0 15 | 16 | # For reader/writer + interprocess locks. 17 | fasteners>=0.17.3 # Apache-2.0 18 | 19 | # Very nice graph library 20 | networkx>=2.1.0 # BSD 21 | 22 | # Used for backend storage engine loading. 23 | stevedore>=1.20.0 # Apache-2.0 24 | 25 | # Used for structured input validation 26 | jsonschema>=3.2.0 # MIT 27 | 28 | # For the state machine we run with 29 | automaton>=1.9.0 # Apache-2.0 30 | 31 | # For common utilities 32 | oslo.utils>=3.33.0 # Apache-2.0 33 | oslo.serialization>=2.18.0 # Apache-2.0 34 | tenacity>=6.0.0 # Apache-2.0 35 | 36 | # For lru caches and such 37 | cachetools>=2.0.0 # MIT License 38 | 39 | # For pydot output tests 40 | pydot>=1.2.4 # MIT License 41 | -------------------------------------------------------------------------------- /run_tests.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | function usage { 4 | echo "Usage: $0 [OPTION]..." 5 | echo "Run Taskflow's test suite(s)" 6 | echo "" 7 | echo " -f, --force Force a clean re-build of the virtual environment. Useful when dependencies have been added." 8 | echo " -p, --pep8 Just run pep8" 9 | echo " -P, --no-pep8 Don't run static code checks" 10 | echo " -v, --verbose Increase verbosity of reporting output" 11 | echo " -h, --help Print this usage message" 12 | echo "" 13 | exit 14 | } 15 | 16 | function process_option { 17 | case "$1" in 18 | -h|--help) usage;; 19 | -p|--pep8) let just_pep8=1;; 20 | -P|--no-pep8) let no_pep8=1;; 21 | -f|--force) let force=1;; 22 | -v|--verbose) let verbose=1;; 23 | *) pos_args="$pos_args $1" 24 | esac 25 | } 26 | 27 | verbose=0 28 | force=0 29 | pos_args="" 30 | just_pep8=0 31 | no_pep8=0 32 | tox_args="" 33 | tox="" 34 | 35 | for arg in "$@"; do 36 | process_option $arg 37 | done 38 | 39 | py=`which python` 40 | if [ -z "$py" ]; then 41 | echo "Python is required to use $0" 42 | echo "Please install it via your distributions package management system." 43 | exit 1 44 | fi 45 | 46 | py_envs=`python -c 'import sys; print("py%s%s" % (sys.version_info[0:2]))'` 47 | py_envs=${PY_ENVS:-$py_envs} 48 | 49 | function run_tests { 50 | local tox_cmd="${tox} ${tox_args} -e $py_envs ${pos_args}" 51 | echo "Running tests for environments $py_envs via $tox_cmd" 52 | bash -c "$tox_cmd" 53 | } 54 | 55 | function run_flake8 { 56 | local tox_cmd="${tox} ${tox_args} -e pep8 ${pos_args}" 57 | echo "Running flake8 via $tox_cmd" 58 | bash -c "$tox_cmd" 59 | } 60 | 61 | if [ $force -eq 1 ]; then 62 | tox_args="$tox_args -r" 63 | fi 64 | 65 | if [ $verbose -eq 1 ]; then 66 | tox_args="$tox_args -v" 67 | fi 68 | 69 | tox=`which tox` 70 | if [ -z "$tox" ]; then 71 | echo "Tox is required to use $0" 72 | echo "Please install it via \`pip\` or via your distributions" \ 73 | "package management system." 74 | echo "Visit http://tox.readthedocs.org/ for additional installation" \ 75 | "instructions." 76 | exit 1 77 | fi 78 | 79 | if [ $just_pep8 -eq 1 ]; then 80 | run_flake8 81 | exit 82 | fi 83 | 84 | run_tests || exit 85 | 86 | if [ $no_pep8 -eq 0 ]; then 87 | run_flake8 88 | fi 89 | -------------------------------------------------------------------------------- /setup-etcd-env.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -eux 3 | if [ -z "$(which etcd)" ]; then 4 | ETCD_VERSION=3.4.27 5 | case `uname -s` in 6 | Darwin) 7 | OS=darwin 8 | SUFFIX=zip 9 | ;; 10 | Linux) 11 | OS=linux 12 | SUFFIX=tar.gz 13 | ;; 14 | *) 15 | echo "Unsupported OS" 16 | exit 1 17 | esac 18 | case `uname -m` in 19 | x86_64) 20 | MACHINE=amd64 21 | ;; 22 | *) 23 | echo "Unsupported machine" 24 | exit 1 25 | esac 26 | TARBALL_NAME=etcd-v${ETCD_VERSION}-$OS-$MACHINE 27 | test ! -d "$TARBALL_NAME" && curl -L https://github.com/coreos/etcd/releases/download/v${ETCD_VERSION}/${TARBALL_NAME}.${SUFFIX} | tar xz 28 | export PATH=$PATH:$TARBALL_NAME 29 | fi 30 | 31 | $* 32 | -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [metadata] 2 | name = taskflow 3 | summary = Taskflow structured state management library. 4 | description_file = 5 | README.rst 6 | author = OpenStack 7 | author_email = openstack-discuss@lists.openstack.org 8 | home_page = https://docs.openstack.org/taskflow/latest/ 9 | keywords = reliable,tasks,execution,parallel,dataflow,workflows,distributed 10 | python_requires = >=3.9 11 | classifier = 12 | Development Status :: 5 - Production/Stable 13 | Environment :: OpenStack 14 | Intended Audience :: Developers 15 | Intended Audience :: Information Technology 16 | License :: OSI Approved :: Apache Software License 17 | Operating System :: POSIX :: Linux 18 | Programming Language :: Python 19 | Programming Language :: Python :: 3 20 | Programming Language :: Python :: 3.9 21 | Programming Language :: Python :: 3.10 22 | Programming Language :: Python :: 3.11 23 | Programming Language :: Python :: 3.12 24 | Programming Language :: Python :: 3 :: Only 25 | Programming Language :: Python :: Implementation :: CPython 26 | Topic :: Software Development :: Libraries 27 | Topic :: System :: Distributed Computing 28 | 29 | [files] 30 | packages = 31 | taskflow 32 | 33 | [entry_points] 34 | taskflow.jobboards = 35 | zookeeper = taskflow.jobs.backends.impl_zookeeper:ZookeeperJobBoard 36 | redis = taskflow.jobs.backends.impl_redis:RedisJobBoard 37 | etcd = taskflow.jobs.backends.impl_etcd:EtcdJobBoard 38 | 39 | taskflow.conductors = 40 | blocking = taskflow.conductors.backends.impl_blocking:BlockingConductor 41 | nonblocking = taskflow.conductors.backends.impl_nonblocking:NonBlockingConductor 42 | 43 | taskflow.persistence = 44 | dir = taskflow.persistence.backends.impl_dir:DirBackend 45 | file = taskflow.persistence.backends.impl_dir:DirBackend 46 | memory = taskflow.persistence.backends.impl_memory:MemoryBackend 47 | mysql = taskflow.persistence.backends.impl_sqlalchemy:SQLAlchemyBackend 48 | postgresql = taskflow.persistence.backends.impl_sqlalchemy:SQLAlchemyBackend 49 | sqlite = taskflow.persistence.backends.impl_sqlalchemy:SQLAlchemyBackend 50 | zookeeper = taskflow.persistence.backends.impl_zookeeper:ZkBackend 51 | 52 | taskflow.engines = 53 | default = taskflow.engines.action_engine.engine:SerialActionEngine 54 | serial = taskflow.engines.action_engine.engine:SerialActionEngine 55 | parallel = taskflow.engines.action_engine.engine:ParallelActionEngine 56 | worker-based = taskflow.engines.worker_based.engine:WorkerBasedActionEngine 57 | workers = taskflow.engines.worker_based.engine:WorkerBasedActionEngine 58 | 59 | [extras] 60 | # NOTE(dhellmann): The entries in this section of the file need to be 61 | # kept consistent with the entries in test-requirements.txt. 62 | zookeeper = 63 | kazoo>=2.6.0 # Apache-2.0 64 | redis = 65 | redis>=4.0.0 # MIT 66 | etcd = 67 | etcd3gw>=2.0.0 # Apache-2.0 68 | workers = 69 | kombu>=4.3.0 # BSD 70 | eventlet = 71 | eventlet>=0.18.2 # MIT 72 | database = 73 | SQLAlchemy>=1.0.10 # MIT 74 | alembic>=0.8.10 # MIT 75 | SQLAlchemy-Utils>=0.30.11 # BSD License 76 | PyMySQL>=0.7.6 # MIT License 77 | psycopg2>=2.8.0 # LGPL/ZPL 78 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2013 Hewlett-Packard Development Company, L.P. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 12 | # implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | import setuptools 17 | 18 | setuptools.setup( 19 | setup_requires=['pbr>=2.0.0'], 20 | pbr=True) 21 | -------------------------------------------------------------------------------- /taskflow/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openstack/taskflow/d01920ef1c0e3432c5d3a1e7708a34ff834bf339/taskflow/__init__.py -------------------------------------------------------------------------------- /taskflow/conductors/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openstack/taskflow/d01920ef1c0e3432c5d3a1e7708a34ff834bf339/taskflow/conductors/__init__.py -------------------------------------------------------------------------------- /taskflow/conductors/backends/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2014 Yahoo! Inc. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); you may 4 | # not use this file except in compliance with the License. You may obtain 5 | # a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 11 | # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 12 | # License for the specific language governing permissions and limitations 13 | # under the License. 14 | 15 | import logging 16 | 17 | import stevedore.driver 18 | 19 | from taskflow import exceptions as exc 20 | 21 | # NOTE(harlowja): this is the entrypoint namespace, not the module namespace. 22 | CONDUCTOR_NAMESPACE = 'taskflow.conductors' 23 | 24 | LOG = logging.getLogger(__name__) 25 | 26 | 27 | def fetch(kind, name, jobboard, namespace=CONDUCTOR_NAMESPACE, **kwargs): 28 | """Fetch a conductor backend with the given options. 29 | 30 | This fetch method will look for the entrypoint 'kind' in the entrypoint 31 | namespace, and then attempt to instantiate that entrypoint using the 32 | provided name, jobboard and any board specific kwargs. 33 | """ 34 | LOG.debug('Looking for %r conductor driver in %r', kind, namespace) 35 | try: 36 | mgr = stevedore.driver.DriverManager( 37 | namespace, kind, 38 | invoke_on_load=True, 39 | invoke_args=(name, jobboard), 40 | invoke_kwds=kwargs) 41 | return mgr.driver 42 | except RuntimeError as e: 43 | raise exc.NotFound("Could not find conductor %s" % (kind), e) 44 | -------------------------------------------------------------------------------- /taskflow/conductors/backends/impl_blocking.py: -------------------------------------------------------------------------------- 1 | # Licensed under the Apache License, Version 2.0 (the "License"); you may 2 | # not use this file except in compliance with the License. You may obtain 3 | # a copy of the License at 4 | # 5 | # http://www.apache.org/licenses/LICENSE-2.0 6 | # 7 | # Unless required by applicable law or agreed to in writing, software 8 | # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 9 | # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 10 | # License for the specific language governing permissions and limitations 11 | # under the License. 12 | 13 | import futurist 14 | 15 | from taskflow.conductors.backends import impl_executor 16 | 17 | 18 | class BlockingConductor(impl_executor.ExecutorConductor): 19 | """Blocking conductor that processes job(s) in a blocking manner.""" 20 | 21 | MAX_SIMULTANEOUS_JOBS = 1 22 | """ 23 | Default maximum number of jobs that can be in progress at the same time. 24 | """ 25 | 26 | @staticmethod 27 | def _executor_factory(): 28 | return futurist.SynchronousExecutor() 29 | 30 | def __init__(self, name, jobboard, 31 | persistence=None, engine=None, 32 | engine_options=None, wait_timeout=None, 33 | log=None, max_simultaneous_jobs=MAX_SIMULTANEOUS_JOBS): 34 | super().__init__( 35 | name, jobboard, 36 | persistence=persistence, engine=engine, 37 | engine_options=engine_options, 38 | wait_timeout=wait_timeout, log=log, 39 | max_simultaneous_jobs=max_simultaneous_jobs) 40 | -------------------------------------------------------------------------------- /taskflow/conductors/backends/impl_nonblocking.py: -------------------------------------------------------------------------------- 1 | # Licensed under the Apache License, Version 2.0 (the "License"); you may 2 | # not use this file except in compliance with the License. You may obtain 3 | # a copy of the License at 4 | # 5 | # http://www.apache.org/licenses/LICENSE-2.0 6 | # 7 | # Unless required by applicable law or agreed to in writing, software 8 | # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 9 | # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 10 | # License for the specific language governing permissions and limitations 11 | # under the License. 12 | 13 | import futurist 14 | 15 | from taskflow.conductors.backends import impl_executor 16 | from taskflow.utils import threading_utils as tu 17 | 18 | 19 | class NonBlockingConductor(impl_executor.ExecutorConductor): 20 | """Non-blocking conductor that processes job(s) using a thread executor. 21 | 22 | NOTE(harlowja): A custom executor factory can be provided via keyword 23 | argument ``executor_factory``, if provided it will be 24 | invoked at 25 | :py:meth:`~taskflow.conductors.base.Conductor.run` time 26 | with one positional argument (this conductor) and it must 27 | return a compatible `executor`_ which can be used 28 | to submit jobs to. If ``None`` is a provided a thread pool 29 | backed executor is selected by default (it will have 30 | an equivalent number of workers as this conductors 31 | simultaneous job count). 32 | 33 | .. _executor: https://docs.python.org/dev/library/\ 34 | concurrent.futures.html#executor-objects 35 | """ 36 | 37 | MAX_SIMULTANEOUS_JOBS = tu.get_optimal_thread_count() 38 | """ 39 | Default maximum number of jobs that can be in progress at the same time. 40 | """ 41 | 42 | def _default_executor_factory(self): 43 | max_simultaneous_jobs = self._max_simultaneous_jobs 44 | if max_simultaneous_jobs <= 0: 45 | max_workers = tu.get_optimal_thread_count() 46 | else: 47 | max_workers = max_simultaneous_jobs 48 | return futurist.ThreadPoolExecutor(max_workers=max_workers) 49 | 50 | def __init__(self, name, jobboard, 51 | persistence=None, engine=None, 52 | engine_options=None, wait_timeout=None, 53 | log=None, max_simultaneous_jobs=MAX_SIMULTANEOUS_JOBS, 54 | executor_factory=None): 55 | super().__init__( 56 | name, jobboard, 57 | persistence=persistence, engine=engine, 58 | engine_options=engine_options, wait_timeout=wait_timeout, 59 | log=log, max_simultaneous_jobs=max_simultaneous_jobs) 60 | if executor_factory is None: 61 | self._executor_factory = self._default_executor_factory 62 | else: 63 | if not callable(executor_factory): 64 | raise ValueError("Provided keyword argument 'executor_factory'" 65 | " must be callable") 66 | self._executor_factory = executor_factory 67 | -------------------------------------------------------------------------------- /taskflow/contrib/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openstack/taskflow/d01920ef1c0e3432c5d3a1e7708a34ff834bf339/taskflow/contrib/__init__.py -------------------------------------------------------------------------------- /taskflow/engines/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2012 Yahoo! Inc. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); you may 4 | # not use this file except in compliance with the License. You may obtain 5 | # a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 11 | # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 12 | # License for the specific language governing permissions and limitations 13 | # under the License. 14 | 15 | from oslo_utils import eventletutils as _eventletutils 16 | 17 | # Give a nice warning that if eventlet is being used these modules 18 | # are highly recommended to be patched (or otherwise bad things could 19 | # happen). 20 | _eventletutils.warn_eventlet_not_patched( 21 | expected_patched_modules=['time', 'thread']) 22 | 23 | 24 | # Promote helpers to this module namespace (for easy access). 25 | from taskflow.engines.helpers import flow_from_detail # noqa 26 | from taskflow.engines.helpers import load # noqa 27 | from taskflow.engines.helpers import load_from_detail # noqa 28 | from taskflow.engines.helpers import load_from_factory # noqa 29 | from taskflow.engines.helpers import run # noqa 30 | from taskflow.engines.helpers import save_factory_details # noqa 31 | -------------------------------------------------------------------------------- /taskflow/engines/action_engine/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openstack/taskflow/d01920ef1c0e3432c5d3a1e7708a34ff834bf339/taskflow/engines/action_engine/__init__.py -------------------------------------------------------------------------------- /taskflow/engines/action_engine/actions/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openstack/taskflow/d01920ef1c0e3432c5d3a1e7708a34ff834bf339/taskflow/engines/action_engine/actions/__init__.py -------------------------------------------------------------------------------- /taskflow/engines/action_engine/actions/base.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2015 Yahoo! Inc. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); you may 4 | # not use this file except in compliance with the License. You may obtain 5 | # a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 11 | # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 12 | # License for the specific language governing permissions and limitations 13 | # under the License. 14 | 15 | import abc 16 | 17 | from taskflow import states 18 | 19 | 20 | class Action(metaclass=abc.ABCMeta): 21 | """An action that handles executing, state changes, ... of atoms.""" 22 | 23 | NO_RESULT = object() 24 | """ 25 | Sentinel use to represent lack of any result (none can be a valid result) 26 | """ 27 | 28 | #: States that are expected to have a result to save... 29 | SAVE_RESULT_STATES = (states.SUCCESS, states.FAILURE, 30 | states.REVERTED, states.REVERT_FAILURE) 31 | 32 | def __init__(self, storage, notifier): 33 | self._storage = storage 34 | self._notifier = notifier 35 | 36 | @abc.abstractmethod 37 | def schedule_execution(self, atom): 38 | """Schedules atom execution.""" 39 | 40 | @abc.abstractmethod 41 | def schedule_reversion(self, atom): 42 | """Schedules atom reversion.""" 43 | 44 | @abc.abstractmethod 45 | def complete_reversion(self, atom, result): 46 | """Completes atom reversion.""" 47 | 48 | @abc.abstractmethod 49 | def complete_execution(self, atom, result): 50 | """Completes atom execution.""" 51 | -------------------------------------------------------------------------------- /taskflow/engines/worker_based/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openstack/taskflow/d01920ef1c0e3432c5d3a1e7708a34ff834bf339/taskflow/engines/worker_based/__init__.py -------------------------------------------------------------------------------- /taskflow/engines/worker_based/endpoint.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2014 Yahoo! Inc. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); you may 4 | # not use this file except in compliance with the License. You may obtain 5 | # a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 11 | # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 12 | # License for the specific language governing permissions and limitations 13 | # under the License. 14 | 15 | from oslo_utils import reflection 16 | 17 | from taskflow.engines.action_engine import executor 18 | 19 | 20 | class Endpoint: 21 | """Represents a single task with execute/revert methods.""" 22 | 23 | def __init__(self, task_cls): 24 | self._task_cls = task_cls 25 | self._task_cls_name = reflection.get_class_name(task_cls) 26 | self._executor = executor.SerialTaskExecutor() 27 | 28 | def __str__(self): 29 | return self._task_cls_name 30 | 31 | @property 32 | def name(self): 33 | return self._task_cls_name 34 | 35 | def generate(self, name=None): 36 | # NOTE(skudriashev): Note that task is created here with the `name` 37 | # argument passed to its constructor. This will be a problem when 38 | # task's constructor requires any other arguments. 39 | return self._task_cls(name=name) 40 | 41 | def execute(self, task, **kwargs): 42 | event, result = self._executor.execute_task(task, **kwargs).result() 43 | return result 44 | 45 | def revert(self, task, **kwargs): 46 | event, result = self._executor.revert_task(task, **kwargs).result() 47 | return result 48 | -------------------------------------------------------------------------------- /taskflow/examples/alphabet_soup.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2014 Yahoo! Inc. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); you may 4 | # not use this file except in compliance with the License. You may obtain 5 | # a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 11 | # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 12 | # License for the specific language governing permissions and limitations 13 | # under the License. 14 | 15 | import fractions 16 | import functools 17 | import logging 18 | import os 19 | import string 20 | import sys 21 | import time 22 | 23 | logging.basicConfig(level=logging.ERROR) 24 | 25 | self_dir = os.path.abspath(os.path.dirname(__file__)) 26 | top_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), 27 | os.pardir, 28 | os.pardir)) 29 | sys.path.insert(0, top_dir) 30 | sys.path.insert(0, self_dir) 31 | 32 | from taskflow import engines 33 | from taskflow import exceptions 34 | from taskflow.patterns import linear_flow 35 | from taskflow import task 36 | 37 | 38 | # In this example we show how a simple linear set of tasks can be executed 39 | # using local processes (and not threads or remote workers) with minimal (if 40 | # any) modification to those tasks to make them safe to run in this mode. 41 | # 42 | # This is useful since it allows further scaling up your workflows when thread 43 | # execution starts to become a bottleneck (which it can start to be due to the 44 | # GIL in python). It also offers a intermediary scalable runner that can be 45 | # used when the scale and/or setup of remote workers is not desirable. 46 | 47 | 48 | def progress_printer(task, event_type, details): 49 | # This callback, attached to each task will be called in the local 50 | # process (not the child processes)... 51 | progress = details.pop('progress') 52 | progress = int(progress * 100.0) 53 | print("Task '%s' reached %d%% completion" % (task.name, progress)) 54 | 55 | 56 | class AlphabetTask(task.Task): 57 | # Second delay between each progress part. 58 | _DELAY = 0.1 59 | 60 | # This task will run in X main stages (each with a different progress 61 | # report that will be delivered back to the running process...). The 62 | # initial 0% and 100% are triggered automatically by the engine when 63 | # a task is started and finished (so that's why those are not emitted 64 | # here). 65 | _PROGRESS_PARTS = [fractions.Fraction("%s/5" % x) for x in range(1, 5)] 66 | 67 | def execute(self): 68 | for p in self._PROGRESS_PARTS: 69 | self.update_progress(p) 70 | time.sleep(self._DELAY) 71 | 72 | 73 | print("Constructing...") 74 | soup = linear_flow.Flow("alphabet-soup") 75 | for letter in string.ascii_lowercase: 76 | abc = AlphabetTask(letter) 77 | abc.notifier.register(task.EVENT_UPDATE_PROGRESS, 78 | functools.partial(progress_printer, abc)) 79 | soup.add(abc) 80 | try: 81 | print("Loading...") 82 | e = engines.load(soup, engine='parallel', executor='processes') 83 | print("Compiling...") 84 | e.compile() 85 | print("Preparing...") 86 | e.prepare() 87 | print("Running...") 88 | e.run() 89 | print("Done: %s" % e.statistics) 90 | except exceptions.NotImplementedError as e: 91 | print(e) 92 | -------------------------------------------------------------------------------- /taskflow/examples/delayed_return.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2014 Yahoo! Inc. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); you may 4 | # not use this file except in compliance with the License. You may obtain 5 | # a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 11 | # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 12 | # License for the specific language governing permissions and limitations 13 | # under the License. 14 | 15 | import logging 16 | import os 17 | import sys 18 | 19 | from concurrent import futures 20 | 21 | logging.basicConfig(level=logging.ERROR) 22 | 23 | self_dir = os.path.abspath(os.path.dirname(__file__)) 24 | top_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), 25 | os.pardir, 26 | os.pardir)) 27 | sys.path.insert(0, top_dir) 28 | sys.path.insert(0, self_dir) 29 | 30 | # INTRO: in this example linear_flow we will attach a listener to an engine 31 | # and delay the return from a function until after the result of a task has 32 | # occurred in that engine. The engine will continue running (in the background) 33 | # while the function will have returned. 34 | 35 | import taskflow.engines 36 | from taskflow.listeners import base 37 | from taskflow.patterns import linear_flow as lf 38 | from taskflow import states 39 | from taskflow import task 40 | from taskflow.types import notifier 41 | 42 | 43 | class PokeFutureListener(base.Listener): 44 | def __init__(self, engine, future, task_name): 45 | super().__init__( 46 | engine, 47 | task_listen_for=(notifier.Notifier.ANY,), 48 | flow_listen_for=[]) 49 | self._future = future 50 | self._task_name = task_name 51 | 52 | def _task_receiver(self, state, details): 53 | if state in (states.SUCCESS, states.FAILURE): 54 | if details.get('task_name') == self._task_name: 55 | if state == states.SUCCESS: 56 | self._future.set_result(details['result']) 57 | else: 58 | failure = details['result'] 59 | self._future.set_exception(failure.exception) 60 | 61 | 62 | class Hi(task.Task): 63 | def execute(self): 64 | # raise IOError("I broken") 65 | return 'hi' 66 | 67 | 68 | class Bye(task.Task): 69 | def execute(self): 70 | return 'bye' 71 | 72 | 73 | def return_from_flow(pool): 74 | wf = lf.Flow("root").add(Hi("hi"), Bye("bye")) 75 | eng = taskflow.engines.load(wf, engine='serial') 76 | f = futures.Future() 77 | watcher = PokeFutureListener(eng, f, 'hi') 78 | watcher.register() 79 | pool.submit(eng.run) 80 | return (eng, f.result()) 81 | 82 | 83 | with futures.ThreadPoolExecutor(1) as pool: 84 | engine, hi_result = return_from_flow(pool) 85 | print(hi_result) 86 | 87 | print(engine.storage.get_flow_state()) 88 | -------------------------------------------------------------------------------- /taskflow/examples/dump_memory_backend.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2015 Yahoo! Inc. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); you may 4 | # not use this file except in compliance with the License. You may obtain 5 | # a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 11 | # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 12 | # License for the specific language governing permissions and limitations 13 | # under the License. 14 | 15 | import logging 16 | import os 17 | import sys 18 | 19 | logging.basicConfig(level=logging.ERROR) 20 | 21 | self_dir = os.path.abspath(os.path.dirname(__file__)) 22 | top_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), 23 | os.pardir, 24 | os.pardir)) 25 | sys.path.insert(0, top_dir) 26 | sys.path.insert(0, self_dir) 27 | 28 | from taskflow import engines 29 | from taskflow.patterns import linear_flow as lf 30 | from taskflow import task 31 | 32 | # INTRO: in this example we create a dummy flow with a dummy task, and run 33 | # it using a in-memory backend and pre/post run we dump out the contents 34 | # of the in-memory backends tree structure (which can be quite useful to 35 | # look at for debugging or other analysis). 36 | 37 | 38 | class PrintTask(task.Task): 39 | def execute(self): 40 | print("Running '%s'" % self.name) 41 | 42 | # Make a little flow and run it... 43 | f = lf.Flow('root') 44 | for alpha in ['a', 'b', 'c']: 45 | f.add(PrintTask(alpha)) 46 | 47 | e = engines.load(f) 48 | e.compile() 49 | e.prepare() 50 | 51 | # After prepare the storage layer + backend can now be accessed safely... 52 | backend = e.storage.backend 53 | 54 | print("----------") 55 | print("Before run") 56 | print("----------") 57 | print(backend.memory.pformat()) 58 | print("----------") 59 | 60 | e.run() 61 | 62 | print("---------") 63 | print("After run") 64 | print("---------") 65 | for path in backend.memory.ls_r(backend.memory.root_path, absolute=True): 66 | value = backend.memory[path] 67 | if value: 68 | print("{} -> {}".format(path, value)) 69 | else: 70 | print("%s" % (path)) 71 | -------------------------------------------------------------------------------- /taskflow/examples/echo_listener.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2015 Yahoo! Inc. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); you may 4 | # not use this file except in compliance with the License. You may obtain 5 | # a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 11 | # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 12 | # License for the specific language governing permissions and limitations 13 | # under the License. 14 | 15 | import logging 16 | import os 17 | import sys 18 | 19 | logging.basicConfig(level=logging.DEBUG) 20 | 21 | top_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), 22 | os.pardir, 23 | os.pardir)) 24 | sys.path.insert(0, top_dir) 25 | 26 | from taskflow import engines 27 | from taskflow.listeners import logging as logging_listener 28 | from taskflow.patterns import linear_flow as lf 29 | from taskflow import task 30 | 31 | # INTRO: This example walks through a miniature workflow which will do a 32 | # simple echo operation; during this execution a listener is associated with 33 | # the engine to receive all notifications about what the flow has performed, 34 | # this example dumps that output to the stdout for viewing (at debug level 35 | # to show all the information which is possible). 36 | 37 | 38 | class Echo(task.Task): 39 | def execute(self): 40 | print(self.name) 41 | 42 | 43 | # Generate the work to be done (but don't do it yet). 44 | wf = lf.Flow('abc') 45 | wf.add(Echo('a')) 46 | wf.add(Echo('b')) 47 | wf.add(Echo('c')) 48 | 49 | # This will associate the listener with the engine (the listener 50 | # will automatically register for notifications with the engine and deregister 51 | # when the context is exited). 52 | e = engines.load(wf) 53 | with logging_listener.DynamicLoggingListener(e): 54 | e.run() 55 | -------------------------------------------------------------------------------- /taskflow/examples/pseudo_scoping.out.txt: -------------------------------------------------------------------------------- 1 | Running simple flow: 2 | Fetching number for Josh. 3 | Calling Josh 777. 4 | 5 | Calling many people using prefixed factory: 6 | Fetching number for Jim. 7 | Calling Jim 444. 8 | Fetching number for Joe. 9 | Calling Joe 555. 10 | Fetching number for Josh. 11 | Calling Josh 777. 12 | -------------------------------------------------------------------------------- /taskflow/examples/resume_from_backend.out.txt: -------------------------------------------------------------------------------- 1 | ----------------------------------- 2 | At the beginning, there is no state 3 | ----------------------------------- 4 | Flow 'resume from backend example' state: None 5 | ------- 6 | Running 7 | ------- 8 | executing first==1.0 9 | ------------- 10 | After running 11 | ------------- 12 | Flow 'resume from backend example' state: SUSPENDED 13 | boom==1.0: SUCCESS, result=None 14 | first==1.0: SUCCESS, result=ok 15 | second==1.0: PENDING, result=None 16 | -------------------------- 17 | Resuming and running again 18 | -------------------------- 19 | executing second==1.0 20 | ---------- 21 | At the end 22 | ---------- 23 | Flow 'resume from backend example' state: SUCCESS 24 | boom==1.0: SUCCESS, result=None 25 | first==1.0: SUCCESS, result=ok 26 | second==1.0: SUCCESS, result=ok 27 | -------------------------------------------------------------------------------- /taskflow/examples/resume_many_flows.out.txt: -------------------------------------------------------------------------------- 1 | Run flow: 2 | Running flow example 18995b55-aaad-49fa-938f-006ac21ea4c7 3 | executing first==1.0 4 | executing boom==1.0 5 | > this time not exiting 6 | executing second==1.0 7 | 8 | 9 | Run flow, something happens: 10 | Running flow example f8f62ea6-1c9b-4e81-9ff9-1acaa299a648 11 | executing first==1.0 12 | executing boom==1.0 13 | > Critical error: boom = exit please 14 | 15 | 16 | Run flow, something happens again: 17 | Running flow example 16f11c15-4d8a-4552-b422-399565c873c4 18 | executing first==1.0 19 | executing boom==1.0 20 | > Critical error: boom = exit please 21 | 22 | 23 | Resuming all failed flows 24 | Resuming flow example f8f62ea6-1c9b-4e81-9ff9-1acaa299a648 25 | executing boom==1.0 26 | > this time not exiting 27 | executing second==1.0 28 | Resuming flow example 16f11c15-4d8a-4552-b422-399565c873c4 29 | executing boom==1.0 30 | > this time not exiting 31 | executing second==1.0 32 | 33 | -------------------------------------------------------------------------------- /taskflow/examples/resume_many_flows.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2013 Yahoo! Inc. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); you may 4 | # not use this file except in compliance with the License. You may obtain 5 | # a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 11 | # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 12 | # License for the specific language governing permissions and limitations 13 | # under the License. 14 | 15 | import os 16 | import subprocess 17 | import sys 18 | import tempfile 19 | 20 | self_dir = os.path.abspath(os.path.dirname(__file__)) 21 | sys.path.insert(0, self_dir) 22 | 23 | import example_utils # noqa 24 | 25 | # INTRO: In this example we create a common persistence database (sqlite based) 26 | # and then we run a few set of processes which themselves use this persistence 27 | # database, those processes 'crash' (in a simulated way) by exiting with a 28 | # system error exception. After this occurs a few times we then activate a 29 | # script which doesn't 'crash' and it will resume all the given engines flows 30 | # that did not complete and run them to completion (instead of crashing). 31 | # 32 | # This shows how a set of tasks can be finished even after repeatedly being 33 | # crashed, *crash resistance* if you may call it, due to the engine concept as 34 | # well as the persistence layer which keeps track of the state a flow 35 | # transitions through and persists the intermediary inputs and outputs and 36 | # overall flow state. 37 | 38 | 39 | def _exec(cmd, add_env=None): 40 | env = None 41 | if add_env: 42 | env = os.environ.copy() 43 | env.update(add_env) 44 | 45 | proc = subprocess.Popen(cmd, env=env, stdin=None, 46 | stdout=subprocess.PIPE, 47 | stderr=sys.stderr) 48 | 49 | stdout, _stderr = proc.communicate() 50 | rc = proc.returncode 51 | if rc != 0: 52 | raise RuntimeError("Could not run %s [%s]", cmd, rc) 53 | print(stdout.decode()) 54 | 55 | 56 | def _path_to(name): 57 | return os.path.abspath(os.path.join(os.path.dirname(__file__), 58 | 'resume_many_flows', name)) 59 | 60 | 61 | def main(): 62 | backend_uri = None 63 | tmp_path = None 64 | try: 65 | if example_utils.SQLALCHEMY_AVAILABLE: 66 | tmp_path = tempfile.mktemp(prefix='tf-resume-example') 67 | backend_uri = "sqlite:///%s" % (tmp_path) 68 | else: 69 | tmp_path = tempfile.mkdtemp(prefix='tf-resume-example') 70 | backend_uri = 'file:///%s' % (tmp_path) 71 | 72 | def run_example(name, add_env=None): 73 | _exec([sys.executable, _path_to(name), backend_uri], add_env) 74 | 75 | print('Run flow:') 76 | run_example('run_flow.py') 77 | 78 | print('\nRun flow, something happens:') 79 | run_example('run_flow.py', {'BOOM': 'exit please'}) 80 | 81 | print('\nRun flow, something happens again:') 82 | run_example('run_flow.py', {'BOOM': 'exit please'}) 83 | 84 | print('\nResuming all failed flows') 85 | run_example('resume_all.py') 86 | finally: 87 | if tmp_path: 88 | example_utils.rm_path(tmp_path) 89 | 90 | if __name__ == '__main__': 91 | main() 92 | -------------------------------------------------------------------------------- /taskflow/examples/resume_many_flows/my_flows.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2013 Yahoo! Inc. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); you may 4 | # not use this file except in compliance with the License. You may obtain 5 | # a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 11 | # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 12 | # License for the specific language governing permissions and limitations 13 | # under the License. 14 | 15 | import os 16 | 17 | from taskflow.patterns import linear_flow as lf 18 | from taskflow import task 19 | 20 | 21 | class UnfortunateTask(task.Task): 22 | def execute(self): 23 | print('executing %s' % self) 24 | boom = os.environ.get('BOOM') 25 | if boom: 26 | print('> Critical error: boom = %s' % boom) 27 | raise SystemExit() 28 | else: 29 | print('> this time not exiting') 30 | 31 | 32 | class TestTask(task.Task): 33 | def execute(self): 34 | print('executing %s' % self) 35 | 36 | 37 | def flow_factory(): 38 | return lf.Flow('example').add( 39 | TestTask(name='first'), 40 | UnfortunateTask(name='boom'), 41 | TestTask(name='second')) 42 | -------------------------------------------------------------------------------- /taskflow/examples/resume_many_flows/resume_all.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2013 Yahoo! Inc. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); you may 4 | # not use this file except in compliance with the License. You may obtain 5 | # a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 11 | # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 12 | # License for the specific language governing permissions and limitations 13 | # under the License. 14 | 15 | import logging 16 | import os 17 | import sys 18 | 19 | logging.basicConfig(level=logging.ERROR) 20 | 21 | self_dir = os.path.abspath(os.path.dirname(__file__)) 22 | top_dir = os.path.abspath( 23 | os.path.join(self_dir, os.pardir, os.pardir, os.pardir)) 24 | example_dir = os.path.abspath(os.path.join(self_dir, os.pardir)) 25 | 26 | sys.path.insert(0, top_dir) 27 | sys.path.insert(0, example_dir) 28 | 29 | 30 | import taskflow.engines 31 | from taskflow import states 32 | 33 | import example_utils # noqa 34 | 35 | 36 | FINISHED_STATES = (states.SUCCESS, states.FAILURE, states.REVERTED) 37 | 38 | 39 | def resume(flowdetail, backend): 40 | print('Resuming flow {} {}'.format(flowdetail.name, flowdetail.uuid)) 41 | engine = taskflow.engines.load_from_detail(flow_detail=flowdetail, 42 | backend=backend) 43 | engine.run() 44 | 45 | 46 | def main(): 47 | with example_utils.get_backend() as backend: 48 | logbooks = list(backend.get_connection().get_logbooks()) 49 | for lb in logbooks: 50 | for fd in lb: 51 | if fd.state not in FINISHED_STATES: 52 | resume(fd, backend) 53 | 54 | 55 | if __name__ == '__main__': 56 | main() 57 | -------------------------------------------------------------------------------- /taskflow/examples/resume_many_flows/run_flow.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2013 Yahoo! Inc. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); you may 4 | # not use this file except in compliance with the License. You may obtain 5 | # a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 11 | # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 12 | # License for the specific language governing permissions and limitations 13 | # under the License. 14 | 15 | import logging 16 | import os 17 | import sys 18 | 19 | logging.basicConfig(level=logging.ERROR) 20 | 21 | self_dir = os.path.abspath(os.path.dirname(__file__)) 22 | top_dir = os.path.abspath( 23 | os.path.join(self_dir, os.pardir, os.pardir, os.pardir)) 24 | example_dir = os.path.abspath(os.path.join(self_dir, os.pardir)) 25 | 26 | sys.path.insert(0, top_dir) 27 | sys.path.insert(0, self_dir) 28 | sys.path.insert(0, example_dir) 29 | 30 | import taskflow.engines 31 | 32 | import example_utils # noqa 33 | import my_flows # noqa 34 | 35 | 36 | with example_utils.get_backend() as backend: 37 | engine = taskflow.engines.load_from_factory(my_flows.flow_factory, 38 | backend=backend) 39 | print('Running flow {} {}'.format(engine.storage.flow_name, 40 | engine.storage.flow_uuid)) 41 | engine.run() 42 | -------------------------------------------------------------------------------- /taskflow/examples/retry_flow.out.txt: -------------------------------------------------------------------------------- 1 | Calling jim 333. 2 | Wrong number, apologizing. 3 | Calling jim 444. 4 | Wrong number, apologizing. 5 | Calling jim 555. 6 | Hello Jim! 7 | -------------------------------------------------------------------------------- /taskflow/examples/retry_flow.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2012-2013 Yahoo! Inc. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); you may 4 | # not use this file except in compliance with the License. You may obtain 5 | # a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 11 | # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 12 | # License for the specific language governing permissions and limitations 13 | # under the License. 14 | 15 | import logging 16 | import os 17 | import sys 18 | 19 | logging.basicConfig(level=logging.ERROR) 20 | 21 | top_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), 22 | os.pardir, 23 | os.pardir)) 24 | sys.path.insert(0, top_dir) 25 | 26 | import taskflow.engines 27 | from taskflow.patterns import linear_flow as lf 28 | from taskflow import retry 29 | from taskflow import task 30 | 31 | # INTRO: In this example we create a retry controller that receives a phone 32 | # directory and tries different phone numbers. The next task tries to call Jim 33 | # using the given number. If it is not a Jim's number, the task raises an 34 | # exception and retry controller takes the next number from the phone 35 | # directory and retries the call. 36 | # 37 | # This example shows a basic usage of retry controllers in a flow. 38 | # Retry controllers allows to revert and retry a failed subflow with new 39 | # parameters. 40 | 41 | 42 | class CallJim(task.Task): 43 | def execute(self, jim_number): 44 | print("Calling jim %s." % jim_number) 45 | if jim_number != 555: 46 | raise Exception("Wrong number!") 47 | else: 48 | print("Hello Jim!") 49 | 50 | def revert(self, jim_number, **kwargs): 51 | print("Wrong number, apologizing.") 52 | 53 | 54 | # Create your flow and associated tasks (the work to be done). 55 | flow = lf.Flow('retrying-linear', 56 | retry=retry.ParameterizedForEach( 57 | rebind=['phone_directory'], 58 | provides='jim_number')).add(CallJim()) 59 | 60 | # Now run that flow using the provided initial data (store below). 61 | taskflow.engines.run(flow, store={'phone_directory': [333, 444, 555, 666]}) 62 | -------------------------------------------------------------------------------- /taskflow/examples/reverting_linear.out.txt: -------------------------------------------------------------------------------- 1 | Calling jim 555. 2 | Calling joe 444. 3 | Calling 444 and apologizing. 4 | Calling 555 and apologizing. 5 | Flow failed: Suzzie not home right now. 6 | -------------------------------------------------------------------------------- /taskflow/examples/run_by_iter.out.txt: -------------------------------------------------------------------------------- 1 | RESUMING 2 | SCHEDULING 3 | A 4 | WAITING 5 | ANALYZING 6 | SCHEDULING 7 | B 8 | WAITING 9 | ANALYZING 10 | SCHEDULING 11 | C 12 | WAITING 13 | ANALYZING 14 | SCHEDULING 15 | D 16 | WAITING 17 | ANALYZING 18 | SCHEDULING 19 | E 20 | WAITING 21 | ANALYZING 22 | SCHEDULING 23 | F 24 | WAITING 25 | ANALYZING 26 | SCHEDULING 27 | G 28 | WAITING 29 | ANALYZING 30 | SCHEDULING 31 | H 32 | WAITING 33 | ANALYZING 34 | SCHEDULING 35 | I 36 | WAITING 37 | ANALYZING 38 | SCHEDULING 39 | J 40 | WAITING 41 | ANALYZING 42 | SCHEDULING 43 | K 44 | WAITING 45 | ANALYZING 46 | SCHEDULING 47 | L 48 | WAITING 49 | ANALYZING 50 | SCHEDULING 51 | M 52 | WAITING 53 | ANALYZING 54 | SCHEDULING 55 | N 56 | WAITING 57 | ANALYZING 58 | SCHEDULING 59 | O 60 | WAITING 61 | ANALYZING 62 | SCHEDULING 63 | P 64 | WAITING 65 | ANALYZING 66 | SCHEDULING 67 | Q 68 | WAITING 69 | ANALYZING 70 | SCHEDULING 71 | R 72 | WAITING 73 | ANALYZING 74 | SCHEDULING 75 | S 76 | WAITING 77 | ANALYZING 78 | SCHEDULING 79 | T 80 | WAITING 81 | ANALYZING 82 | SCHEDULING 83 | U 84 | WAITING 85 | ANALYZING 86 | SCHEDULING 87 | V 88 | WAITING 89 | ANALYZING 90 | SCHEDULING 91 | W 92 | WAITING 93 | ANALYZING 94 | SCHEDULING 95 | X 96 | WAITING 97 | ANALYZING 98 | SCHEDULING 99 | Y 100 | WAITING 101 | ANALYZING 102 | SCHEDULING 103 | Z 104 | WAITING 105 | ANALYZING 106 | SUCCESS 107 | -------------------------------------------------------------------------------- /taskflow/examples/run_by_iter.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2014 Yahoo! Inc. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); you may 4 | # not use this file except in compliance with the License. You may obtain 5 | # a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 11 | # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 12 | # License for the specific language governing permissions and limitations 13 | # under the License. 14 | 15 | import logging 16 | import os 17 | import sys 18 | 19 | logging.basicConfig(level=logging.ERROR) 20 | 21 | self_dir = os.path.abspath(os.path.dirname(__file__)) 22 | top_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), 23 | os.pardir, 24 | os.pardir)) 25 | sys.path.insert(0, top_dir) 26 | sys.path.insert(0, self_dir) 27 | 28 | 29 | from taskflow import engines 30 | from taskflow.patterns import linear_flow as lf 31 | from taskflow import task 32 | 33 | 34 | # INTRO: This example shows how to run a set of engines at the same time, each 35 | # running in different engines using a single thread of control to iterate over 36 | # each engine (which causes that engine to advanced to its next state during 37 | # each iteration). 38 | 39 | 40 | class EchoTask(task.Task): 41 | def execute(self, value): 42 | print(value) 43 | return chr(ord(value) + 1) 44 | 45 | 46 | def make_alphabet_flow(i): 47 | f = lf.Flow("alphabet_%s" % (i)) 48 | start_value = 'A' 49 | end_value = 'Z' 50 | curr_value = start_value 51 | while ord(curr_value) <= ord(end_value): 52 | next_value = chr(ord(curr_value) + 1) 53 | if curr_value != end_value: 54 | f.add(EchoTask(name="echoer_%s" % curr_value, 55 | rebind={'value': curr_value}, 56 | provides=next_value)) 57 | else: 58 | f.add(EchoTask(name="echoer_%s" % curr_value, 59 | rebind={'value': curr_value})) 60 | curr_value = next_value 61 | return f 62 | 63 | 64 | # Adjust this number to change how many engines/flows run at once. 65 | flow_count = 1 66 | flows = [] 67 | for i in range(0, flow_count): 68 | f = make_alphabet_flow(i + 1) 69 | flows.append(make_alphabet_flow(i + 1)) 70 | engine_iters = [] 71 | for f in flows: 72 | e = engines.load(f) 73 | e.compile() 74 | e.storage.inject({'A': 'A'}) 75 | e.prepare() 76 | engine_iters.append(e.run_iter()) 77 | while engine_iters: 78 | for it in list(engine_iters): 79 | try: 80 | print(next(it)) 81 | except StopIteration: 82 | engine_iters.remove(it) 83 | -------------------------------------------------------------------------------- /taskflow/examples/run_by_iter_enumerate.out.txt: -------------------------------------------------------------------------------- 1 | Transition 1: RESUMING 2 | Transition 2: SCHEDULING 3 | echo_1 4 | Transition 3: WAITING 5 | Transition 4: ANALYZING 6 | Transition 5: SCHEDULING 7 | echo_2 8 | Transition 6: WAITING 9 | Transition 7: ANALYZING 10 | Transition 8: SCHEDULING 11 | echo_3 12 | Transition 9: WAITING 13 | Transition 10: ANALYZING 14 | Transition 11: SCHEDULING 15 | echo_4 16 | Transition 12: WAITING 17 | Transition 13: ANALYZING 18 | Transition 14: SCHEDULING 19 | echo_5 20 | Transition 15: WAITING 21 | Transition 16: ANALYZING 22 | Transition 17: SCHEDULING 23 | echo_6 24 | Transition 18: WAITING 25 | Transition 19: ANALYZING 26 | Transition 20: SCHEDULING 27 | echo_7 28 | Transition 21: WAITING 29 | Transition 22: ANALYZING 30 | Transition 23: SCHEDULING 31 | echo_8 32 | Transition 24: WAITING 33 | Transition 25: ANALYZING 34 | Transition 26: SCHEDULING 35 | echo_9 36 | Transition 27: WAITING 37 | Transition 28: ANALYZING 38 | Transition 29: SCHEDULING 39 | echo_10 40 | Transition 30: WAITING 41 | Transition 31: ANALYZING 42 | Transition 32: SUCCESS 43 | -------------------------------------------------------------------------------- /taskflow/examples/run_by_iter_enumerate.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2014 Yahoo! Inc. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); you may 4 | # not use this file except in compliance with the License. You may obtain 5 | # a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 11 | # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 12 | # License for the specific language governing permissions and limitations 13 | # under the License. 14 | 15 | import logging 16 | import os 17 | import sys 18 | 19 | logging.basicConfig(level=logging.ERROR) 20 | 21 | self_dir = os.path.abspath(os.path.dirname(__file__)) 22 | top_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), 23 | os.pardir, 24 | os.pardir)) 25 | sys.path.insert(0, top_dir) 26 | sys.path.insert(0, self_dir) 27 | 28 | from taskflow import engines 29 | from taskflow.patterns import linear_flow as lf 30 | from taskflow import task 31 | 32 | # INTRO: These examples show how to run an engine using the engine iteration 33 | # capability, in between iterations other activities occur (in this case a 34 | # value is output to stdout); but more complicated actions can occur at the 35 | # boundary when an engine yields its current state back to the caller. 36 | 37 | 38 | class EchoNameTask(task.Task): 39 | def execute(self): 40 | print(self.name) 41 | 42 | 43 | f = lf.Flow("counter") 44 | for i in range(0, 10): 45 | f.add(EchoNameTask("echo_%s" % (i + 1))) 46 | 47 | e = engines.load(f) 48 | e.compile() 49 | e.prepare() 50 | 51 | for i, st in enumerate(e.run_iter(), 1): 52 | print("Transition {}: {}".format(i, st)) 53 | -------------------------------------------------------------------------------- /taskflow/examples/share_engine_thread.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2012-2013 Yahoo! Inc. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); you may 4 | # not use this file except in compliance with the License. You may obtain 5 | # a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 11 | # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 12 | # License for the specific language governing permissions and limitations 13 | # under the License. 14 | 15 | import logging 16 | import os 17 | import random 18 | import sys 19 | import time 20 | 21 | logging.basicConfig(level=logging.ERROR) 22 | 23 | top_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), 24 | os.pardir, 25 | os.pardir)) 26 | sys.path.insert(0, top_dir) 27 | 28 | import futurist 29 | 30 | from taskflow import engines 31 | from taskflow.patterns import unordered_flow as uf 32 | from taskflow import task 33 | from taskflow.utils import threading_utils as tu 34 | 35 | # INTRO: in this example we create 2 dummy flow(s) with a 2 dummy task(s), and 36 | # run it using a shared thread pool executor to show how a single executor can 37 | # be used with more than one engine (sharing the execution thread pool between 38 | # them); this allows for saving resources and reusing threads in situations 39 | # where this is benefical. 40 | 41 | 42 | class DelayedTask(task.Task): 43 | def __init__(self, name): 44 | super().__init__(name=name) 45 | self._wait_for = random.random() 46 | 47 | def execute(self): 48 | print("Running '{}' in thread '{}'".format(self.name, tu.get_ident())) 49 | time.sleep(self._wait_for) 50 | 51 | 52 | f1 = uf.Flow("f1") 53 | f1.add(DelayedTask("f1-1")) 54 | f1.add(DelayedTask("f1-2")) 55 | 56 | f2 = uf.Flow("f2") 57 | f2.add(DelayedTask("f2-1")) 58 | f2.add(DelayedTask("f2-2")) 59 | 60 | # Run them all using the same futures (thread-pool based) executor... 61 | with futurist.ThreadPoolExecutor() as ex: 62 | e1 = engines.load(f1, engine='parallel', executor=ex) 63 | e2 = engines.load(f2, engine='parallel', executor=ex) 64 | iters = [e1.run_iter(), e2.run_iter()] 65 | # Iterate over a copy (so we can remove from the source list). 66 | cloned_iters = list(iters) 67 | while iters: 68 | # Run a single 'step' of each iterator, forcing each engine to perform 69 | # some work, then yield, and repeat until each iterator is consumed 70 | # and there is no more engine work to be done. 71 | for it in cloned_iters: 72 | try: 73 | next(it) 74 | except StopIteration: 75 | try: 76 | iters.remove(it) 77 | except ValueError: 78 | pass 79 | -------------------------------------------------------------------------------- /taskflow/examples/simple_linear.out.txt: -------------------------------------------------------------------------------- 1 | Calling jim 555. 2 | Calling joe 444. 3 | -------------------------------------------------------------------------------- /taskflow/examples/simple_linear.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2012-2013 Yahoo! Inc. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); you may 4 | # not use this file except in compliance with the License. You may obtain 5 | # a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 11 | # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 12 | # License for the specific language governing permissions and limitations 13 | # under the License. 14 | 15 | import logging 16 | import os 17 | import sys 18 | 19 | logging.basicConfig(level=logging.ERROR) 20 | 21 | top_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), 22 | os.pardir, 23 | os.pardir)) 24 | sys.path.insert(0, top_dir) 25 | 26 | import taskflow.engines 27 | from taskflow.patterns import linear_flow as lf 28 | from taskflow import task 29 | 30 | # INTRO: In this example we create two tasks, each of which ~calls~ a given 31 | # ~phone~ number (provided as a function input) in a linear fashion (one after 32 | # the other). For a workflow which is serial this shows a extremely simple way 33 | # of structuring your tasks (the code that does the work) into a linear 34 | # sequence (the flow) and then passing the work off to an engine, with some 35 | # initial data to be ran in a reliable manner. 36 | # 37 | # NOTE(harlowja): This example shows a basic usage of the taskflow structures 38 | # without involving the complexity of persistence. Using the structures that 39 | # taskflow provides via tasks and flows makes it possible for you to easily at 40 | # a later time hook in a persistence layer (and then gain the functionality 41 | # that offers) when you decide the complexity of adding that layer in 42 | # is 'worth it' for your application's usage pattern (which certain 43 | # applications may not need). 44 | 45 | 46 | class CallJim(task.Task): 47 | def execute(self, jim_number, *args, **kwargs): 48 | print("Calling jim %s." % jim_number) 49 | 50 | 51 | class CallJoe(task.Task): 52 | def execute(self, joe_number, *args, **kwargs): 53 | print("Calling joe %s." % joe_number) 54 | 55 | 56 | # Create your flow and associated tasks (the work to be done). 57 | flow = lf.Flow('simple-linear').add( 58 | CallJim(), 59 | CallJoe() 60 | ) 61 | 62 | # Now run that flow using the provided initial data (store below). 63 | taskflow.engines.run(flow, store=dict(joe_number=444, 64 | jim_number=555)) 65 | -------------------------------------------------------------------------------- /taskflow/examples/simple_linear_listening.out.txt: -------------------------------------------------------------------------------- 1 | Flow => RUNNING 2 | Task __main__.call_jim => RUNNING 3 | Calling jim. 4 | Context = [('jim_number', 555), ('joe_number', 444)] 5 | Task __main__.call_jim => SUCCESS 6 | Task __main__.call_joe => RUNNING 7 | Calling joe. 8 | Context = [('jim_number', 555), ('joe_number', 444)] 9 | Task __main__.call_joe => SUCCESS 10 | Flow => SUCCESS 11 | -------------------------------------------------------------------------------- /taskflow/examples/simple_linear_pass.out.txt: -------------------------------------------------------------------------------- 1 | Constructing... 2 | Loading... 3 | Compiling... 4 | Preparing... 5 | Running... 6 | Executing 'a' 7 | Executing 'b' 8 | Got input 'a' 9 | Done... 10 | -------------------------------------------------------------------------------- /taskflow/examples/simple_linear_pass.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2014 Yahoo! Inc. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); you may 4 | # not use this file except in compliance with the License. You may obtain 5 | # a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 11 | # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 12 | # License for the specific language governing permissions and limitations 13 | # under the License. 14 | 15 | import logging 16 | import os 17 | import sys 18 | 19 | logging.basicConfig(level=logging.ERROR) 20 | 21 | self_dir = os.path.abspath(os.path.dirname(__file__)) 22 | top_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), 23 | os.pardir, 24 | os.pardir)) 25 | sys.path.insert(0, top_dir) 26 | sys.path.insert(0, self_dir) 27 | 28 | from taskflow import engines 29 | from taskflow.patterns import linear_flow 30 | from taskflow import task 31 | 32 | # INTRO: This example shows how a task (in a linear/serial workflow) can 33 | # produce an output that can be then consumed/used by a downstream task. 34 | 35 | 36 | class TaskA(task.Task): 37 | default_provides = 'a' 38 | 39 | def execute(self): 40 | print("Executing '%s'" % (self.name)) 41 | return 'a' 42 | 43 | 44 | class TaskB(task.Task): 45 | def execute(self, a): 46 | print("Executing '%s'" % (self.name)) 47 | print("Got input '%s'" % (a)) 48 | 49 | 50 | print("Constructing...") 51 | wf = linear_flow.Flow("pass-from-to") 52 | wf.add(TaskA('a'), TaskB('b')) 53 | 54 | print("Loading...") 55 | e = engines.load(wf) 56 | 57 | print("Compiling...") 58 | e.compile() 59 | 60 | print("Preparing...") 61 | e.prepare() 62 | 63 | print("Running...") 64 | e.run() 65 | 66 | print("Done...") 67 | -------------------------------------------------------------------------------- /taskflow/examples/switch_graph_flow.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2014 Yahoo! Inc. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); you may 4 | # not use this file except in compliance with the License. You may obtain 5 | # a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 11 | # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 12 | # License for the specific language governing permissions and limitations 13 | # under the License. 14 | 15 | import logging 16 | import os 17 | import sys 18 | 19 | logging.basicConfig(level=logging.ERROR) 20 | 21 | top_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), 22 | os.pardir, 23 | os.pardir)) 24 | sys.path.insert(0, top_dir) 25 | 26 | from taskflow import engines 27 | from taskflow.patterns import graph_flow as gf 28 | from taskflow import task 29 | 30 | 31 | class DummyTask(task.Task): 32 | def execute(self): 33 | print("Running %s" % self.name) 34 | 35 | 36 | def allow(history): 37 | print(history) 38 | return False 39 | 40 | 41 | # Declare our work to be done... 42 | r = gf.Flow("root") 43 | r_a = DummyTask('r-a') 44 | r_b = DummyTask('r-b') 45 | r.add(r_a, r_b) 46 | r.link(r_a, r_b, decider=allow) 47 | 48 | # Setup and run the engine layer. 49 | e = engines.load(r) 50 | e.compile() 51 | e.prepare() 52 | e.run() 53 | 54 | 55 | print("---------") 56 | print("After run") 57 | print("---------") 58 | backend = e.storage.backend 59 | entries = [os.path.join(backend.memory.root_path, child) 60 | for child in backend.memory.ls(backend.memory.root_path)] 61 | while entries: 62 | path = entries.pop() 63 | value = backend.memory[path] 64 | if value: 65 | print("{} -> {}".format(path, value)) 66 | else: 67 | print("%s" % (path)) 68 | entries.extend(os.path.join(path, child) 69 | for child in backend.memory.ls(path)) 70 | -------------------------------------------------------------------------------- /taskflow/examples/timing_listener.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2014 Yahoo! Inc. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); you may 4 | # not use this file except in compliance with the License. You may obtain 5 | # a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 11 | # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 12 | # License for the specific language governing permissions and limitations 13 | # under the License. 14 | 15 | import logging 16 | import os 17 | import random 18 | import sys 19 | import time 20 | 21 | logging.basicConfig(level=logging.ERROR) 22 | 23 | top_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), 24 | os.pardir, 25 | os.pardir)) 26 | sys.path.insert(0, top_dir) 27 | 28 | from taskflow import engines 29 | from taskflow.listeners import timing 30 | from taskflow.patterns import linear_flow as lf 31 | from taskflow import task 32 | 33 | # INTRO: in this example we will attach a listener to an engine 34 | # and have variable run time tasks run and show how the listener will print 35 | # out how long those tasks took (when they started and when they finished). 36 | # 37 | # This shows how timing metrics can be gathered (or attached onto an engine) 38 | # after a workflow has been constructed, making it easy to gather metrics 39 | # dynamically for situations where this kind of information is applicable (or 40 | # even adding this information on at a later point in the future when your 41 | # application starts to slow down). 42 | 43 | 44 | class VariableTask(task.Task): 45 | def __init__(self, name): 46 | super().__init__(name) 47 | self._sleepy_time = random.random() 48 | 49 | def execute(self): 50 | time.sleep(self._sleepy_time) 51 | 52 | 53 | f = lf.Flow('root') 54 | f.add(VariableTask('a'), VariableTask('b'), VariableTask('c')) 55 | e = engines.load(f) 56 | with timing.PrintingDurationListener(e): 57 | e.run() 58 | -------------------------------------------------------------------------------- /taskflow/examples/wbe_mandelbrot.out.txt: -------------------------------------------------------------------------------- 1 | Calculating your mandelbrot fractal of size 512x512. 2 | Running 2 workers. 3 | Execution finished. 4 | Stopping workers. 5 | Writing image... 6 | Gathered 262144 results that represents a mandelbrot image (using 8 chunks that are computed jointly by 2 workers). 7 | -------------------------------------------------------------------------------- /taskflow/examples/wbe_simple_linear.out.txt: -------------------------------------------------------------------------------- 1 | Running 2 workers. 2 | Executing some work. 3 | Execution finished. 4 | Result = {"result1": 1, "result2": 666, "x": 111, "y": 222, "z": 333} 5 | Stopping workers. 6 | -------------------------------------------------------------------------------- /taskflow/jobs/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openstack/taskflow/d01920ef1c0e3432c5d3a1e7708a34ff834bf339/taskflow/jobs/__init__.py -------------------------------------------------------------------------------- /taskflow/jobs/backends/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2014 Yahoo! Inc. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); you may 4 | # not use this file except in compliance with the License. You may obtain 5 | # a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 11 | # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 12 | # License for the specific language governing permissions and limitations 13 | # under the License. 14 | 15 | import contextlib 16 | 17 | from stevedore import driver 18 | 19 | from taskflow import exceptions as exc 20 | from taskflow import logging 21 | from taskflow.utils import misc 22 | 23 | 24 | # NOTE(harlowja): this is the entrypoint namespace, not the module namespace. 25 | BACKEND_NAMESPACE = 'taskflow.jobboards' 26 | 27 | LOG = logging.getLogger(__name__) 28 | 29 | 30 | def fetch(name, conf, namespace=BACKEND_NAMESPACE, **kwargs): 31 | """Fetch a jobboard backend with the given configuration. 32 | 33 | This fetch method will look for the entrypoint name in the entrypoint 34 | namespace, and then attempt to instantiate that entrypoint using the 35 | provided name, configuration and any board specific kwargs. 36 | 37 | NOTE(harlowja): to aid in making it easy to specify configuration and 38 | options to a board the configuration (which is typical just a dictionary) 39 | can also be a URI string that identifies the entrypoint name and any 40 | configuration specific to that board. 41 | 42 | For example, given the following configuration URI:: 43 | 44 | zookeeper:///?a=b&c=d 45 | 46 | This will look for the entrypoint named 'zookeeper' and will provide 47 | a configuration object composed of the URI's components, in this case that 48 | is ``{'a': 'b', 'c': 'd'}`` to the constructor of that board 49 | instance (also including the name specified). 50 | """ 51 | board, conf = misc.extract_driver_and_conf(conf, 'board') 52 | LOG.debug('Looking for %r jobboard driver in %r', board, namespace) 53 | try: 54 | mgr = driver.DriverManager(namespace, board, 55 | invoke_on_load=True, 56 | invoke_args=(name, conf), 57 | invoke_kwds=kwargs) 58 | return mgr.driver 59 | except RuntimeError as e: 60 | raise exc.NotFound("Could not find jobboard %s" % (board), e) 61 | 62 | 63 | @contextlib.contextmanager 64 | def backend(name, conf, namespace=BACKEND_NAMESPACE, **kwargs): 65 | """Fetches a jobboard, connects to it and closes it on completion. 66 | 67 | This allows a board instance to fetched, connected to, and then used in a 68 | context manager statement with the board being closed upon context 69 | manager exit. 70 | """ 71 | jb = fetch(name, conf, namespace=namespace, **kwargs) 72 | jb.connect() 73 | with contextlib.closing(jb): 74 | yield jb 75 | -------------------------------------------------------------------------------- /taskflow/listeners/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openstack/taskflow/d01920ef1c0e3432c5d3a1e7708a34ff834bf339/taskflow/listeners/__init__.py -------------------------------------------------------------------------------- /taskflow/listeners/printing.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2013 Yahoo! Inc. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); you may 4 | # not use this file except in compliance with the License. You may obtain 5 | # a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 11 | # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 12 | # License for the specific language governing permissions and limitations 13 | # under the License. 14 | 15 | import sys 16 | import traceback 17 | 18 | from taskflow.listeners import base 19 | 20 | 21 | class PrintingListener(base.DumpingListener): 22 | """Writes the task and flow notifications messages to stdout or stderr.""" 23 | def __init__(self, engine, 24 | task_listen_for=base.DEFAULT_LISTEN_FOR, 25 | flow_listen_for=base.DEFAULT_LISTEN_FOR, 26 | retry_listen_for=base.DEFAULT_LISTEN_FOR, 27 | stderr=False): 28 | super().__init__( 29 | engine, task_listen_for=task_listen_for, 30 | flow_listen_for=flow_listen_for, retry_listen_for=retry_listen_for) 31 | if stderr: 32 | self._file = sys.stderr 33 | else: 34 | self._file = sys.stdout 35 | 36 | def _dump(self, message, *args, **kwargs): 37 | print(message % args, file=self._file) 38 | exc_info = kwargs.get('exc_info') 39 | if exc_info is not None: 40 | traceback.print_exception(exc_info[0], exc_info[1], exc_info[2], 41 | file=self._file) 42 | -------------------------------------------------------------------------------- /taskflow/logging.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2014 Yahoo! Inc. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); you may 4 | # not use this file except in compliance with the License. You may obtain 5 | # a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 11 | # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 12 | # License for the specific language governing permissions and limitations 13 | # under the License. 14 | 15 | import logging 16 | 17 | _BASE = __name__.split(".", 1)[0] 18 | 19 | # Add a BLATHER/TRACE level, this matches the multiprocessing 20 | # utils.py module (and oslo.log, kazoo and others) that declares a similar 21 | # level, this level is for information that is even lower level than regular 22 | # DEBUG and gives out so much runtime information that it is only 23 | # useful by low-level/certain users... 24 | BLATHER = 5 25 | TRACE = BLATHER 26 | 27 | 28 | # Copy over *select* attributes to make it easy to use this module. 29 | CRITICAL = logging.CRITICAL 30 | DEBUG = logging.DEBUG 31 | ERROR = logging.ERROR 32 | FATAL = logging.FATAL 33 | INFO = logging.INFO 34 | NOTSET = logging.NOTSET 35 | WARN = logging.WARN 36 | WARNING = logging.WARNING 37 | 38 | 39 | class _TraceLoggerAdapter(logging.LoggerAdapter): 40 | 41 | def trace(self, msg, *args, **kwargs): 42 | """Delegate a trace call to the underlying logger.""" 43 | self.log(TRACE, msg, *args, **kwargs) 44 | 45 | def warn(self, msg, *args, **kwargs): 46 | """Delegate a warning call to the underlying logger.""" 47 | self.warning(msg, *args, **kwargs) 48 | 49 | 50 | def getLogger(name=_BASE, extra=None): 51 | logger = logging.getLogger(name) 52 | if not logger.handlers: 53 | logger.addHandler(logging.NullHandler()) 54 | return _TraceLoggerAdapter(logger, extra=extra) 55 | -------------------------------------------------------------------------------- /taskflow/patterns/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openstack/taskflow/d01920ef1c0e3432c5d3a1e7708a34ff834bf339/taskflow/patterns/__init__.py -------------------------------------------------------------------------------- /taskflow/patterns/linear_flow.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2012 Yahoo! Inc. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); you may 4 | # not use this file except in compliance with the License. You may obtain 5 | # a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 11 | # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 12 | # License for the specific language governing permissions and limitations 13 | # under the License. 14 | 15 | from taskflow import flow 16 | from taskflow.types import graph as gr 17 | 18 | 19 | class Flow(flow.Flow): 20 | """Linear flow pattern. 21 | 22 | A linear (potentially nested) flow of *tasks/flows* that can be 23 | applied in order as one unit and rolled back as one unit using 24 | the reverse order that the *tasks/flows* have been applied in. 25 | """ 26 | 27 | _no_last_item = object() 28 | """Sentinel object used to denote no last item has been assigned. 29 | 30 | This is used to track no last item being added, since at creation there 31 | is no last item, but since the :meth:`.add` routine can take any object 32 | including none, we have to use a different object to be able to 33 | distinguish the lack of any last item... 34 | """ 35 | 36 | def __init__(self, name, retry=None): 37 | super().__init__(name, retry) 38 | self._graph = gr.OrderedDiGraph(name=name) 39 | self._last_item = self._no_last_item 40 | 41 | def add(self, *items): 42 | """Adds a given task/tasks/flow/flows to this flow.""" 43 | for item in items: 44 | if not self._graph.has_node(item): 45 | self._graph.add_node(item) 46 | if self._last_item is not self._no_last_item: 47 | self._graph.add_edge(self._last_item, item, 48 | attr_dict={flow.LINK_INVARIANT: True}) 49 | self._last_item = item 50 | return self 51 | 52 | def __len__(self): 53 | return len(self._graph) 54 | 55 | def __iter__(self): 56 | yield from self._graph.nodes 57 | 58 | @property 59 | def requires(self): 60 | requires = set() 61 | prior_provides = set() 62 | if self._retry is not None: 63 | requires.update(self._retry.requires) 64 | prior_provides.update(self._retry.provides) 65 | for item in self: 66 | requires.update(item.requires - prior_provides) 67 | prior_provides.update(item.provides) 68 | return frozenset(requires) 69 | 70 | def iter_nodes(self): 71 | yield from self._graph.nodes(data=True) 72 | 73 | def iter_links(self): 74 | yield from self._graph.edges(data=True) 75 | -------------------------------------------------------------------------------- /taskflow/patterns/unordered_flow.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2012 Yahoo! Inc. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); you may 4 | # not use this file except in compliance with the License. You may obtain 5 | # a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 11 | # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 12 | # License for the specific language governing permissions and limitations 13 | # under the License. 14 | 15 | from taskflow import flow 16 | from taskflow.types import graph as gr 17 | 18 | 19 | class Flow(flow.Flow): 20 | """Unordered flow pattern. 21 | 22 | A unordered (potentially nested) flow of *tasks/flows* that can be 23 | executed in any order as one unit and rolled back as one unit. 24 | """ 25 | 26 | def __init__(self, name, retry=None): 27 | super().__init__(name, retry) 28 | self._graph = gr.Graph(name=name) 29 | 30 | def add(self, *items): 31 | """Adds a given task/tasks/flow/flows to this flow.""" 32 | for item in items: 33 | if not self._graph.has_node(item): 34 | self._graph.add_node(item) 35 | return self 36 | 37 | def __len__(self): 38 | return len(self._graph) 39 | 40 | def __iter__(self): 41 | yield from self._graph 42 | 43 | def iter_links(self): 44 | yield from self._graph.edges(data=True) 45 | 46 | def iter_nodes(self): 47 | yield from self._graph.nodes(data=True) 48 | 49 | @property 50 | def requires(self): 51 | requires = set() 52 | retry_provides = set() 53 | if self._retry is not None: 54 | requires.update(self._retry.requires) 55 | retry_provides.update(self._retry.provides) 56 | for item in self: 57 | item_requires = item.requires - retry_provides 58 | requires.update(item_requires) 59 | return frozenset(requires) 60 | -------------------------------------------------------------------------------- /taskflow/persistence/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openstack/taskflow/d01920ef1c0e3432c5d3a1e7708a34ff834bf339/taskflow/persistence/__init__.py -------------------------------------------------------------------------------- /taskflow/persistence/backends/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2013 Rackspace Hosting Inc. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); you may 4 | # not use this file except in compliance with the License. You may obtain 5 | # a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 11 | # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 12 | # License for the specific language governing permissions and limitations 13 | # under the License. 14 | 15 | import contextlib 16 | 17 | from stevedore import driver 18 | 19 | from taskflow import exceptions as exc 20 | from taskflow import logging 21 | from taskflow.utils import misc 22 | 23 | 24 | # NOTE(harlowja): this is the entrypoint namespace, not the module namespace. 25 | BACKEND_NAMESPACE = 'taskflow.persistence' 26 | 27 | LOG = logging.getLogger(__name__) 28 | 29 | 30 | def fetch(conf, namespace=BACKEND_NAMESPACE, **kwargs): 31 | """Fetch a persistence backend with the given configuration. 32 | 33 | This fetch method will look for the entrypoint name in the entrypoint 34 | namespace, and then attempt to instantiate that entrypoint using the 35 | provided configuration and any persistence backend specific kwargs. 36 | 37 | NOTE(harlowja): to aid in making it easy to specify configuration and 38 | options to a backend the configuration (which is typical just a dictionary) 39 | can also be a URI string that identifies the entrypoint name and any 40 | configuration specific to that backend. 41 | 42 | For example, given the following configuration URI:: 43 | 44 | mysql:///?a=b&c=d 45 | 46 | This will look for the entrypoint named 'mysql' and will provide 47 | a configuration object composed of the URI's components, in this case that 48 | is ``{'a': 'b', 'c': 'd'}`` to the constructor of that persistence backend 49 | instance. 50 | """ 51 | backend, conf = misc.extract_driver_and_conf(conf, 'connection') 52 | # If the backend is like 'mysql+pymysql://...' which informs the 53 | # backend to use a dialect (supported by sqlalchemy at least) we just want 54 | # to look at the first component to find our entrypoint backend name... 55 | if backend.find("+") != -1: 56 | backend = backend.split("+", 1)[0] 57 | LOG.debug('Looking for %r backend driver in %r', backend, namespace) 58 | try: 59 | mgr = driver.DriverManager(namespace, backend, 60 | invoke_on_load=True, 61 | invoke_args=(conf,), 62 | invoke_kwds=kwargs) 63 | return mgr.driver 64 | except RuntimeError as e: 65 | raise exc.NotFound("Could not find backend {}: {}".format(backend, e)) 66 | 67 | 68 | @contextlib.contextmanager 69 | def backend(conf, namespace=BACKEND_NAMESPACE, **kwargs): 70 | """Fetches a backend, connects, upgrades, then closes it on completion. 71 | 72 | This allows a backend instance to be fetched, connected to, have its schema 73 | upgraded (if the schema is already up to date this is a no-op) and then 74 | used in a context manager statement with the backend being closed upon 75 | context manager exit. 76 | """ 77 | with contextlib.closing(fetch(conf, namespace=namespace, **kwargs)) as be: 78 | with contextlib.closing(be.get_connection()) as conn: 79 | conn.upgrade() 80 | yield be 81 | -------------------------------------------------------------------------------- /taskflow/persistence/backends/sqlalchemy/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openstack/taskflow/d01920ef1c0e3432c5d3a1e7708a34ff834bf339/taskflow/persistence/backends/sqlalchemy/__init__.py -------------------------------------------------------------------------------- /taskflow/persistence/backends/sqlalchemy/alembic/README: -------------------------------------------------------------------------------- 1 | Please see https://alembic.readthedocs.org/en/latest/index.html for general documentation 2 | 3 | To create alembic migrations you need to have alembic installed and available in PATH: 4 | # pip install alembic 5 | $ cd ./taskflow/persistence/backends/sqlalchemy/alembic 6 | $ alembic revision -m "migration_description" 7 | 8 | See Operation Reference https://alembic.readthedocs.org/en/latest/ops.html#ops 9 | for a short list of commands 10 | -------------------------------------------------------------------------------- /taskflow/persistence/backends/sqlalchemy/alembic/alembic.ini: -------------------------------------------------------------------------------- 1 | # A generic, single database configuration. 2 | 3 | [alembic] 4 | # path to migration scripts 5 | script_location = %(here)s 6 | 7 | # template used to generate migration files 8 | # file_template = %%(rev)s_%%(slug)s 9 | 10 | # set to 'true' to run the environment during 11 | # the 'revision' command, regardless of autogenerate 12 | # revision_environment = false 13 | 14 | # This is set inside of migration script 15 | # sqlalchemy.url = driver://user:pass@localhost/dbname 16 | -------------------------------------------------------------------------------- /taskflow/persistence/backends/sqlalchemy/alembic/env.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2013 Yahoo! Inc. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); you may 4 | # not use this file except in compliance with the License. You may obtain 5 | # a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 11 | # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 12 | # License for the specific language governing permissions and limitations 13 | # under the License. 14 | from alembic import context 15 | from sqlalchemy import engine_from_config, pool 16 | 17 | # this is the Alembic Config object, which provides 18 | # access to the values within the .ini file in use. 19 | config = context.config 20 | 21 | # add your model's MetaData object here 22 | # for 'autogenerate' support 23 | # from myapp import mymodel 24 | # target_metadata = mymodel.Base.metadata 25 | target_metadata = None 26 | 27 | # other values from the config, defined by the needs of env.py, 28 | # can be acquired: 29 | # my_important_option = config.get_main_option("my_important_option") 30 | # ... etc. 31 | 32 | 33 | def run_migrations_offline(): 34 | """Run migrations in 'offline' mode. 35 | 36 | This configures the context with just a URL 37 | and not an Engine, though an Engine is acceptable 38 | here as well. By skipping the Engine creation 39 | we don't even need a DBAPI to be available. 40 | 41 | Calls to context.execute() here emit the given string to the 42 | script output. 43 | 44 | """ 45 | url = config.get_main_option("sqlalchemy.url") 46 | context.configure(url=url) 47 | 48 | with context.begin_transaction(): 49 | context.run_migrations() 50 | 51 | 52 | def run_migrations_online(): 53 | """Run migrations in 'online' mode. 54 | 55 | In this scenario we need to create an Engine 56 | and associate a connection with the context. 57 | 58 | """ 59 | connectable = config.attributes.get('connection', None) 60 | if connectable is None: 61 | connectable = engine_from_config( 62 | config.get_section(config.config_ini_section), 63 | prefix='sqlalchemy.', poolclass=pool.NullPool) 64 | with connectable.connect() as connection: 65 | context.configure(connection=connection, 66 | target_metadata=target_metadata) 67 | with context.begin_transaction(): 68 | context.run_migrations() 69 | else: 70 | context.configure( 71 | connection=connectable, 72 | target_metadata=target_metadata) 73 | with context.begin_transaction(): 74 | context.run_migrations() 75 | 76 | if context.is_offline_mode(): 77 | run_migrations_offline() 78 | else: 79 | run_migrations_online() 80 | -------------------------------------------------------------------------------- /taskflow/persistence/backends/sqlalchemy/alembic/script.py.mako: -------------------------------------------------------------------------------- 1 | """${message} 2 | 3 | Revision ID: ${up_revision} 4 | Revises: ${down_revision} 5 | Create Date: ${create_date} 6 | 7 | """ 8 | 9 | # revision identifiers, used by Alembic. 10 | revision = ${repr(up_revision)} 11 | down_revision = ${repr(down_revision)} 12 | 13 | from alembic import op 14 | import sqlalchemy as sa 15 | ${imports if imports else ""} 16 | 17 | def upgrade(): 18 | ${upgrades if upgrades else "pass"} 19 | 20 | 21 | def downgrade(): 22 | ${downgrades if downgrades else "pass"} 23 | -------------------------------------------------------------------------------- /taskflow/persistence/backends/sqlalchemy/alembic/versions/00af93df9d77_add_unique_into_all_indexes.py: -------------------------------------------------------------------------------- 1 | # Licensed under the Apache License, Version 2.0 (the "License"); you may 2 | # not use this file except in compliance with the License. You may obtain 3 | # a copy of the License at 4 | # 5 | # http://www.apache.org/licenses/LICENSE-2.0 6 | # 7 | # Unless required by applicable law or agreed to in writing, software 8 | # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 9 | # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 10 | # License for the specific language governing permissions and limitations 11 | # under the License. 12 | 13 | """Add unique into all indexes 14 | 15 | Revision ID: 00af93df9d77 16 | Revises: 40fc8c914bd2 17 | Create Date: 2025-02-28 15:44:37.066720 18 | 19 | """ 20 | 21 | # revision identifiers, used by Alembic. 22 | revision = '00af93df9d77' 23 | down_revision = '40fc8c914bd2' 24 | 25 | from alembic import op 26 | 27 | 28 | def upgrade(): 29 | bind = op.get_bind() 30 | engine = bind.engine 31 | if engine.name == 'mysql': 32 | with op.batch_alter_table("logbooks") as batch_op: 33 | batch_op.drop_index("logbook_uuid_idx") 34 | batch_op.create_index( 35 | index_name="logbook_uuid_idx", 36 | columns=['uuid'], 37 | unique=True) 38 | 39 | with op.batch_alter_table("flowdetails") as batch_op: 40 | batch_op.drop_index("flowdetails_uuid_idx") 41 | batch_op.create_index( 42 | index_name="flowdetails_uuid_idx", 43 | columns=['uuid'], 44 | unique=True) 45 | 46 | with op.batch_alter_table("atomdetails") as batch_op: 47 | batch_op.drop_index("taskdetails_uuid_idx") 48 | batch_op.create_index( 49 | index_name="taskdetails_uuid_idx", 50 | columns=['uuid'], 51 | unique=True) 52 | 53 | 54 | def downgrade(): 55 | bind = op.get_bind() 56 | engine = bind.engine 57 | if engine.name == 'mysql': 58 | with op.batch_alter_table("logbooks") as batch_op: 59 | batch_op.drop_index("logbook_uuid_idx") 60 | batch_op.create_index( 61 | index_name="logbook_uuid_idx", 62 | columns=['uuid']) 63 | 64 | with op.batch_alter_table("flowdetails") as batch_op: 65 | batch_op.drop_index("flowdetails_uuid_idx") 66 | batch_op.create_index( 67 | index_name="flowdetails_uuid_idx", 68 | columns=['uuid']) 69 | 70 | with op.batch_alter_table("atomdetails") as batch_op: 71 | batch_op.drop_index("taskdetails_uuid_idx") 72 | batch_op.create_index( 73 | index_name="taskdetails_uuid_idx", 74 | columns=['uuid']) 75 | -------------------------------------------------------------------------------- /taskflow/persistence/backends/sqlalchemy/alembic/versions/0bc3e1a3c135_set_result_meduimtext_type.py: -------------------------------------------------------------------------------- 1 | # Licensed under the Apache License, Version 2.0 (the "License"); you may 2 | # not use this file except in compliance with the License. You may obtain 3 | # a copy of the License at 4 | # 5 | # http://www.apache.org/licenses/LICENSE-2.0 6 | # 7 | # Unless required by applicable law or agreed to in writing, software 8 | # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 9 | # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 10 | # License for the specific language governing permissions and limitations 11 | # under the License. 12 | 13 | """set_result_meduimtext_type 14 | 15 | Revision ID: 0bc3e1a3c135 16 | Revises: 2ad4984f2864 17 | Create Date: 2019-08-08 16:11:36.221164 18 | 19 | """ 20 | 21 | # revision identifiers, used by Alembic. 22 | revision = '0bc3e1a3c135' 23 | down_revision = '2ad4984f2864' 24 | 25 | from alembic import op 26 | import sqlalchemy as sa 27 | from sqlalchemy.dialects import mysql 28 | 29 | 30 | def upgrade(): 31 | bind = op.get_bind() 32 | engine = bind.engine 33 | if engine.name == 'mysql': 34 | op.alter_column('atomdetails', 'results', type_=mysql.LONGTEXT, 35 | existing_nullable=True) 36 | 37 | 38 | def downgrade(): 39 | bind = op.get_bind() 40 | engine = bind.engine 41 | if engine.name == 'mysql': 42 | op.alter_column('atomdetails', 'results', type_=sa.Text(), 43 | existing_nullable=True) 44 | -------------------------------------------------------------------------------- /taskflow/persistence/backends/sqlalchemy/alembic/versions/14b227d79a87_add_intention_column.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2012-2013 Yahoo! Inc. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); you may 4 | # not use this file except in compliance with the License. You may obtain 5 | # a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 11 | # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 12 | # License for the specific language governing permissions and limitations 13 | # under the License. 14 | 15 | # revision identifiers, used by Alembic. 16 | revision = '14b227d79a87' 17 | down_revision = '84d6e888850' 18 | 19 | from alembic import op 20 | import sqlalchemy as sa 21 | 22 | from taskflow import states 23 | 24 | 25 | def upgrade(): 26 | bind = op.get_bind() 27 | intention_type = sa.Enum(*states.INTENTIONS, name='intention_type') 28 | column = sa.Column('intention', intention_type, 29 | server_default=states.EXECUTE) 30 | impl = intention_type.dialect_impl(bind.dialect) 31 | impl.create(bind, checkfirst=True) 32 | op.add_column('taskdetails', column) 33 | 34 | 35 | def downgrade(): 36 | op.drop_column('taskdetails', 'intention') 37 | -------------------------------------------------------------------------------- /taskflow/persistence/backends/sqlalchemy/alembic/versions/1c783c0c2875_replace_exception_an.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2012-2013 Yahoo! Inc. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); you may 4 | # not use this file except in compliance with the License. You may obtain 5 | # a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 11 | # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 12 | # License for the specific language governing permissions and limitations 13 | # under the License. 14 | 15 | """Replace exception and stacktrace with failure column 16 | 17 | Revision ID: 1c783c0c2875 18 | Revises: 1cea328f0f65 19 | Create Date: 2013-09-26 12:33:30.970122 20 | 21 | """ 22 | 23 | # revision identifiers, used by Alembic. 24 | revision = '1c783c0c2875' 25 | down_revision = '1cea328f0f65' 26 | 27 | from alembic import op 28 | import sqlalchemy as sa 29 | 30 | 31 | def upgrade(): 32 | op.add_column('taskdetails', 33 | sa.Column('failure', sa.Text(), nullable=True)) 34 | op.drop_column('taskdetails', 'exception') 35 | op.drop_column('taskdetails', 'stacktrace') 36 | 37 | 38 | def downgrade(): 39 | op.drop_column('taskdetails', 'failure') 40 | op.add_column('taskdetails', 41 | sa.Column('stacktrace', sa.Text(), nullable=True)) 42 | op.add_column('taskdetails', 43 | sa.Column('exception', sa.Text(), nullable=True)) 44 | -------------------------------------------------------------------------------- /taskflow/persistence/backends/sqlalchemy/alembic/versions/2ad4984f2864_switch_postgres_to_json_native.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2012-2013 Yahoo! Inc. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); you may 4 | # not use this file except in compliance with the License. You may obtain 5 | # a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 11 | # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 12 | # License for the specific language governing permissions and limitations 13 | # under the License. 14 | 15 | """Switch postgres to json native type. 16 | 17 | Revision ID: 2ad4984f2864 18 | Revises: 3162c0f3f8e4 19 | Create Date: 2015-06-04 13:08:36.667948 20 | 21 | """ 22 | 23 | # revision identifiers, used by Alembic. 24 | revision = '2ad4984f2864' 25 | down_revision = '3162c0f3f8e4' 26 | 27 | from alembic import op 28 | 29 | 30 | _ALTER_TO_JSON_TPL = 'ALTER TABLE %s ALTER COLUMN %s TYPE JSON USING %s::JSON' 31 | _TABLES_COLS = tuple([ 32 | ('logbooks', 'meta'), 33 | ('flowdetails', 'meta'), 34 | ('atomdetails', 'meta'), 35 | ('atomdetails', 'failure'), 36 | ('atomdetails', 'revert_failure'), 37 | ('atomdetails', 'results'), 38 | ('atomdetails', 'revert_results'), 39 | ]) 40 | _ALTER_TO_TEXT_TPL = 'ALTER TABLE %s ALTER COLUMN %s TYPE TEXT' 41 | 42 | 43 | def upgrade(): 44 | b = op.get_bind() 45 | if b.dialect.name.startswith('postgresql'): 46 | for (table_name, col_name) in _TABLES_COLS: 47 | q = _ALTER_TO_JSON_TPL % (table_name, col_name, col_name) 48 | op.execute(q) 49 | 50 | 51 | def downgrade(): 52 | b = op.get_bind() 53 | if b.dialect.name.startswith('postgresql'): 54 | for (table_name, col_name) in _TABLES_COLS: 55 | q = _ALTER_TO_TEXT_TPL % (table_name, col_name) 56 | op.execute(q) 57 | -------------------------------------------------------------------------------- /taskflow/persistence/backends/sqlalchemy/alembic/versions/3162c0f3f8e4_add_revert_results_and_revert_failure_.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2015 Yahoo! Inc. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); you may 4 | # not use this file except in compliance with the License. You may obtain 5 | # a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 11 | # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 12 | # License for the specific language governing permissions and limitations 13 | # under the License. 14 | 15 | """Add 'revert_results' and 'revert_failure' atom detail column. 16 | 17 | Revision ID: 3162c0f3f8e4 18 | Revises: 589dccdf2b6e 19 | Create Date: 2015-06-17 15:52:56.575245 20 | 21 | """ 22 | 23 | # revision identifiers, used by Alembic. 24 | revision = '3162c0f3f8e4' 25 | down_revision = '589dccdf2b6e' 26 | 27 | from alembic import op 28 | import sqlalchemy as sa 29 | 30 | 31 | def upgrade(): 32 | op.add_column('atomdetails', 33 | sa.Column('revert_results', sa.Text(), nullable=True)) 34 | op.add_column('atomdetails', 35 | sa.Column('revert_failure', sa.Text(), nullable=True)) 36 | 37 | 38 | def downgrade(): 39 | op.drop_column('atomdetails', 'revert_results') 40 | op.drop_column('atomdetails', 'revert_failure') 41 | -------------------------------------------------------------------------------- /taskflow/persistence/backends/sqlalchemy/alembic/versions/40fc8c914bd2_fix_atomdetails_failure_size.py: -------------------------------------------------------------------------------- 1 | # Licensed under the Apache License, Version 2.0 (the "License"); you may 2 | # not use this file except in compliance with the License. You may obtain 3 | # a copy of the License at 4 | # 5 | # http://www.apache.org/licenses/LICENSE-2.0 6 | # 7 | # Unless required by applicable law or agreed to in writing, software 8 | # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 9 | # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 10 | # License for the specific language governing permissions and limitations 11 | # under the License. 12 | 13 | """fix atomdetails failure size 14 | 15 | Revision ID: 40fc8c914bd2 16 | Revises: 6df9422fcb43 17 | Create Date: 2022-01-27 18:10:06.176006 18 | 19 | """ 20 | 21 | # revision identifiers, used by Alembic. 22 | revision = '40fc8c914bd2' 23 | down_revision = '6df9422fcb43' 24 | 25 | from alembic import op 26 | from sqlalchemy.dialects import mysql 27 | 28 | 29 | def upgrade(): 30 | bind = op.get_bind() 31 | engine = bind.engine 32 | if engine.name == 'mysql': 33 | op.alter_column('atomdetails', 'failure', type_=mysql.LONGTEXT, 34 | existing_nullable=True) 35 | op.alter_column('atomdetails', 'revert_failure', type_=mysql.LONGTEXT, 36 | existing_nullable=True) 37 | -------------------------------------------------------------------------------- /taskflow/persistence/backends/sqlalchemy/alembic/versions/589dccdf2b6e_rename_taskdetails_to_atomdetails.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2014 Yahoo! Inc. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); you may 4 | # not use this file except in compliance with the License. You may obtain 5 | # a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 11 | # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 12 | # License for the specific language governing permissions and limitations 13 | # under the License. 14 | 15 | """Rename taskdetails to atomdetails 16 | 17 | Revision ID: 589dccdf2b6e 18 | Revises: 14b227d79a87 19 | Create Date: 2014-03-19 11:49:16.533227 20 | 21 | """ 22 | 23 | # revision identifiers, used by Alembic. 24 | revision = '589dccdf2b6e' 25 | down_revision = '14b227d79a87' 26 | 27 | from alembic import op 28 | 29 | 30 | def upgrade(): 31 | op.rename_table("taskdetails", "atomdetails") 32 | 33 | 34 | def downgrade(): 35 | op.rename_table("atomdetails", "taskdetails") 36 | -------------------------------------------------------------------------------- /taskflow/persistence/backends/sqlalchemy/alembic/versions/6df9422fcb43_fix_flowdetails_meta_size.py: -------------------------------------------------------------------------------- 1 | # Licensed under the Apache License, Version 2.0 (the "License"); you may 2 | # not use this file except in compliance with the License. You may obtain 3 | # a copy of the License at 4 | # 5 | # http://www.apache.org/licenses/LICENSE-2.0 6 | # 7 | # Unless required by applicable law or agreed to in writing, software 8 | # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 9 | # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 10 | # License for the specific language governing permissions and limitations 11 | # under the License. 12 | 13 | """fix flowdetails meta size 14 | 15 | Revision ID: 6df9422fcb43 16 | Revises: 0bc3e1a3c135 17 | Create Date: 2021-04-27 14:51:53.618249 18 | 19 | """ 20 | 21 | # revision identifiers, used by Alembic. 22 | revision = '6df9422fcb43' 23 | down_revision = '0bc3e1a3c135' 24 | 25 | from alembic import op 26 | from sqlalchemy.dialects import mysql 27 | 28 | 29 | def upgrade(): 30 | bind = op.get_bind() 31 | engine = bind.engine 32 | if engine.name == 'mysql': 33 | op.alter_column('flowdetails', 'meta', type_=mysql.LONGTEXT, 34 | existing_nullable=True) 35 | -------------------------------------------------------------------------------- /taskflow/persistence/backends/sqlalchemy/alembic/versions/84d6e888850_add_task_detail_type.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2012-2013 Yahoo! Inc. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); you may 4 | # not use this file except in compliance with the License. You may obtain 5 | # a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 11 | # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 12 | # License for the specific language governing permissions and limitations 13 | # under the License. 14 | 15 | """Add task detail type 16 | 17 | Revision ID: 84d6e888850 18 | Revises: 1c783c0c2875 19 | Create Date: 2014-01-20 18:12:42.503267 20 | 21 | """ 22 | 23 | # revision identifiers, used by Alembic. 24 | revision = '84d6e888850' 25 | down_revision = '1c783c0c2875' 26 | 27 | from alembic import op 28 | import sqlalchemy as sa 29 | 30 | from taskflow.persistence import models 31 | 32 | 33 | def upgrade(): 34 | atom_types = sa.Enum(*models.ATOM_TYPES, name='atom_types') 35 | column = sa.Column('atom_type', atom_types) 36 | bind = op.get_bind() 37 | impl = atom_types.dialect_impl(bind.dialect) 38 | impl.create(bind, checkfirst=True) 39 | op.add_column('taskdetails', column) 40 | 41 | 42 | def downgrade(): 43 | op.drop_column('taskdetails', 'atom_type') 44 | -------------------------------------------------------------------------------- /taskflow/persistence/backends/sqlalchemy/alembic/versions/README: -------------------------------------------------------------------------------- 1 | Directory for alembic migration files 2 | -------------------------------------------------------------------------------- /taskflow/persistence/backends/sqlalchemy/migration.py: -------------------------------------------------------------------------------- 1 | # Copyright 2010 United States Government as represented by the 2 | # Administrator of the National Aeronautics and Space Administration. 3 | # All Rights Reserved. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); you may 6 | # not use this file except in compliance with the License. You may obtain 7 | # a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 13 | # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 14 | # License for the specific language governing permissions and limitations 15 | # under the License. 16 | 17 | """Database setup and migration commands.""" 18 | 19 | import os 20 | 21 | from alembic import command 22 | from alembic import config 23 | 24 | 25 | def _make_alembic_config(): 26 | path = os.path.join(os.path.dirname(__file__), 'alembic', 'alembic.ini') 27 | return config.Config(path) 28 | 29 | 30 | def db_sync(connection, revision='head'): 31 | cfg = _make_alembic_config() 32 | cfg.attributes['connection'] = connection 33 | command.upgrade(cfg, revision) 34 | -------------------------------------------------------------------------------- /taskflow/tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openstack/taskflow/d01920ef1c0e3432c5d3a1e7708a34ff834bf339/taskflow/tests/__init__.py -------------------------------------------------------------------------------- /taskflow/tests/fixtures.py: -------------------------------------------------------------------------------- 1 | # Licensed under the Apache License, Version 2.0 (the "License"); you may 2 | # not use this file except in compliance with the License. You may obtain 3 | # a copy of the License at 4 | # 5 | # http://www.apache.org/licenses/LICENSE-2.0 6 | # 7 | # Unless required by applicable law or agreed to in writing, software 8 | # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 9 | # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 10 | # License for the specific language governing permissions and limitations 11 | # under the License. 12 | 13 | import warnings 14 | 15 | import fixtures 16 | from sqlalchemy import exc as sqla_exc 17 | 18 | 19 | class WarningsFixture(fixtures.Fixture): 20 | """Filters out warnings during test runs.""" 21 | def setUp(self): 22 | super().setUp() 23 | 24 | self._original_warning_filters = warnings.filters[:] 25 | 26 | warnings.simplefilter('once', DeprecationWarning) 27 | 28 | # The UUIDFields emits a warning if the value is not a valid UUID. 29 | # Let's escalate that to an exception in the test to prevent adding 30 | # violations. 31 | 32 | warnings.filterwarnings('error', message='.*invalid UUID.*') 33 | 34 | # Enable deprecation warnings for taskflow itself to capture upcoming 35 | # SQLAlchemy changes 36 | warnings.filterwarnings( 37 | 'ignore', 38 | category=sqla_exc.SADeprecationWarning, 39 | ) 40 | 41 | warnings.filterwarnings( 42 | 'error', 43 | module='taskflow', 44 | category=sqla_exc.SADeprecationWarning, 45 | ) 46 | 47 | # Enable general SQLAlchemy warnings also to ensure we're not doing 48 | # silly stuff. It's possible that we'll need to filter things out here 49 | # with future SQLAlchemy versions, but that's a good thing 50 | warnings.filterwarnings( 51 | 'error', 52 | module='taskflow', 53 | category=sqla_exc.SAWarning, 54 | ) 55 | self.addCleanup(self._reset_warning_filters) 56 | 57 | def _reset_warning_filters(self): 58 | warnings.filters[:] = self._original_warning_filters 59 | -------------------------------------------------------------------------------- /taskflow/tests/unit/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openstack/taskflow/d01920ef1c0e3432c5d3a1e7708a34ff834bf339/taskflow/tests/unit/__init__.py -------------------------------------------------------------------------------- /taskflow/tests/unit/action_engine/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openstack/taskflow/d01920ef1c0e3432c5d3a1e7708a34ff834bf339/taskflow/tests/unit/action_engine/__init__.py -------------------------------------------------------------------------------- /taskflow/tests/unit/action_engine/test_creation.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2014 Yahoo! Inc. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); you may 4 | # not use this file except in compliance with the License. You may obtain 5 | # a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 11 | # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 12 | # License for the specific language governing permissions and limitations 13 | # under the License. 14 | 15 | import futurist 16 | import testtools 17 | 18 | from taskflow.engines.action_engine import engine 19 | from taskflow.engines.action_engine import executor 20 | from taskflow.patterns import linear_flow as lf 21 | from taskflow.persistence import backends 22 | from taskflow import test 23 | from taskflow.tests import utils 24 | from taskflow.utils import eventlet_utils as eu 25 | from taskflow.utils import persistence_utils as pu 26 | 27 | 28 | class ParallelCreationTest(test.TestCase): 29 | @staticmethod 30 | def _create_engine(**kwargs): 31 | flow = lf.Flow('test-flow').add(utils.DummyTask()) 32 | backend = backends.fetch({'connection': 'memory'}) 33 | flow_detail = pu.create_flow_detail(flow, backend=backend) 34 | options = kwargs.copy() 35 | return engine.ParallelActionEngine(flow, flow_detail, 36 | backend, options) 37 | 38 | def test_thread_string_creation(self): 39 | for s in ['threads', 'threaded', 'thread']: 40 | eng = self._create_engine(executor=s) 41 | self.assertIsInstance(eng._task_executor, 42 | executor.ParallelThreadTaskExecutor) 43 | 44 | def test_thread_executor_creation(self): 45 | with futurist.ThreadPoolExecutor(1) as e: 46 | eng = self._create_engine(executor=e) 47 | self.assertIsInstance(eng._task_executor, 48 | executor.ParallelThreadTaskExecutor) 49 | 50 | @testtools.skipIf(not eu.EVENTLET_AVAILABLE, 'eventlet is not available') 51 | def test_green_executor_creation(self): 52 | with futurist.GreenThreadPoolExecutor(1) as e: 53 | eng = self._create_engine(executor=e) 54 | self.assertIsInstance(eng._task_executor, 55 | executor.ParallelThreadTaskExecutor) 56 | 57 | def test_sync_executor_creation(self): 58 | with futurist.SynchronousExecutor() as e: 59 | eng = self._create_engine(executor=e) 60 | self.assertIsInstance(eng._task_executor, 61 | executor.ParallelThreadTaskExecutor) 62 | 63 | def test_invalid_creation(self): 64 | self.assertRaises(ValueError, self._create_engine, executor='crap') 65 | self.assertRaises(TypeError, self._create_engine, executor=2) 66 | self.assertRaises(TypeError, self._create_engine, executor=object()) 67 | -------------------------------------------------------------------------------- /taskflow/tests/unit/jobs/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openstack/taskflow/d01920ef1c0e3432c5d3a1e7708a34ff834bf339/taskflow/tests/unit/jobs/__init__.py -------------------------------------------------------------------------------- /taskflow/tests/unit/jobs/test_entrypoint.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2014 Yahoo! Inc. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); you may 4 | # not use this file except in compliance with the License. You may obtain 5 | # a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 11 | # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 12 | # License for the specific language governing permissions and limitations 13 | # under the License. 14 | 15 | import contextlib 16 | 17 | from zake import fake_client 18 | 19 | from taskflow.jobs import backends 20 | from taskflow.jobs.backends import impl_redis 21 | from taskflow.jobs.backends import impl_zookeeper 22 | from taskflow import test 23 | 24 | 25 | class BackendFetchingTest(test.TestCase): 26 | def test_zk_entry_point_text(self): 27 | conf = 'zookeeper' 28 | with contextlib.closing(backends.fetch('test', conf)) as be: 29 | self.assertIsInstance(be, impl_zookeeper.ZookeeperJobBoard) 30 | 31 | def test_zk_entry_point(self): 32 | conf = { 33 | 'board': 'zookeeper', 34 | } 35 | with contextlib.closing(backends.fetch('test', conf)) as be: 36 | self.assertIsInstance(be, impl_zookeeper.ZookeeperJobBoard) 37 | 38 | def test_zk_entry_point_existing_client(self): 39 | existing_client = fake_client.FakeClient() 40 | conf = { 41 | 'board': 'zookeeper', 42 | } 43 | kwargs = { 44 | 'client': existing_client, 45 | } 46 | with contextlib.closing(backends.fetch('test', conf, **kwargs)) as be: 47 | self.assertIsInstance(be, impl_zookeeper.ZookeeperJobBoard) 48 | self.assertIs(existing_client, be._client) 49 | 50 | def test_redis_entry_point_text(self): 51 | conf = 'redis' 52 | with contextlib.closing(backends.fetch('test', conf)) as be: 53 | self.assertIsInstance(be, impl_redis.RedisJobBoard) 54 | 55 | def test_redis_entry_point(self): 56 | conf = { 57 | 'board': 'redis', 58 | } 59 | with contextlib.closing(backends.fetch('test', conf)) as be: 60 | self.assertIsInstance(be, impl_redis.RedisJobBoard) 61 | -------------------------------------------------------------------------------- /taskflow/tests/unit/patterns/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openstack/taskflow/d01920ef1c0e3432c5d3a1e7708a34ff834bf339/taskflow/tests/unit/patterns/__init__.py -------------------------------------------------------------------------------- /taskflow/tests/unit/persistence/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openstack/taskflow/d01920ef1c0e3432c5d3a1e7708a34ff834bf339/taskflow/tests/unit/persistence/__init__.py -------------------------------------------------------------------------------- /taskflow/tests/unit/test_deciders.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2015 Yahoo! Inc. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); you may 4 | # not use this file except in compliance with the License. You may obtain 5 | # a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 11 | # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 12 | # License for the specific language governing permissions and limitations 13 | # under the License. 14 | 15 | from taskflow import deciders 16 | from taskflow import test 17 | 18 | 19 | class TestDeciders(test.TestCase): 20 | def test_translate(self): 21 | for val in ['all', 'ALL', 'aLL', deciders.Depth.ALL]: 22 | self.assertEqual(deciders.Depth.ALL, 23 | deciders.Depth.translate(val)) 24 | for val in ['atom', 'ATOM', 'atOM', deciders.Depth.ATOM]: 25 | self.assertEqual(deciders.Depth.ATOM, 26 | deciders.Depth.translate(val)) 27 | for val in ['neighbors', 'Neighbors', 28 | 'NEIGHBORS', deciders.Depth.NEIGHBORS]: 29 | self.assertEqual(deciders.Depth.NEIGHBORS, 30 | deciders.Depth.translate(val)) 31 | for val in ['flow', 'FLOW', 'flOW', deciders.Depth.FLOW]: 32 | self.assertEqual(deciders.Depth.FLOW, 33 | deciders.Depth.translate(val)) 34 | 35 | def test_bad_translate(self): 36 | self.assertRaises(TypeError, deciders.Depth.translate, 3) 37 | self.assertRaises(TypeError, deciders.Depth.translate, object()) 38 | self.assertRaises(ValueError, deciders.Depth.translate, "stuff") 39 | 40 | def test_pick_widest(self): 41 | choices = [deciders.Depth.ATOM, deciders.Depth.FLOW] 42 | self.assertEqual(deciders.Depth.FLOW, deciders.pick_widest(choices)) 43 | choices = [deciders.Depth.ATOM, deciders.Depth.FLOW, 44 | deciders.Depth.ALL] 45 | self.assertEqual(deciders.Depth.ALL, deciders.pick_widest(choices)) 46 | choices = [deciders.Depth.ATOM, deciders.Depth.FLOW, 47 | deciders.Depth.ALL, deciders.Depth.NEIGHBORS] 48 | self.assertEqual(deciders.Depth.ALL, deciders.pick_widest(choices)) 49 | choices = [deciders.Depth.ATOM, deciders.Depth.NEIGHBORS] 50 | self.assertEqual(deciders.Depth.NEIGHBORS, 51 | deciders.pick_widest(choices)) 52 | 53 | def test_bad_pick_widest(self): 54 | self.assertRaises(ValueError, deciders.pick_widest, []) 55 | self.assertRaises(ValueError, deciders.pick_widest, ["a"]) 56 | self.assertRaises(ValueError, deciders.pick_widest, {'b'}) 57 | -------------------------------------------------------------------------------- /taskflow/tests/unit/test_functor_task.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2012-2013 Yahoo! Inc. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); you may 4 | # not use this file except in compliance with the License. You may obtain 5 | # a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 11 | # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 12 | # License for the specific language governing permissions and limitations 13 | # under the License. 14 | 15 | import taskflow.engines 16 | from taskflow.patterns import linear_flow 17 | from taskflow import task as base 18 | from taskflow import test 19 | 20 | 21 | def add(a, b): 22 | return a + b 23 | 24 | 25 | class BunchOfFunctions: 26 | 27 | def __init__(self, values): 28 | self.values = values 29 | 30 | def run_one(self, *args, **kwargs): 31 | self.values.append('one') 32 | 33 | def revert_one(self, *args, **kwargs): 34 | self.values.append('revert one') 35 | 36 | def run_fail(self, *args, **kwargs): 37 | self.values.append('fail') 38 | raise RuntimeError('Woot!') 39 | 40 | 41 | five = lambda: 5 42 | 43 | multiply = lambda x, y: x * y 44 | 45 | 46 | class FunctorTaskTest(test.TestCase): 47 | 48 | def test_simple(self): 49 | task = base.FunctorTask(add) 50 | self.assertEqual(__name__ + '.add', task.name) 51 | 52 | def test_other_name(self): 53 | task = base.FunctorTask(add, name='my task') 54 | self.assertEqual('my task', task.name) 55 | 56 | def test_it_runs(self): 57 | values = [] 58 | bof = BunchOfFunctions(values) 59 | t = base.FunctorTask 60 | 61 | flow = linear_flow.Flow('test') 62 | flow.add( 63 | t(bof.run_one, revert=bof.revert_one), 64 | t(bof.run_fail) 65 | ) 66 | self.assertRaisesRegex(RuntimeError, '^Woot', 67 | taskflow.engines.run, flow) 68 | self.assertEqual(['one', 'fail', 'revert one'], values) 69 | 70 | def test_lambda_functors(self): 71 | t = base.FunctorTask 72 | 73 | flow = linear_flow.Flow('test') 74 | flow.add( 75 | t(five, provides='five', name='five'), 76 | t(multiply, provides='product', name='product') 77 | ) 78 | 79 | flow_store = { 80 | 'x': 2, 81 | 'y': 3 82 | } 83 | 84 | result = taskflow.engines.run(flow, store=flow_store) 85 | 86 | expected = flow_store.copy() 87 | expected.update({ 88 | 'five': 5, 89 | 'product': 6 90 | }) 91 | 92 | self.assertDictEqual(expected, result) 93 | -------------------------------------------------------------------------------- /taskflow/tests/unit/test_mapfunctor_task.py: -------------------------------------------------------------------------------- 1 | # Copyright 2015 Hewlett-Packard Development Company, L.P. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); you may 4 | # not use this file except in compliance with the License. You may obtain 5 | # a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 11 | # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 12 | # License for the specific language governing permissions and limitations 13 | # under the License. 14 | 15 | import taskflow.engines as engines 16 | from taskflow.patterns import linear_flow 17 | from taskflow import task as base 18 | from taskflow import test 19 | 20 | 21 | def double(x): 22 | return x * 2 23 | 24 | square = lambda x: x * x 25 | 26 | 27 | class MapFunctorTaskTest(test.TestCase): 28 | 29 | def setUp(self): 30 | super().setUp() 31 | 32 | self.flow_store = { 33 | 'a': 1, 34 | 'b': 2, 35 | 'c': 3, 36 | 'd': 4, 37 | 'e': 5, 38 | } 39 | 40 | def test_double_array(self): 41 | expected = self.flow_store.copy() 42 | expected.update({ 43 | 'double_a': 2, 44 | 'double_b': 4, 45 | 'double_c': 6, 46 | 'double_d': 8, 47 | 'double_e': 10, 48 | }) 49 | 50 | requires = self.flow_store.keys() 51 | provides = ["double_%s" % k for k in requires] 52 | 53 | flow = linear_flow.Flow("double array flow") 54 | flow.add(base.MapFunctorTask(double, requires=requires, 55 | provides=provides)) 56 | 57 | result = engines.run(flow, store=self.flow_store) 58 | self.assertDictEqual(expected, result) 59 | 60 | def test_square_array(self): 61 | expected = self.flow_store.copy() 62 | expected.update({ 63 | 'square_a': 1, 64 | 'square_b': 4, 65 | 'square_c': 9, 66 | 'square_d': 16, 67 | 'square_e': 25, 68 | }) 69 | 70 | requires = self.flow_store.keys() 71 | provides = ["square_%s" % k for k in requires] 72 | 73 | flow = linear_flow.Flow("square array flow") 74 | flow.add(base.MapFunctorTask(square, requires=requires, 75 | provides=provides)) 76 | 77 | result = engines.run(flow, store=self.flow_store) 78 | self.assertDictEqual(expected, result) 79 | -------------------------------------------------------------------------------- /taskflow/tests/unit/test_reducefunctor_task.py: -------------------------------------------------------------------------------- 1 | # Copyright 2015 Hewlett-Packard Development Company, L.P. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); you may 4 | # not use this file except in compliance with the License. You may obtain 5 | # a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 11 | # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 12 | # License for the specific language governing permissions and limitations 13 | # under the License. 14 | 15 | import taskflow.engines as engines 16 | from taskflow.patterns import linear_flow 17 | from taskflow import task as base 18 | from taskflow import test 19 | 20 | 21 | def sum(x, y): 22 | return x + y 23 | 24 | multiply = lambda x, y: x * y 25 | 26 | 27 | class ReduceFunctorTaskTest(test.TestCase): 28 | 29 | def setUp(self): 30 | super().setUp() 31 | 32 | self.flow_store = { 33 | 'a': 1, 34 | 'b': 2, 35 | 'c': 3, 36 | 'd': 4, 37 | 'e': 5, 38 | } 39 | 40 | def test_sum_array(self): 41 | expected = self.flow_store.copy() 42 | expected.update({ 43 | 'sum': 15 44 | }) 45 | 46 | requires = self.flow_store.keys() 47 | provides = 'sum' 48 | 49 | flow = linear_flow.Flow("sum array flow") 50 | flow.add(base.ReduceFunctorTask(sum, requires=requires, 51 | provides=provides)) 52 | 53 | result = engines.run(flow, store=self.flow_store) 54 | self.assertDictEqual(expected, result) 55 | 56 | def test_multiply_array(self): 57 | expected = self.flow_store.copy() 58 | expected.update({ 59 | 'product': 120 60 | }) 61 | 62 | requires = self.flow_store.keys() 63 | provides = 'product' 64 | 65 | flow = linear_flow.Flow("square array flow") 66 | flow.add(base.ReduceFunctorTask(multiply, requires=requires, 67 | provides=provides)) 68 | 69 | result = engines.run(flow, store=self.flow_store) 70 | self.assertDictEqual(expected, result) 71 | -------------------------------------------------------------------------------- /taskflow/tests/unit/test_utils_async_utils.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2013 Yahoo! Inc. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); you may 4 | # not use this file except in compliance with the License. You may obtain 5 | # a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 11 | # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 12 | # License for the specific language governing permissions and limitations 13 | # under the License. 14 | 15 | from taskflow import test 16 | from taskflow.utils import async_utils as au 17 | 18 | 19 | class MakeCompletedFutureTest(test.TestCase): 20 | 21 | def test_make_completed_future(self): 22 | result = object() 23 | future = au.make_completed_future(result) 24 | self.assertTrue(future.done()) 25 | self.assertIs(future.result(), result) 26 | -------------------------------------------------------------------------------- /taskflow/tests/unit/test_utils_binary.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2014 Yahoo! Inc. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); you may 4 | # not use this file except in compliance with the License. You may obtain 5 | # a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 11 | # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 12 | # License for the specific language governing permissions and limitations 13 | # under the License. 14 | 15 | from taskflow import test 16 | from taskflow.utils import misc 17 | 18 | 19 | def _bytes(data): 20 | return data.encode(encoding='utf-8') 21 | 22 | 23 | class BinaryEncodeTest(test.TestCase): 24 | 25 | def _check(self, data, expected_result): 26 | result = misc.binary_encode(data) 27 | self.assertIsInstance(result, bytes) 28 | self.assertEqual(expected_result, result) 29 | 30 | def test_simple_binary(self): 31 | data = _bytes('hello') 32 | self._check(data, data) 33 | 34 | def test_unicode_binary(self): 35 | data = _bytes('привет') 36 | self._check(data, data) 37 | 38 | def test_simple_text(self): 39 | self._check('hello', _bytes('hello')) 40 | 41 | def test_unicode_text(self): 42 | self._check('привет', _bytes('привет')) 43 | 44 | def test_unicode_other_encoding(self): 45 | result = misc.binary_encode('mañana', 'latin-1') 46 | self.assertIsInstance(result, bytes) 47 | self.assertEqual('mañana'.encode('latin-1'), result) 48 | 49 | 50 | class BinaryDecodeTest(test.TestCase): 51 | 52 | def _check(self, data, expected_result): 53 | result = misc.binary_decode(data) 54 | self.assertIsInstance(result, str) 55 | self.assertEqual(expected_result, result) 56 | 57 | def test_simple_text(self): 58 | data = 'hello' 59 | self._check(data, data) 60 | 61 | def test_unicode_text(self): 62 | data = 'привет' 63 | self._check(data, data) 64 | 65 | def test_simple_binary(self): 66 | self._check(_bytes('hello'), 'hello') 67 | 68 | def test_unicode_binary(self): 69 | self._check(_bytes('привет'), 'привет') 70 | 71 | def test_unicode_other_encoding(self): 72 | data = 'mañana'.encode('latin-1') 73 | result = misc.binary_decode(data, 'latin-1') 74 | self.assertIsInstance(result, str) 75 | self.assertEqual('mañana', result) 76 | 77 | 78 | class DecodeJsonTest(test.TestCase): 79 | 80 | def test_it_works(self): 81 | self.assertEqual({"foo": 1}, 82 | misc.decode_json(_bytes('{"foo": 1}'))) 83 | 84 | def test_it_works_with_unicode(self): 85 | data = _bytes('{"foo": "фуу"}') 86 | self.assertEqual({"foo": 'фуу'}, misc.decode_json(data)) 87 | 88 | def test_handles_invalid_unicode(self): 89 | self.assertRaises(ValueError, misc.decode_json, 90 | b'{"\xf1": 1}') 91 | 92 | def test_handles_bad_json(self): 93 | self.assertRaises(ValueError, misc.decode_json, 94 | _bytes('{"foo":')) 95 | 96 | def test_handles_wrong_types(self): 97 | self.assertRaises(ValueError, misc.decode_json, 98 | _bytes('42')) 99 | -------------------------------------------------------------------------------- /taskflow/tests/unit/test_utils_kazoo_utils.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) Red Hat 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); you may 4 | # not use this file except in compliance with the License. You may obtain 5 | # a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 11 | # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 12 | # License for the specific language governing permissions and limitations 13 | # under the License. 14 | 15 | from unittest import mock 16 | 17 | from taskflow import test 18 | from taskflow.utils import kazoo_utils 19 | 20 | 21 | class MakeClientTest(test.TestCase): 22 | 23 | @mock.patch("kazoo.client.KazooClient") 24 | def test_make_client_config(self, mock_kazoo_client): 25 | conf = {} 26 | expected = { 27 | 'hosts': 'localhost:2181', 28 | 'logger': mock.ANY, 29 | 'read_only': False, 30 | 'randomize_hosts': False, 31 | 'keyfile': None, 32 | 'keyfile_password': None, 33 | 'certfile': None, 34 | 'use_ssl': False, 35 | 'verify_certs': True 36 | } 37 | 38 | kazoo_utils.make_client(conf) 39 | 40 | mock_kazoo_client.assert_called_once_with(**expected) 41 | 42 | mock_kazoo_client.reset_mock() 43 | 44 | # With boolean passed as strings 45 | conf = { 46 | 'use_ssl': 'True', 47 | 'verify_certs': 'False' 48 | } 49 | expected = { 50 | 'hosts': 'localhost:2181', 51 | 'logger': mock.ANY, 52 | 'read_only': False, 53 | 'randomize_hosts': False, 54 | 'keyfile': None, 55 | 'keyfile_password': None, 56 | 'certfile': None, 57 | 'use_ssl': True, 58 | 'verify_certs': False 59 | } 60 | 61 | kazoo_utils.make_client(conf) 62 | 63 | mock_kazoo_client.assert_called_once_with(**expected) 64 | 65 | mock_kazoo_client.reset_mock() 66 | -------------------------------------------------------------------------------- /taskflow/tests/unit/worker_based/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openstack/taskflow/d01920ef1c0e3432c5d3a1e7708a34ff834bf339/taskflow/tests/unit/worker_based/__init__.py -------------------------------------------------------------------------------- /taskflow/tests/unit/worker_based/test_dispatcher.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2014 Yahoo! Inc. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); you may 4 | # not use this file except in compliance with the License. You may obtain 5 | # a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 11 | # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 12 | # License for the specific language governing permissions and limitations 13 | # under the License. 14 | 15 | try: 16 | from kombu import message # noqa 17 | except ImportError: 18 | from kombu.transport import base as message 19 | 20 | from taskflow.engines.worker_based import dispatcher 21 | from taskflow import test 22 | from taskflow.test import mock 23 | 24 | 25 | def mock_acked_message(ack_ok=True, **kwargs): 26 | msg = mock.create_autospec(message.Message, spec_set=True, instance=True, 27 | channel=None, **kwargs) 28 | 29 | def ack_side_effect(*args, **kwargs): 30 | msg.acknowledged = True 31 | 32 | if ack_ok: 33 | msg.ack_log_error.side_effect = ack_side_effect 34 | msg.acknowledged = False 35 | return msg 36 | 37 | 38 | class TestDispatcher(test.TestCase): 39 | def test_creation(self): 40 | on_hello = mock.MagicMock() 41 | handlers = {'hello': dispatcher.Handler(on_hello)} 42 | dispatcher.TypeDispatcher(type_handlers=handlers) 43 | 44 | def test_on_message(self): 45 | on_hello = mock.MagicMock() 46 | handlers = {'hello': dispatcher.Handler(on_hello)} 47 | d = dispatcher.TypeDispatcher(type_handlers=handlers) 48 | msg = mock_acked_message(properties={'type': 'hello'}) 49 | d.on_message("", msg) 50 | self.assertTrue(on_hello.called) 51 | self.assertTrue(msg.ack_log_error.called) 52 | self.assertTrue(msg.acknowledged) 53 | 54 | def test_on_rejected_message(self): 55 | d = dispatcher.TypeDispatcher() 56 | msg = mock_acked_message(properties={'type': 'hello'}) 57 | d.on_message("", msg) 58 | self.assertTrue(msg.reject_log_error.called) 59 | self.assertFalse(msg.acknowledged) 60 | 61 | def test_on_requeue_message(self): 62 | d = dispatcher.TypeDispatcher() 63 | d.requeue_filters.append(lambda data, message: True) 64 | msg = mock_acked_message() 65 | d.on_message("", msg) 66 | self.assertTrue(msg.requeue.called) 67 | self.assertFalse(msg.acknowledged) 68 | 69 | def test_failed_ack(self): 70 | on_hello = mock.MagicMock() 71 | handlers = {'hello': dispatcher.Handler(on_hello)} 72 | d = dispatcher.TypeDispatcher(type_handlers=handlers) 73 | msg = mock_acked_message(ack_ok=False, 74 | properties={'type': 'hello'}) 75 | d.on_message("", msg) 76 | self.assertTrue(msg.ack_log_error.called) 77 | self.assertFalse(msg.acknowledged) 78 | self.assertFalse(on_hello.called) 79 | -------------------------------------------------------------------------------- /taskflow/tests/unit/worker_based/test_endpoint.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2014 Yahoo! Inc. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); you may 4 | # not use this file except in compliance with the License. You may obtain 5 | # a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 11 | # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 12 | # License for the specific language governing permissions and limitations 13 | # under the License. 14 | 15 | from oslo_utils import reflection 16 | 17 | from taskflow.engines.worker_based import endpoint as ep 18 | from taskflow import task 19 | from taskflow import test 20 | from taskflow.tests import utils 21 | 22 | 23 | class Task(task.Task): 24 | 25 | def __init__(self, a, *args, **kwargs): 26 | super().__init__(*args, **kwargs) 27 | 28 | def execute(self, *args, **kwargs): 29 | pass 30 | 31 | 32 | class TestEndpoint(test.TestCase): 33 | 34 | def setUp(self): 35 | super().setUp() 36 | self.task_cls = utils.TaskOneReturn 37 | self.task_uuid = 'task-uuid' 38 | self.task_args = {'context': 'context'} 39 | self.task_cls_name = reflection.get_class_name(self.task_cls) 40 | self.task_ep = ep.Endpoint(self.task_cls) 41 | self.task_result = 1 42 | 43 | def test_creation(self): 44 | task = self.task_ep.generate() 45 | self.assertEqual(self.task_cls_name, self.task_ep.name) 46 | self.assertIsInstance(task, self.task_cls) 47 | self.assertEqual(self.task_cls_name, task.name) 48 | 49 | def test_creation_with_task_name(self): 50 | task_name = 'test' 51 | task = self.task_ep.generate(name=task_name) 52 | self.assertEqual(self.task_cls_name, self.task_ep.name) 53 | self.assertIsInstance(task, self.task_cls) 54 | self.assertEqual(task_name, task.name) 55 | 56 | def test_creation_task_with_constructor_args(self): 57 | # NOTE(skudriashev): Exception is expected here since task 58 | # is created without any arguments passing to its constructor. 59 | endpoint = ep.Endpoint(Task) 60 | self.assertRaises(TypeError, endpoint.generate) 61 | 62 | def test_to_str(self): 63 | self.assertEqual(self.task_cls_name, str(self.task_ep)) 64 | 65 | def test_execute(self): 66 | task = self.task_ep.generate(self.task_cls_name) 67 | result = self.task_ep.execute(task, 68 | task_uuid=self.task_uuid, 69 | arguments=self.task_args, 70 | progress_callback=None) 71 | self.assertEqual(self.task_result, result) 72 | 73 | def test_revert(self): 74 | task = self.task_ep.generate(self.task_cls_name) 75 | result = self.task_ep.revert(task, 76 | task_uuid=self.task_uuid, 77 | arguments=self.task_args, 78 | progress_callback=None, 79 | result=self.task_result, 80 | failures={}) 81 | self.assertIsNone(result) 82 | -------------------------------------------------------------------------------- /taskflow/tests/unit/worker_based/test_types.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2014 Yahoo! Inc. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); you may 4 | # not use this file except in compliance with the License. You may obtain 5 | # a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 11 | # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 12 | # License for the specific language governing permissions and limitations 13 | # under the License. 14 | 15 | from oslo_utils import reflection 16 | 17 | from taskflow.engines.worker_based import types as worker_types 18 | from taskflow import test 19 | from taskflow.test import mock 20 | from taskflow.tests import utils 21 | 22 | 23 | class TestTopicWorker(test.TestCase): 24 | def test_topic_worker(self): 25 | worker = worker_types.TopicWorker("dummy-topic", 26 | [utils.DummyTask], identity="dummy") 27 | self.assertTrue(worker.performs(utils.DummyTask)) 28 | self.assertFalse(worker.performs(utils.NastyTask)) 29 | self.assertEqual('dummy', worker.identity) 30 | self.assertEqual('dummy-topic', worker.topic) 31 | 32 | 33 | class TestProxyFinder(test.TestCase): 34 | 35 | @mock.patch("oslo_utils.timeutils.now") 36 | def test_expiry(self, mock_now): 37 | finder = worker_types.ProxyWorkerFinder('me', mock.MagicMock(), [], 38 | worker_expiry=60) 39 | w, emit = finder._add('dummy-topic', [utils.DummyTask]) 40 | w.last_seen = 0 41 | mock_now.side_effect = [120] 42 | gone = finder.clean() 43 | self.assertEqual(0, finder.total_workers) 44 | self.assertEqual(1, gone) 45 | 46 | def test_single_topic_worker(self): 47 | finder = worker_types.ProxyWorkerFinder('me', mock.MagicMock(), []) 48 | w, emit = finder._add('dummy-topic', [utils.DummyTask]) 49 | self.assertIsNotNone(w) 50 | self.assertTrue(emit) 51 | self.assertEqual(1, finder.total_workers) 52 | w2 = finder.get_worker_for_task(utils.DummyTask) 53 | self.assertEqual(w.identity, w2.identity) 54 | 55 | def test_multi_same_topic_workers(self): 56 | finder = worker_types.ProxyWorkerFinder('me', mock.MagicMock(), []) 57 | w, emit = finder._add('dummy-topic', [utils.DummyTask]) 58 | self.assertIsNotNone(w) 59 | self.assertTrue(emit) 60 | w2, emit = finder._add('dummy-topic-2', [utils.DummyTask]) 61 | self.assertIsNotNone(w2) 62 | self.assertTrue(emit) 63 | w3 = finder.get_worker_for_task( 64 | reflection.get_class_name(utils.DummyTask)) 65 | self.assertIn(w3.identity, [w.identity, w2.identity]) 66 | 67 | def test_multi_different_topic_workers(self): 68 | finder = worker_types.ProxyWorkerFinder('me', mock.MagicMock(), []) 69 | added = [] 70 | added.append(finder._add('dummy-topic', [utils.DummyTask])) 71 | added.append(finder._add('dummy-topic-2', [utils.DummyTask])) 72 | added.append(finder._add('dummy-topic-3', [utils.NastyTask])) 73 | self.assertEqual(3, finder.total_workers) 74 | w = finder.get_worker_for_task(utils.NastyTask) 75 | self.assertEqual(added[-1][0].identity, w.identity) 76 | w = finder.get_worker_for_task(utils.DummyTask) 77 | self.assertIn(w.identity, [w_a[0].identity for w_a in added[0:2]]) 78 | -------------------------------------------------------------------------------- /taskflow/types/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openstack/taskflow/d01920ef1c0e3432c5d3a1e7708a34ff834bf339/taskflow/types/__init__.py -------------------------------------------------------------------------------- /taskflow/types/entity.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2015 Rackspace Inc. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); you may 4 | # not use this file except in compliance with the License. You may obtain 5 | # a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 11 | # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 12 | # License for the specific language governing permissions and limitations 13 | # under the License. 14 | 15 | 16 | class Entity: 17 | """Entity object that identifies some resource/item/other. 18 | 19 | :ivar kind: **immutable** type/kind that identifies this 20 | entity (typically unique to a library/application) 21 | :type kind: string 22 | :ivar Entity.name: **immutable** name that can be used to uniquely 23 | identify this entity among many other entities 24 | :type name: string 25 | :ivar metadata: **immutable** dictionary of metadata that is 26 | associated with this entity (and typically 27 | has keys/values that further describe this 28 | entity) 29 | :type metadata: dict 30 | """ 31 | def __init__(self, kind, name, metadata): 32 | self.kind = kind 33 | self.name = name 34 | self.metadata = metadata 35 | 36 | def to_dict(self): 37 | return { 38 | 'kind': self.kind, 39 | 'name': self.name, 40 | 'metadata': self.metadata 41 | } 42 | -------------------------------------------------------------------------------- /taskflow/types/latch.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2014 Yahoo! Inc. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); you may 4 | # not use this file except in compliance with the License. You may obtain 5 | # a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 11 | # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 12 | # License for the specific language governing permissions and limitations 13 | # under the License. 14 | 15 | import threading 16 | 17 | from oslo_utils import timeutils 18 | 19 | 20 | class Latch: 21 | """A class that ensures N-arrivals occur before unblocking. 22 | 23 | TODO(harlowja): replace with http://bugs.python.org/issue8777 when we no 24 | longer have to support python 2.6 or 2.7 and we can only support 3.2 or 25 | later. 26 | """ 27 | 28 | def __init__(self, count): 29 | count = int(count) 30 | if count <= 0: 31 | raise ValueError("Count must be greater than zero") 32 | self._count = count 33 | self._cond = threading.Condition() 34 | 35 | @property 36 | def needed(self): 37 | """Returns how many decrements are needed before latch is released.""" 38 | return max(0, self._count) 39 | 40 | def countdown(self): 41 | """Decrements the internal counter due to an arrival.""" 42 | with self._cond: 43 | self._count -= 1 44 | if self._count <= 0: 45 | self._cond.notify_all() 46 | 47 | def wait(self, timeout=None): 48 | """Waits until the latch is released. 49 | 50 | :param timeout: wait until the timeout expires 51 | :type timeout: number 52 | :returns: true if the latch has been released before the 53 | timeout expires otherwise false 54 | :rtype: boolean 55 | """ 56 | watch = timeutils.StopWatch(duration=timeout) 57 | watch.start() 58 | with self._cond: 59 | while self._count > 0: 60 | if watch.expired(): 61 | return False 62 | else: 63 | self._cond.wait(watch.leftover(return_none=True)) 64 | return True 65 | -------------------------------------------------------------------------------- /taskflow/types/timing.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2014 Yahoo! Inc. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); you may 4 | # not use this file except in compliance with the License. You may obtain 5 | # a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 11 | # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 12 | # License for the specific language governing permissions and limitations 13 | # under the License. 14 | 15 | import threading 16 | 17 | 18 | class Timeout: 19 | """An object which represents a timeout. 20 | 21 | This object has the ability to be interrupted before the actual timeout 22 | is reached. 23 | """ 24 | def __init__(self, value, event_factory=threading.Event): 25 | if value < 0: 26 | raise ValueError("Timeout value must be greater or" 27 | " equal to zero and not '%s'" % (value)) 28 | self._value = value 29 | self._event = event_factory() 30 | 31 | @property 32 | def value(self): 33 | """Immutable value of the internally used timeout.""" 34 | return self._value 35 | 36 | def interrupt(self): 37 | """Forcefully set the timeout (releases any waiters).""" 38 | self._event.set() 39 | 40 | def is_stopped(self): 41 | """Returns if the timeout has been interrupted.""" 42 | return self._event.is_set() 43 | 44 | def wait(self): 45 | """Block current thread (up to timeout) and wait until interrupted.""" 46 | self._event.wait(self._value) 47 | 48 | def reset(self): 49 | """Reset so that interruption (and waiting) can happen again.""" 50 | self._event.clear() 51 | 52 | 53 | def convert_to_timeout(value=None, default_value=None, 54 | event_factory=threading.Event): 55 | """Converts a given value to a timeout instance (and returns it). 56 | 57 | Does nothing if the value provided is already a timeout instance. 58 | """ 59 | if value is None: 60 | value = default_value 61 | if isinstance(value, (int, float, str)): 62 | return Timeout(float(value), event_factory=event_factory) 63 | elif isinstance(value, Timeout): 64 | return value 65 | else: 66 | raise ValueError("Invalid timeout literal '%s'" % (value)) 67 | -------------------------------------------------------------------------------- /taskflow/utils/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openstack/taskflow/d01920ef1c0e3432c5d3a1e7708a34ff834bf339/taskflow/utils/__init__.py -------------------------------------------------------------------------------- /taskflow/utils/async_utils.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2013 Yahoo! Inc. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); you may 4 | # not use this file except in compliance with the License. You may obtain 5 | # a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 11 | # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 12 | # License for the specific language governing permissions and limitations 13 | # under the License. 14 | 15 | import futurist 16 | 17 | 18 | def make_completed_future(result): 19 | """Make and return a future completed with a given result.""" 20 | future = futurist.Future() 21 | future.set_result(result) 22 | return future 23 | -------------------------------------------------------------------------------- /taskflow/utils/eventlet_utils.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2015 Yahoo! Inc. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); you may 4 | # not use this file except in compliance with the License. You may obtain 5 | # a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 11 | # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 12 | # License for the specific language governing permissions and limitations 13 | # under the License. 14 | 15 | import debtcollector.removals 16 | from oslo_utils import importutils 17 | 18 | _eventlet = importutils.try_import('eventlet') 19 | 20 | EVENTLET_AVAILABLE = bool(_eventlet) 21 | 22 | 23 | @debtcollector.removals.remove(message='Eventlet support is deprecated.') 24 | def check_for_eventlet(exc=None): 25 | """Check if eventlet is available and if not raise a runtime error. 26 | 27 | :param exc: exception to raise instead of raising a runtime error 28 | :type exc: exception 29 | """ 30 | if not EVENTLET_AVAILABLE: 31 | if exc is None: 32 | raise RuntimeError('Eventlet is not currently available') 33 | else: 34 | raise exc 35 | -------------------------------------------------------------------------------- /taskflow/utils/kombu_utils.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2015 Yahoo! Inc. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); you may 4 | # not use this file except in compliance with the License. You may obtain 5 | # a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 11 | # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 12 | # License for the specific language governing permissions and limitations 13 | # under the License. 14 | 15 | # Keys extracted from the message properties when formatting... 16 | _MSG_PROPERTIES = tuple([ 17 | 'correlation_id', 18 | 'delivery_info/routing_key', 19 | 'type', 20 | ]) 21 | 22 | 23 | class DelayedPretty: 24 | """Wraps a message and delays prettifying it until requested. 25 | 26 | TODO(harlowja): remove this when https://github.com/celery/kombu/pull/454/ 27 | is merged and a release is made that contains it (since that pull 28 | request is equivalent and/or better than this). 29 | """ 30 | 31 | def __init__(self, message): 32 | self._message = message 33 | self._message_pretty = None 34 | 35 | def __str__(self): 36 | if self._message_pretty is None: 37 | self._message_pretty = _prettify_message(self._message) 38 | return self._message_pretty 39 | 40 | 41 | def _get_deep(properties, *keys): 42 | """Get a final key among a list of keys (each with its own sub-dict).""" 43 | for key in keys: 44 | properties = properties[key] 45 | return properties 46 | 47 | 48 | def _prettify_message(message): 49 | """Kombu doesn't currently have a useful ``__str__()`` or ``__repr__()``. 50 | 51 | This provides something decent(ish) for debugging (or other purposes) so 52 | that messages are more nice and understandable.... 53 | """ 54 | if message.content_type is not None: 55 | properties = { 56 | 'content_type': message.content_type, 57 | } 58 | else: 59 | properties = {} 60 | for name in _MSG_PROPERTIES: 61 | segments = name.split("/") 62 | try: 63 | value = _get_deep(message.properties, *segments) 64 | except (KeyError, ValueError, TypeError): 65 | pass 66 | else: 67 | if value is not None: 68 | properties[segments[-1]] = value 69 | if message.body is not None: 70 | properties['body_length'] = len(message.body) 71 | return "{delivery_tag}: {properties}".format( 72 | delivery_tag=message.delivery_tag, 73 | properties=properties, 74 | ) 75 | -------------------------------------------------------------------------------- /taskflow/utils/schema_utils.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2015 Yahoo! Inc. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); you may 4 | # not use this file except in compliance with the License. You may obtain 5 | # a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 11 | # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 12 | # License for the specific language governing permissions and limitations 13 | # under the License. 14 | 15 | import jsonschema 16 | from jsonschema import exceptions as schema_exc 17 | 18 | 19 | # Expose these types so that people don't have to import the same exceptions. 20 | ValidationError = schema_exc.ValidationError 21 | SchemaError = schema_exc.SchemaError 22 | 23 | 24 | def schema_validate(data, schema): 25 | """Validates given data using provided json schema.""" 26 | Validator = jsonschema.validators.validator_for(schema) 27 | # Special jsonschema validation types/adjustments. 28 | # See: https://github.com/Julian/jsonschema/issues/148 29 | type_checker = Validator.TYPE_CHECKER.redefine( 30 | "array", lambda checker, data: isinstance(data, (list, tuple))) 31 | TupleAllowingValidator = jsonschema.validators.extend( 32 | Validator, type_checker=type_checker) 33 | TupleAllowingValidator(schema).validate(data) 34 | -------------------------------------------------------------------------------- /taskflow/version.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2013 Yahoo! Inc. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); you may 4 | # not use this file except in compliance with the License. You may obtain 5 | # a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 11 | # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 12 | # License for the specific language governing permissions and limitations 13 | # under the License. 14 | 15 | from pbr import version as pbr_version 16 | 17 | TASK_VENDOR = "OpenStack Foundation" 18 | TASK_PRODUCT = "OpenStack TaskFlow" 19 | TASK_PACKAGE = None # OS distro package version suffix 20 | 21 | _version_info = pbr_version.VersionInfo('taskflow') 22 | version_string = _version_info.version_string 23 | 24 | 25 | def version_string_with_package(): 26 | if TASK_PACKAGE is None: 27 | return version_string() 28 | else: 29 | return "{}-{}".format(version_string(), TASK_PACKAGE) 30 | -------------------------------------------------------------------------------- /test-requirements.txt: -------------------------------------------------------------------------------- 1 | # NOTE(dhellmann): This file contains duplicate dependency information 2 | # that is also present in the "extras" section of setup.cfg, and the 3 | # entries need to be kept consistent. 4 | 5 | # zookeeper 6 | kazoo>=2.6.0 # Apache-2.0 7 | 8 | # redis 9 | redis>=4.0.0 # MIT 10 | 11 | # etcd3gw 12 | etcd3gw>=2.0.0 # Apache-2.0 13 | 14 | # workers 15 | kombu>=4.3.0 # BSD 16 | 17 | # eventlet 18 | eventlet>=0.18.2 # MIT 19 | 20 | # database 21 | SQLAlchemy>=1.0.10 # MIT 22 | alembic>=0.8.10 # MIT 23 | SQLAlchemy-Utils>=0.30.11 # BSD License 24 | PyMySQL>=0.7.6 # MIT License 25 | psycopg2>=2.8.0 # LGPL/ZPL 26 | 27 | # test 28 | zake>=0.1.6 # Apache-2.0 29 | pydotplus>=2.0.2 # MIT License 30 | oslotest>=3.2.0 # Apache-2.0 31 | testtools>=2.2.0 # MIT 32 | testscenarios>=0.4 # Apache-2.0/BSD 33 | stestr>=2.0.0 # Apache-2.0 34 | pifpaf>=0.10.0 # Apache-2.0 35 | -------------------------------------------------------------------------------- /tools/clear_zk.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # This requires https://pypi.org/project/zk_shell/ to be installed... 4 | 5 | set -e 6 | 7 | ZK_HOSTS=${ZK_HOSTS:-localhost:2181} 8 | TF_PATH=${TF_PATH:-taskflow} 9 | 10 | for path in `zk-shell --run-once "ls" $ZK_HOSTS`; do 11 | if [[ $path == ${TF_PATH}* ]]; then 12 | echo "Removing (recursively) path \"$path\"" 13 | zk-shell --run-once "rmr $path" $ZK_HOSTS 14 | fi 15 | done 16 | -------------------------------------------------------------------------------- /tools/pretty_tox.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | set -o pipefail 4 | 5 | TESTRARGS=$1 6 | 7 | # --until-failure is not compatible with --subunit see: 8 | # 9 | # https://bugs.launchpad.net/testrepository/+bug/1411804 10 | # 11 | # this work around exists until that is addressed 12 | if [[ "$TESTARGS" =~ "until-failure" ]]; then 13 | python setup.py testr --slowest --testr-args="$TESTRARGS" 14 | else 15 | python setup.py testr --slowest --testr-args="--subunit $TESTRARGS" | $(dirname $0)/subunit_trace.py -f 16 | fi 17 | 18 | -------------------------------------------------------------------------------- /tools/schema_generator.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | # Copyright (C) 2014 Yahoo! Inc. All Rights Reserved. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); you may 6 | # not use this file except in compliance with the License. You may obtain 7 | # a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 13 | # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 14 | # License for the specific language governing permissions and limitations 15 | # under the License. 16 | 17 | import contextlib 18 | import re 19 | 20 | import sqlalchemy as sa 21 | import tabulate 22 | 23 | from taskflow.persistence.backends import impl_sqlalchemy 24 | 25 | NAME_MAPPING = { 26 | 'flowdetails': 'Flow details', 27 | 'atomdetails': 'Atom details', 28 | 'logbooks': 'Logbooks', 29 | } 30 | CONN_CONF = { 31 | # This uses an in-memory database (aka nothing is written) 32 | "connection": "sqlite://", 33 | } 34 | TABLE_QUERY = "SELECT name, sql FROM sqlite_master WHERE type='table'" 35 | SCHEMA_QUERY = "pragma table_info(%s)" 36 | 37 | 38 | def to_bool_string(val): 39 | if isinstance(val, (int, bool)): 40 | return str(bool(val)) 41 | if not isinstance(val, str): 42 | val = str(val) 43 | if val.lower() in ('0', 'false'): 44 | return 'False' 45 | if val.lower() in ('1', 'true'): 46 | return 'True' 47 | raise ValueError("Unknown boolean input '%s'" % (val)) 48 | 49 | 50 | def main(): 51 | backend = impl_sqlalchemy.SQLAlchemyBackend(CONN_CONF) 52 | with contextlib.closing(backend) as backend: 53 | # Make the schema exist... 54 | with contextlib.closing(backend.get_connection()) as conn: 55 | conn.upgrade() 56 | # Now make a prettier version of that schema... 57 | with backend.engine.connect() as conn, conn.begin(): 58 | tables = conn.execute(sa.text(TABLE_QUERY)) 59 | table_names = [r[0] for r in tables] 60 | for i, table_name in enumerate(table_names): 61 | pretty_name = NAME_MAPPING.get(table_name, table_name) 62 | print("*" + pretty_name + "*") 63 | # http://www.sqlite.org/faq.html#q24 64 | table_name = table_name.replace("\"", "\"\"") 65 | rows = [] 66 | for r in conn.execute(sa.text(SCHEMA_QUERY % table_name)): 67 | # Cut out the numbers from things like VARCHAR(12) since 68 | # this is not very useful to show users who just want to 69 | # see the basic schema... 70 | row_type = re.sub(r"\(.*?\)", "", r['type']).strip() 71 | if not row_type: 72 | raise ValueError("Row %s of table '%s' was empty after" 73 | " cleaning" % (r['cid'], table_name)) 74 | rows.append([r['name'], row_type, to_bool_string(r['pk'])]) 75 | contents = tabulate.tabulate( 76 | rows, headers=['Name', 'Type', 'Primary Key'], 77 | tablefmt="rst") 78 | print("\n%s" % contents.strip()) 79 | if i + 1 != len(table_names): 80 | print("") 81 | 82 | 83 | if __name__ == '__main__': 84 | main() 85 | -------------------------------------------------------------------------------- /tools/test-setup.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash -xe 2 | 3 | # This script will be run by OpenStack CI before unit tests are run, 4 | # it sets up the test system as needed. 5 | # Developers should setup their test systems in a similar way. 6 | 7 | # This setup needs to be run as a user that can run sudo. 8 | 9 | # The root password for the MySQL database; pass it in via 10 | # MYSQL_ROOT_PW. 11 | DB_ROOT_PW=${MYSQL_ROOT_PW:-insecure_slave} 12 | 13 | # This user and its password are used by the tests, if you change it, 14 | # your tests might fail. 15 | DB_USER=openstack_citest 16 | DB_PW=openstack_citest 17 | 18 | sudo -H mysqladmin -u root password $DB_ROOT_PW 19 | 20 | # It's best practice to remove anonymous users from the database. If 21 | # a anonymous user exists, then it matches first for connections and 22 | # other connections from that host will not work. 23 | sudo -H mysql -u root -p$DB_ROOT_PW -h localhost -e " 24 | DELETE FROM mysql.user WHERE User=''; 25 | FLUSH PRIVILEGES; 26 | CREATE USER '$DB_USER'@'%' IDENTIFIED BY '$DB_PW'; 27 | GRANT ALL PRIVILEGES ON *.* TO '$DB_USER'@'%' WITH GRANT OPTION;" 28 | 29 | # Now create our database. 30 | mysql -u $DB_USER -p$DB_PW -h 127.0.0.1 -e " 31 | SET default_storage_engine=MYISAM; 32 | DROP DATABASE IF EXISTS openstack_citest; 33 | CREATE DATABASE openstack_citest CHARACTER SET utf8;" 34 | 35 | # Same for PostgreSQL 36 | 37 | # Setup user 38 | root_roles=$(sudo -H -u postgres psql -t -c " 39 | SELECT 'HERE' from pg_roles where rolname='$DB_USER'") 40 | if [[ ${root_roles} == *HERE ]];then 41 | sudo -H -u postgres psql -c "ALTER ROLE $DB_USER WITH SUPERUSER LOGIN PASSWORD '$DB_PW'" 42 | else 43 | sudo -H -u postgres psql -c "CREATE ROLE $DB_USER WITH SUPERUSER LOGIN PASSWORD '$DB_PW'" 44 | fi 45 | 46 | # Store password for tests 47 | cat << EOF > $HOME/.pgpass 48 | *:*:*:$DB_USER:$DB_PW 49 | EOF 50 | chmod 0600 $HOME/.pgpass 51 | 52 | # Now create our database 53 | psql -h 127.0.0.1 -U $DB_USER -d template1 -c "DROP DATABASE IF EXISTS openstack_citest" 54 | createdb -h 127.0.0.1 -U $DB_USER -l C -T template0 -E utf8 openstack_citest 55 | -------------------------------------------------------------------------------- /tools/update_states.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -u 4 | xsltproc=`which xsltproc` 5 | if [ -z "$xsltproc" ]; then 6 | echo "Please install xsltproc before continuing." 7 | exit 1 8 | fi 9 | 10 | set -e 11 | if [ ! -d "$PWD/.diagram-tools" ]; then 12 | git clone "https://github.com/vidarh/diagram-tools.git" "$PWD/.diagram-tools" 13 | fi 14 | 15 | script_dir=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd ) 16 | img_dir="$script_dir/../doc/source/user/img" 17 | 18 | echo "---- Updating task state diagram ----" 19 | python $script_dir/state_graph.py -t -f /tmp/states.svg 20 | $xsltproc $PWD/.diagram-tools/notugly.xsl /tmp/states.svg > $img_dir/task_states.svg 21 | 22 | echo "---- Updating flow state diagram ----" 23 | python $script_dir/state_graph.py --flow -f /tmp/states.svg 24 | $xsltproc $PWD/.diagram-tools/notugly.xsl /tmp/states.svg > $img_dir/flow_states.svg 25 | 26 | echo "---- Updating engine state diagram ----" 27 | python $script_dir/state_graph.py -e -f /tmp/states.svg 28 | $xsltproc $PWD/.diagram-tools/notugly.xsl /tmp/states.svg > $img_dir/engine_states.svg 29 | 30 | echo "---- Updating retry state diagram ----" 31 | python $script_dir/state_graph.py -r -f /tmp/states.svg 32 | $xsltproc $PWD/.diagram-tools/notugly.xsl /tmp/states.svg > $img_dir/retry_states.svg 33 | 34 | # NOTE(tkajinam): This is broken since 148963805626f6246554961bd3ff39055de3e317 35 | # echo "---- Updating wbe request state diagram ----" 36 | # python $script_dir/state_graph.py -w -f /tmp/states.svg 37 | # $xsltproc $PWD/.diagram-tools/notugly.xsl /tmp/states.svg > $img_dir/wbe_request_states.svg 38 | 39 | echo "---- Updating job state diagram ----" 40 | python $script_dir/state_graph.py -j -f /tmp/states.svg 41 | $xsltproc $PWD/.diagram-tools/notugly.xsl /tmp/states.svg > $img_dir/job_states.svg 42 | -------------------------------------------------------------------------------- /tox.ini: -------------------------------------------------------------------------------- 1 | [tox] 2 | minversion = 3.18.0 3 | envlist = cover,docs,pep8,py3,pylint,update-states 4 | 5 | [testenv] 6 | # We need to install a bit more than just `test' because those drivers have 7 | # custom tests that we always run 8 | deps = 9 | -c{env:TOX_CONSTRAINTS_FILE:https://releases.openstack.org/constraints/upper/master} 10 | -r{toxinidir}/test-requirements.txt 11 | -r{toxinidir}/requirements.txt 12 | commands = 13 | stestr run {posargs} 14 | 15 | [testenv:docs] 16 | deps = 17 | {[testenv]deps} 18 | -r{toxinidir}/doc/requirements.txt 19 | commands = 20 | sphinx-build -E -W -b html doc/source doc/build/html 21 | doc8 doc/source 22 | 23 | [testenv:functional] 24 | commands = 25 | find . -type f -name "*.pyc" -delete 26 | {env:SETUP_ENV_SCRIPT} pifpaf -e TAKSFLOW_TEST run {env:PIFPAF_DAEMON} {env:PIFPAF_OPTS} -- stestr run 27 | allowlist_externals = 28 | find 29 | ./setup-etcd-env.sh 30 | 31 | [testenv:update-states] 32 | deps = 33 | {[testenv]deps} 34 | commands = {toxinidir}/tools/update_states.sh 35 | allowlist_externals = 36 | {toxinidir}/tools/update_states.sh 37 | 38 | [testenv:pep8] 39 | skip_install = true 40 | deps = 41 | pre-commit 42 | commands = 43 | pre-commit run -a 44 | 45 | [testenv:pylint] 46 | deps = 47 | {[testenv]deps} 48 | pylint==3.2.0 # GPLv2 49 | commands = pylint taskflow 50 | 51 | [testenv:cover] 52 | deps = 53 | {[testenv]deps} 54 | coverage>=3.6 55 | setenv = 56 | PYTHON=coverage run --source taskflow --parallel-mode 57 | commands = 58 | stestr run {posargs} 59 | coverage combine 60 | coverage html -d cover 61 | coverage xml -o cover/coverage.xml 62 | 63 | [testenv:venv] 64 | commands = {posargs} 65 | 66 | [flake8] 67 | builtins = _ 68 | exclude = .venv,.tox,dist,doc,*egg,.git,build,tools 69 | ignore = E305,E402,E721,E731,E741,W503,W504 70 | 71 | [hacking] 72 | import_exceptions = 73 | taskflow.test.mock 74 | unittest.mock 75 | 76 | [doc8] 77 | # Settings for doc8: 78 | # Ignore doc/source/user/history.rst, it includes generated ChangeLog 79 | # file that fails with "D000 Inline emphasis start-string without 80 | # end-string." 81 | ignore-path = doc/*/target,doc/*/build* 82 | 83 | [testenv:releasenotes] 84 | deps = -r{toxinidir}/doc/requirements.txt 85 | commands = sphinx-build -a -E -W -d releasenotes/build/doctrees -b html releasenotes/source releasenotes/build/html 86 | 87 | [testenv:bindep] 88 | # Do not install any requirements. We want this to be fast and work even if 89 | # system dependencies are missing, since it's used to tell you what system 90 | # dependencies are missing! This also means that bindep must be installed 91 | # separately, outside of the requirements files, and develop mode disabled 92 | # explicitly to avoid unnecessarily installing the checked-out repo too (this 93 | # further relies on "tox.skipsdist = True" above). 94 | deps = bindep 95 | commands = bindep test 96 | usedevelop = False 97 | 98 | --------------------------------------------------------------------------------