├── tests ├── __init__.py ├── unit │ ├── __init__.py │ ├── test_utils_type_utils.py │ ├── test_utils_task_utils.py │ ├── test_client.py │ ├── test_utils_yaml_utils.py │ ├── test_expressions_base.py │ ├── test_pack_client.py │ ├── test_expressions_mixed.py │ ├── test_expressions_jinja.py │ ├── test_expressions_yaql.py │ └── test_expressions.py ├── integration │ ├── __init__.py │ ├── test_end_to_end.py │ └── test_convert_pack.py ├── fixtures │ ├── yaml │ │ ├── simple.yaml │ │ └── no_line_wrap.yaml │ ├── mistral │ │ ├── transition_strings.yaml │ │ ├── dashes_in_task_names.yaml │ │ ├── null_publish_parameters.yaml │ │ ├── unsupported_attributes.yaml │ │ ├── boolean_publish_parameters.yaml │ │ ├── int_publish_parameters.yaml │ │ ├── output_test.yaml │ │ ├── nasa_apod_twitter_post_yaql.yaml │ │ ├── nasa_apod_twitter_post.yaml │ │ ├── complex_publish_conditions.yaml │ │ ├── emptywee_test.yaml │ │ └── immediately_referenced_context_variables.yaml │ ├── orquesta │ │ ├── transition_strings.yaml │ │ ├── dashes_in_task_names.yaml │ │ ├── null_publish_parameters.yaml │ │ ├── nasa_apod_twitter_post.yaml │ │ ├── nasa_apod_twitter_post_yaql.yaml │ │ ├── output_test.yaml │ │ ├── boolean_publish_parameters.yaml │ │ ├── int_publish_parameters.yaml │ │ ├── retry-with-break-on.yaml │ │ ├── retry-with-continue-on.yaml │ │ ├── complex_publish_conditions.yaml │ │ ├── emptywee_test.yaml │ │ └── immediately_referenced_context_variables.yaml │ ├── pack │ │ ├── o_actions │ │ │ ├── mistral-with-items-yaql-list.yaml │ │ │ ├── mistral-with-items-static-list.yaml │ │ │ ├── mistral-retry.yaml │ │ │ ├── mistral-with-items-yaml-list.yaml │ │ │ ├── mistral-with-items-yaql.yaml │ │ │ ├── mistral-with-items-jinja.yaml │ │ │ ├── workflows │ │ │ │ ├── mistral-with-items-yaml-list.yaml │ │ │ │ ├── mistral-with-items-yaql-list.yaml │ │ │ │ ├── mistral-with-items-static-list.yaml │ │ │ │ ├── mistral-with-items-mixed-list.yaml │ │ │ │ ├── mistral-with-items-yaql.yaml │ │ │ │ ├── mistral-with-items-jinja.yaml │ │ │ │ ├── mistral-test-cancel.yaml │ │ │ │ ├── mistral-retry.yaml │ │ │ │ ├── mistral-retry-break-on.yaml │ │ │ │ ├── mistral-retry-continue-on.yaml │ │ │ │ ├── mistral-with-items-concurrency.yaml │ │ │ │ ├── mistral-with-items-concurrency-str.yaml │ │ │ │ ├── mistral-with-items-concurrency-jinja.yaml │ │ │ │ ├── mistral-with-items-concurrency-yaql.yaml │ │ │ │ ├── mistral-retry-continue-and-break-on.yaml │ │ │ │ ├── mistral-transition-expressions.yaml │ │ │ │ └── mistral-publish-only-transitions.yaml │ │ │ ├── mistral-with-items-mixed-list.yaml │ │ │ ├── mistral-test-cancel.yaml │ │ │ ├── mistral-retry-break-on.yaml │ │ │ ├── mistral-with-items-concurrency.yaml │ │ │ ├── mistral-retry-continue-on.yaml │ │ │ ├── mistral-publish-only-transitions.yaml │ │ │ ├── mistral-with-items-concurrency-str.yaml │ │ │ ├── mistral-with-items-concurrency-yaql.yaml │ │ │ ├── mistral-with-items-concurrency-jinja.yaml │ │ │ ├── mistral-retry-continue-and-break-on.yaml │ │ │ └── mistral-transition-expressions.yaml │ │ └── pristine_actions │ │ │ ├── mistral-with-items-static-list.yaml │ │ │ ├── mistral-with-items-yaql-list.yaml │ │ │ ├── mistral-with-items-yaml-list.yaml │ │ │ ├── mistral-retry.yaml │ │ │ ├── mistral-with-items-yaql.yaml │ │ │ ├── mistral-with-items-jinja.yaml │ │ │ ├── mistral-with-items-mixed-list.yaml │ │ │ ├── mistral-test-cancel.yaml │ │ │ ├── mistral-with-items-concurrency.yaml │ │ │ ├── mistral-publish-only-transitions.yaml │ │ │ ├── mistral-retry-break-on.yaml │ │ │ ├── mistral-with-items-concurrency-str.yaml │ │ │ ├── mistral-with-items-concurrency-yaql.yaml │ │ │ ├── mistral-with-items-concurrency-jinja.yaml │ │ │ ├── mistral-retry-continue-on.yaml │ │ │ ├── workflows │ │ │ ├── mistral-with-items-yaql-list.yaml │ │ │ ├── mistral-with-items-static-list.yaml │ │ │ ├── mistral-with-items-yaml-list.yaml │ │ │ ├── mistral-with-items-mixed-list.yaml │ │ │ ├── mistral-with-items-yaql.yaml │ │ │ ├── mistral-with-items-jinja.yaml │ │ │ ├── mistral-test-cancel.yaml │ │ │ ├── mistral-retry.yaml │ │ │ ├── mistral-retry-break-on.yaml │ │ │ ├── mistral-retry-continue-on.yaml │ │ │ ├── mistral-with-items-concurrency.yaml │ │ │ ├── mistral-with-items-concurrency-str.yaml │ │ │ ├── mistral-with-items-concurrency-yaql.yaml │ │ │ ├── mistral-with-items-concurrency-jinja.yaml │ │ │ ├── mistral-retry-continue-and-break-on.yaml │ │ │ ├── mistral-fail-retry-continue-and-break-on.yaml │ │ │ ├── mistral-transition-expressions.yaml │ │ │ └── mistral-publish-only-transitions.yaml │ │ │ ├── mistral-transition-expressions.yaml │ │ │ ├── mistral-retry-continue-and-break-on.yaml │ │ │ └── mistral-fail-retry-continue-and-break-on.yaml │ └── broken │ │ └── unsupported_attributes.yaml └── base_test_case.py ├── orquestaconvert ├── specs │ ├── __init__.py │ └── mistral │ │ ├── v2 │ │ ├── types.py │ │ ├── __init__.py │ │ ├── base.py │ │ ├── policies.py │ │ ├── workflows.py │ │ └── tasks.py │ │ └── __init__.py ├── utils │ ├── __init__.py │ ├── task_utils.py │ ├── type_utils.py │ └── yaml_utils.py ├── workflows │ └── __init__.py ├── __init__.py ├── expressions │ ├── yaql.py │ ├── jinja.py │ ├── __init__.py │ ├── mixed.py │ └── base.py ├── client.py └── pack_client.py ├── lint-configs └── python │ ├── .flake8 │ └── .pylintrc ├── requirements.txt ├── requirements-orquesta.txt ├── requirements-test.txt ├── .circleci └── config.yml ├── bin ├── orquestaconvert.sh └── orquestaconvert-pack.sh ├── doc └── expressions.md ├── .gitignore ├── setup.py ├── dist_utils.py ├── README.md └── Makefile /tests/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tests/unit/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /orquestaconvert/specs/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /orquestaconvert/utils/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tests/integration/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /orquestaconvert/workflows/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /orquestaconvert/__init__.py: -------------------------------------------------------------------------------- 1 | __version__ = '0.1' 2 | -------------------------------------------------------------------------------- /tests/fixtures/yaml/simple.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | test_dict: 3 | a: true 4 | -------------------------------------------------------------------------------- /lint-configs/python/.flake8: -------------------------------------------------------------------------------- 1 | [flake8] 2 | max-line-length = 120 3 | ignore = E128,E402 4 | exclude=*.egg/* -------------------------------------------------------------------------------- /orquestaconvert/utils/task_utils.py: -------------------------------------------------------------------------------- 1 | 2 | def translate_task_name(task_name): 3 | return task_name.replace('-', '_') 4 | -------------------------------------------------------------------------------- /orquestaconvert/utils/type_utils.py: -------------------------------------------------------------------------------- 1 | import ruamel.yaml 2 | 3 | 4 | dict_types = (dict, ruamel.yaml.comments.CommentedMap) 5 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | ruamel.yaml>=0.15.0 2 | yamlloader>=0.5.0 3 | -e git+https://github.com/StackStorm/orquesta.git#egg=orquesta 4 | -------------------------------------------------------------------------------- /tests/fixtures/yaml/no_line_wrap.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | key: this is some super super super long line that would normally cause a line wrap when converting from dict to yaml. instead we don't want to do that and just keep this one crazy long line. 3 | -------------------------------------------------------------------------------- /tests/fixtures/mistral/transition_strings.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | version: '2.0' 3 | 4 | circleci.test_transition_string: 5 | tasks: 6 | task_one: 7 | action: core.noop 8 | on-success: task_two 9 | 10 | task_two: 11 | action: core.noop 12 | -------------------------------------------------------------------------------- /requirements-orquesta.txt: -------------------------------------------------------------------------------- 1 | chardet 2 | Jinja2>=2.8 # BSD License (3 clause) 3 | jsonschema!=2.5.0,<3.0.0,>=2.0.0 # MIT 4 | networkx>=1.10,<2.0 5 | python-dateutil 6 | PyYAML>=3.1.0 # MIT 7 | six>=1.9.0 8 | stevedore>=1.3.0 # Apache-2.0 9 | yaql>=1.1.0 # Apache-2.0 10 | -------------------------------------------------------------------------------- /tests/fixtures/orquesta/transition_strings.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | version: '1.0' 3 | tasks: 4 | task_one: 5 | action: core.noop 6 | next: 7 | - when: '{{ succeeded() }}' 8 | do: 9 | - task_two 10 | task_two: 11 | action: core.noop 12 | -------------------------------------------------------------------------------- /tests/fixtures/pack/o_actions/mistral-with-items-yaql-list.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | name: mistral-with-items-yaql-list 3 | description: Mistral's with-items with a YAQL list 4 | pack: examples 5 | runner_type: orquesta 6 | entry_point: workflows/mistral-with-items-yaql-list.yaml 7 | enabled: true 8 | parameters: 9 | tempfile: 10 | type: string 11 | required: true 12 | -------------------------------------------------------------------------------- /tests/fixtures/pack/o_actions/mistral-with-items-static-list.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | name: mistral-with-items-static-list 3 | description: Mistral's with-items with a static list 4 | pack: examples 5 | runner_type: orquesta 6 | entry_point: workflows/mistral-with-items-static-list.yaml 7 | enabled: true 8 | parameters: 9 | tempfile: 10 | type: string 11 | required: true 12 | -------------------------------------------------------------------------------- /tests/fixtures/mistral/dashes_in_task_names.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | version: '2.0' 3 | 4 | circleci.test_dashes_in_task_names: 5 | tasks: 6 | odd-job: 7 | action: core.noop 8 | on-success: 9 | - random-task 10 | - weird-todo: "{{ _.jinja_expr }}" 11 | 12 | random-task: 13 | action: core.noop 14 | 15 | weird-todo: 16 | action: core.noop 17 | -------------------------------------------------------------------------------- /tests/fixtures/pack/pristine_actions/mistral-with-items-static-list.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | name: mistral-with-items-static-list 3 | description: Mistral's with-items with a static list 4 | pack: examples 5 | runner_type: mistral-v2 6 | entry_point: workflows/mistral-with-items-static-list.yaml 7 | enabled: true 8 | parameters: 9 | tempfile: 10 | type: string 11 | required: true 12 | -------------------------------------------------------------------------------- /tests/fixtures/pack/pristine_actions/mistral-with-items-yaql-list.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | name: mistral-with-items-yaql-list 3 | description: Mistral's with-items with a YAQL list 4 | pack: examples 5 | runner_type: mistral-v2 6 | entry_point: workflows/mistral-with-items-yaql-list.yaml 7 | enabled: true 8 | parameters: 9 | tempfile: 10 | type: string 11 | required: true 12 | -------------------------------------------------------------------------------- /tests/fixtures/pack/o_actions/mistral-retry.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | name: mistral-retry 3 | description: A sample workflow used to test the retry feature. 4 | pack: examples 5 | runner_type: orquesta 6 | entry_point: workflows/mistral-retry.yaml 7 | enabled: true 8 | parameters: 9 | tempfile: 10 | type: string 11 | required: true 12 | message: 13 | type: string 14 | required: true 15 | -------------------------------------------------------------------------------- /tests/fixtures/pack/o_actions/mistral-with-items-yaml-list.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | description: Mistral's with-items with a YAML list 3 | enabled: true 4 | entry_point: workflows/mistral-with-items-yaml-list.yaml 5 | name: mistral-with-items-list 6 | pack: examples 7 | parameters: 8 | cmds: 9 | items: 10 | type: string 11 | minItems: 1 12 | type: array 13 | runner_type: orquesta 14 | -------------------------------------------------------------------------------- /tests/fixtures/pack/o_actions/mistral-with-items-yaql.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | description: Run several linux commands in a single task. 3 | enabled: true 4 | entry_point: workflows/mistral-with-items-yaql.yaml 5 | name: mistral-with-items-yaql 6 | pack: examples 7 | parameters: 8 | yaql_cmds: 9 | items: 10 | type: string 11 | minItems: 1 12 | type: array 13 | runner_type: orquesta 14 | -------------------------------------------------------------------------------- /tests/fixtures/pack/o_actions/mistral-with-items-jinja.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | description: Run several linux commands in a single task. 3 | enabled: true 4 | entry_point: workflows/mistral-with-items-jinja.yaml 5 | name: mistral-with-items-jinja 6 | pack: examples 7 | parameters: 8 | jinja_cmds: 9 | items: 10 | type: string 11 | minItems: 1 12 | type: array 13 | runner_type: orquesta 14 | -------------------------------------------------------------------------------- /tests/fixtures/pack/o_actions/workflows/mistral-with-items-yaml-list.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | version: '1.0' 3 | description: Mistral's with-items with a YAML list 4 | input: 5 | - tempfile 6 | tasks: 7 | task1: 8 | with: 9 | items: i, j, k in <% zip([0, 1, 2, 3], [4, 5, 6, 7], [8, 9]) %> 10 | action: core.local 11 | input: 12 | cmd: echo "<% item(i) %><% item(j) %><% item(k) %>" 13 | -------------------------------------------------------------------------------- /tests/fixtures/pack/pristine_actions/mistral-with-items-yaml-list.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | description: Mistral's with-items with a YAML list 3 | enabled: true 4 | entry_point: workflows/mistral-with-items-yaml-list.yaml 5 | name: mistral-with-items-list 6 | pack: examples 7 | parameters: 8 | cmds: 9 | items: 10 | type: string 11 | minItems: 1 12 | type: array 13 | runner_type: mistral-v2 14 | -------------------------------------------------------------------------------- /tests/fixtures/broken/unsupported_attributes.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | version: '1.0' 3 | output-on-error: 4 | error: Error condition 5 | input: 6 | - items: 7 | - foo 8 | - bar 9 | tasks: 10 | random_task: 11 | pause-before: yes 12 | with: 13 | items: i in <% ctx().items %> 14 | concurrency: 4 15 | action: core.noop 16 | retry: 17 | count: 15 18 | delay: 4 19 | -------------------------------------------------------------------------------- /tests/fixtures/pack/o_actions/mistral-with-items-mixed-list.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | description: Mistral's with-items with a mixed YAQL/Jinja list 3 | enabled: true 4 | entry_point: workflows/mistral-with-items-mixed-list.yaml 5 | name: mistral-with-items-list 6 | pack: examples 7 | parameters: 8 | cmds: 9 | items: 10 | type: string 11 | minItems: 1 12 | type: array 13 | runner_type: orquesta 14 | -------------------------------------------------------------------------------- /tests/fixtures/pack/o_actions/workflows/mistral-with-items-yaql-list.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | version: '1.0' 3 | description: Mistral's with-items with a YAQL list 4 | input: 5 | - tempfile 6 | tasks: 7 | task1: 8 | with: 9 | items: i in <% [0, 1, 2, 3] %> 10 | action: core.local 11 | input: 12 | cmd: x=`cat <% ctx().tempfile %>`; y=`echo "$x * <% item(i) %> % 2" | bc`; exit `echo $y` 13 | -------------------------------------------------------------------------------- /tests/fixtures/pack/pristine_actions/mistral-retry.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | name: mistral-retry 3 | description: A sample workflow used to test the retry feature. 4 | pack: examples 5 | runner_type: mistral-v2 6 | entry_point: workflows/mistral-retry.yaml 7 | enabled: true 8 | parameters: 9 | tempfile: 10 | type: string 11 | required: true 12 | message: 13 | type: string 14 | required: true 15 | -------------------------------------------------------------------------------- /tests/fixtures/pack/pristine_actions/mistral-with-items-yaql.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | description: Run several linux commands in a single task. 3 | enabled: true 4 | entry_point: workflows/mistral-with-items-yaql.yaml 5 | name: mistral-with-items-yaql 6 | pack: examples 7 | parameters: 8 | yaql_cmds: 9 | items: 10 | type: string 11 | minItems: 1 12 | type: array 13 | runner_type: mistral-v2 14 | -------------------------------------------------------------------------------- /tests/fixtures/pack/o_actions/workflows/mistral-with-items-static-list.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | version: '1.0' 3 | description: Mistral's with-items with a static list 4 | input: 5 | - tempfile 6 | tasks: 7 | task1: 8 | with: 9 | items: i in <% [0, 1, 2, 3] %> 10 | action: core.local 11 | input: 12 | cmd: x=`cat <% ctx().tempfile %>`; y=`echo "$x * <% item(i) %> % 2" | bc`; exit `echo $y` 13 | -------------------------------------------------------------------------------- /tests/fixtures/pack/pristine_actions/mistral-with-items-jinja.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | description: Run several linux commands in a single task. 3 | enabled: true 4 | entry_point: workflows/mistral-with-items-jinja.yaml 5 | name: mistral-with-items-jinja 6 | pack: examples 7 | parameters: 8 | jinja_cmds: 9 | items: 10 | type: string 11 | minItems: 1 12 | type: array 13 | runner_type: mistral-v2 14 | -------------------------------------------------------------------------------- /tests/fixtures/mistral/null_publish_parameters.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | version: '2.0' 3 | 4 | st2cicd.null_publish_parameters: 5 | type: direct 6 | output: 7 | continue: '{{ _.continue }}' 8 | tasks: 9 | random_task_one: 10 | action: core.noop 11 | on-success: 12 | - publish_nothing 13 | publish_nothing: 14 | action: core.noop 15 | publish: 16 | continue: null 17 | -------------------------------------------------------------------------------- /tests/fixtures/pack/o_actions/mistral-test-cancel.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | name: mistral-test-cancel 3 | description: A sample workflow used to test the cancel feature. 4 | pack: examples 5 | runner_type: orquesta 6 | entry_point: workflows/mistral-test-cancel.yaml 7 | enabled: true 8 | parameters: 9 | tempfile: 10 | type: string 11 | required: true 12 | message: 13 | type: string 14 | required: true 15 | -------------------------------------------------------------------------------- /tests/fixtures/pack/o_actions/workflows/mistral-with-items-mixed-list.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | version: '1.0' 3 | description: Mistral's with-items with a mixed YAQL/Jinja list 4 | input: 5 | - tempfile 6 | tasks: 7 | task1: 8 | with: 9 | items: i, j, k in <% zip([0, 1, 2, 3], [4, 5, 6, 7], [8, 9]) %> 10 | action: core.local 11 | input: 12 | cmd: echo "<% item(i) %>{{ item(j) }}<% item(k) %>" 13 | -------------------------------------------------------------------------------- /tests/fixtures/pack/pristine_actions/mistral-with-items-mixed-list.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | description: Mistral's with-items with a mixed YAQL/Jinja list 3 | enabled: true 4 | entry_point: workflows/mistral-with-items-mixed-list.yaml 5 | name: mistral-with-items-list 6 | pack: examples 7 | parameters: 8 | cmds: 9 | items: 10 | type: string 11 | minItems: 1 12 | type: array 13 | runner_type: mistral-v2 14 | -------------------------------------------------------------------------------- /tests/fixtures/orquesta/dashes_in_task_names.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | version: '1.0' 3 | tasks: 4 | odd_job: 5 | action: core.noop 6 | next: 7 | - when: '{{ succeeded() }}' 8 | do: 9 | - random_task 10 | - when: '{{ succeeded() and (ctx().jinja_expr) }}' 11 | do: 12 | - weird_todo 13 | random_task: 14 | action: core.noop 15 | weird_todo: 16 | action: core.noop 17 | -------------------------------------------------------------------------------- /tests/fixtures/pack/pristine_actions/mistral-test-cancel.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | name: mistral-test-cancel 3 | description: A sample workflow used to test the cancel feature. 4 | pack: examples 5 | runner_type: mistral-v2 6 | entry_point: workflows/mistral-test-cancel.yaml 7 | enabled: true 8 | parameters: 9 | tempfile: 10 | type: string 11 | required: true 12 | message: 13 | type: string 14 | required: true 15 | -------------------------------------------------------------------------------- /tests/fixtures/pack/o_actions/mistral-retry-break-on.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | name: mistral-retry-break-on 3 | description: A sample workflow used to test the retry feature with break-on. 4 | pack: examples 5 | runner_type: orquesta 6 | entry_point: workflows/mistral-retry-break-on.yaml 7 | enabled: true 8 | parameters: 9 | tempfile: 10 | type: string 11 | required: true 12 | message: 13 | type: string 14 | required: true 15 | -------------------------------------------------------------------------------- /requirements-test.txt: -------------------------------------------------------------------------------- 1 | coverage 2 | codecov 3 | pep8==1.7.1 4 | flake8<3.9.0,>=3.8.0 5 | hacking 6 | # fix isort dependency of pylint 1.9.4 by keeping it below its 5.x version 7 | isort>=4.2.5,<5 8 | pylint==1.9.4 9 | pylint-plugin-utils>=0.4,<1.0 10 | mock==2.0.0 11 | nose>=1.3.7 12 | unittest2 13 | # nosetests enhancements 14 | rednose 15 | nose-timer>=0.7.2,<0.8 16 | # splitting tests run on a separate CI machines 17 | nose-parallel 18 | -------------------------------------------------------------------------------- /tests/fixtures/pack/o_actions/mistral-with-items-concurrency.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | description: Repeat a local linux command for given number of times. 3 | enabled: true 4 | entry_point: workflows/mistral-with-items-concurrency.yaml 5 | name: mistral-with-items-concurrency 6 | pack: examples 7 | parameters: 8 | cmd: 9 | required: true 10 | type: string 11 | count: 12 | default: 6 13 | type: integer 14 | runner_type: orquesta 15 | -------------------------------------------------------------------------------- /tests/fixtures/orquesta/null_publish_parameters.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | version: '1.0' 3 | output: 4 | - continue: '{{ ctx().continue }}' 5 | tasks: 6 | random_task_one: 7 | action: core.noop 8 | next: 9 | - when: '{{ succeeded() }}' 10 | do: 11 | - publish_nothing 12 | publish_nothing: 13 | action: core.noop 14 | next: 15 | - when: '{{ succeeded() }}' 16 | publish: 17 | - continue: 18 | -------------------------------------------------------------------------------- /tests/fixtures/pack/o_actions/mistral-retry-continue-on.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | name: mistral-retry-continue-on 3 | description: A sample workflow used to test the retry feature with continue-on. 4 | pack: examples 5 | runner_type: orquesta 6 | entry_point: workflows/mistral-retry-continue-on.yaml 7 | enabled: true 8 | parameters: 9 | tempfile: 10 | type: string 11 | required: true 12 | message: 13 | type: string 14 | required: true 15 | -------------------------------------------------------------------------------- /tests/fixtures/pack/pristine_actions/mistral-with-items-concurrency.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | description: Repeat a local linux command for given number of times. 3 | enabled: true 4 | entry_point: workflows/mistral-with-items-concurrency.yaml 5 | name: mistral-with-items-concurrency 6 | pack: examples 7 | parameters: 8 | cmd: 9 | required: true 10 | type: string 11 | count: 12 | default: 6 13 | type: integer 14 | runner_type: mistral-v2 15 | -------------------------------------------------------------------------------- /.circleci/config.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | 3 | jobs: 4 | build_and_test: 5 | docker: 6 | - image: circleci/python:3.6.4 7 | 8 | steps: 9 | - checkout 10 | - run: 11 | name: Run tests 12 | command: make 13 | - run: 14 | name: Upload Codecov report 15 | command: make codecov 16 | 17 | workflows: 18 | version: 2 19 | build_test_deploy: 20 | jobs: 21 | - build_and_test 22 | -------------------------------------------------------------------------------- /tests/fixtures/mistral/unsupported_attributes.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | version: '2.0' 3 | 4 | circleci.test_with_items: 5 | input: 6 | - items: 7 | - foo 8 | - bar 9 | output-on-error: 10 | error: Error condition 11 | tasks: 12 | random_task: 13 | action: core.noop 14 | pause-before: yes 15 | with-items: i in <% $.items %> 16 | concurrency: 4 17 | retry: 18 | delay: 4 19 | count: 15 20 | -------------------------------------------------------------------------------- /tests/fixtures/pack/o_actions/mistral-publish-only-transitions.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | name: mistral-publish-only-transitions 3 | description: Test the conversion of publish-only task transitions 4 | pack: examples 5 | runner_type: orquesta 6 | entry_point: workflows/mistral-publish-only-transitions.yaml 7 | enabled: true 8 | parameters: 9 | tempfile: 10 | type: string 11 | required: true 12 | message: 13 | type: string 14 | required: true 15 | -------------------------------------------------------------------------------- /tests/fixtures/pack/o_actions/mistral-with-items-concurrency-str.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | description: Repeat a local linux command for given number of times. 3 | enabled: true 4 | entry_point: workflows/mistral-with-items-concurrency-str.yaml 5 | name: mistral-with-items-concurrency-str 6 | pack: examples 7 | parameters: 8 | cmd: 9 | required: true 10 | type: string 11 | count: 12 | default: 6 13 | type: integer 14 | runner_type: orquesta 15 | -------------------------------------------------------------------------------- /tests/fixtures/pack/o_actions/mistral-with-items-concurrency-yaql.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | description: Repeat a local linux command for given number of times. 3 | enabled: true 4 | entry_point: workflows/mistral-with-items-concurrency-yaql.yaml 5 | name: mistral-with-items-concurrency-yaql 6 | pack: examples 7 | parameters: 8 | cmd: 9 | required: true 10 | type: string 11 | count: 12 | default: 6 13 | type: integer 14 | runner_type: orquesta 15 | -------------------------------------------------------------------------------- /tests/fixtures/pack/o_actions/mistral-with-items-concurrency-jinja.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | description: Repeat a local linux command for given number of times. 3 | enabled: true 4 | entry_point: workflows/mistral-with-items-concurrency-jinja.yaml 5 | name: mistral-with-items-concurrency-jinja 6 | pack: examples 7 | parameters: 8 | cmd: 9 | required: true 10 | type: string 11 | count: 12 | default: 6 13 | type: integer 14 | runner_type: orquesta 15 | -------------------------------------------------------------------------------- /tests/fixtures/pack/pristine_actions/mistral-publish-only-transitions.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | name: mistral-publish-only-transitions 3 | description: Test the conversion of publish-only task transitions 4 | pack: examples 5 | runner_type: mistral-v2 6 | entry_point: workflows/mistral-publish-only-transitions.yaml 7 | enabled: true 8 | parameters: 9 | tempfile: 10 | type: string 11 | required: true 12 | message: 13 | type: string 14 | required: true 15 | -------------------------------------------------------------------------------- /tests/fixtures/pack/pristine_actions/mistral-retry-break-on.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | name: mistral-retry-break-on 3 | description: A sample workflow used to test the retry feature with break-on. 4 | pack: examples 5 | runner_type: mistral-v2 6 | entry_point: workflows/mistral-retry-break-on.yaml 7 | enabled: true 8 | parameters: 9 | tempfile: 10 | type: string 11 | required: true 12 | message: 13 | type: string 14 | required: true 15 | -------------------------------------------------------------------------------- /tests/fixtures/pack/pristine_actions/mistral-with-items-concurrency-str.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | description: Repeat a local linux command for given number of times. 3 | enabled: true 4 | entry_point: workflows/mistral-with-items-concurrency-str.yaml 5 | name: mistral-with-items-concurrency-str 6 | pack: examples 7 | parameters: 8 | cmd: 9 | required: true 10 | type: string 11 | count: 12 | default: 6 13 | type: integer 14 | runner_type: mistral-v2 15 | -------------------------------------------------------------------------------- /tests/fixtures/pack/pristine_actions/mistral-with-items-concurrency-yaql.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | description: Repeat a local linux command for given number of times. 3 | enabled: true 4 | entry_point: workflows/mistral-with-items-concurrency-yaql.yaml 5 | name: mistral-with-items-concurrency-yaql 6 | pack: examples 7 | parameters: 8 | cmd: 9 | required: true 10 | type: string 11 | count: 12 | default: 6 13 | type: integer 14 | runner_type: mistral-v2 15 | -------------------------------------------------------------------------------- /tests/fixtures/pack/pristine_actions/mistral-with-items-concurrency-jinja.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | description: Repeat a local linux command for given number of times. 3 | enabled: true 4 | entry_point: workflows/mistral-with-items-concurrency-jinja.yaml 5 | name: mistral-with-items-concurrency-jinja 6 | pack: examples 7 | parameters: 8 | cmd: 9 | required: true 10 | type: string 11 | count: 12 | default: 6 13 | type: integer 14 | runner_type: mistral-v2 15 | -------------------------------------------------------------------------------- /tests/fixtures/pack/pristine_actions/mistral-retry-continue-on.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | name: mistral-retry-continue-on 3 | description: A sample workflow used to test the retry feature with continue-on. 4 | pack: examples 5 | runner_type: mistral-v2 6 | entry_point: workflows/mistral-retry-continue-on.yaml 7 | enabled: true 8 | parameters: 9 | tempfile: 10 | type: string 11 | required: true 12 | message: 13 | type: string 14 | required: true 15 | -------------------------------------------------------------------------------- /tests/fixtures/pack/o_actions/mistral-retry-continue-and-break-on.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | name: mistral-retry-continue-and-break-on 3 | description: A sample workflow used to test the retry feature with both continue-on and break-on. 4 | pack: examples 5 | runner_type: orquesta 6 | entry_point: workflows/mistral-retry-continue-and-break-on.yaml 7 | enabled: true 8 | parameters: 9 | tempfile: 10 | type: string 11 | required: true 12 | message: 13 | type: string 14 | required: true 15 | -------------------------------------------------------------------------------- /tests/fixtures/orquesta/nasa_apod_twitter_post.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | version: '1.0' 3 | input: 4 | - extra_message 5 | tasks: 6 | get_apod_url: 7 | action: tutorial.nasa_apod 8 | next: 9 | - when: '{{ succeeded() }}' 10 | publish: 11 | - apod_url: '{{ result().result.url }}' 12 | do: 13 | - post_to_twitter 14 | post_to_twitter: 15 | action: twitter.update_status 16 | input: 17 | status: '{{ ctx().extra_message }} {{ ctx().apod_url }}' 18 | -------------------------------------------------------------------------------- /tests/fixtures/orquesta/nasa_apod_twitter_post_yaql.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | version: '1.0' 3 | input: 4 | - extra_message 5 | tasks: 6 | get_apod_url: 7 | action: tutorial.nasa_apod 8 | next: 9 | - when: <% succeeded() %> 10 | publish: 11 | - apod_url: <% result().result.url %> 12 | do: 13 | - post_to_twitter 14 | post_to_twitter: 15 | action: twitter.update_status 16 | input: 17 | status: <% ctx().extra_message %> <% ctx().apod_url %> 18 | -------------------------------------------------------------------------------- /tests/fixtures/pack/o_actions/mistral-transition-expressions.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | name: mistral-transition-expressions 3 | description: > 4 | Test the conversion of task transition expressions in conjunction with the 5 | publish attribute 6 | pack: examples 7 | runner_type: orquesta 8 | entry_point: workflows/mistral-transition-expressions.yaml 9 | enabled: true 10 | parameters: 11 | tempfile: 12 | type: string 13 | required: true 14 | message: 15 | type: string 16 | required: true 17 | -------------------------------------------------------------------------------- /tests/fixtures/pack/pristine_actions/workflows/mistral-with-items-yaql-list.yaml: -------------------------------------------------------------------------------- 1 | version: '2.0' 2 | 3 | examples.mistral-rerun-with-items: 4 | description: Mistral's with-items with a YAQL list 5 | type: direct 6 | input: 7 | - tempfile 8 | tasks: 9 | task1: 10 | with-items: i in <% [0, 1, 2, 3] %> 11 | action: core.local 12 | input: 13 | cmd: 'x=`cat <% $.tempfile %>`; y=`echo "$x * <% $.i %> % 2" | bc`; exit `echo $y`' 14 | -------------------------------------------------------------------------------- /tests/fixtures/pack/pristine_actions/mistral-transition-expressions.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | name: mistral-transition-expressions 3 | description: > 4 | Test the conversion of task transition expressions in conjunction with the 5 | publish attribute 6 | pack: examples 7 | runner_type: mistral-v2 8 | entry_point: workflows/mistral-transition-expressions.yaml 9 | enabled: true 10 | parameters: 11 | tempfile: 12 | type: string 13 | required: true 14 | message: 15 | type: string 16 | required: true 17 | -------------------------------------------------------------------------------- /tests/fixtures/pack/pristine_actions/workflows/mistral-with-items-static-list.yaml: -------------------------------------------------------------------------------- 1 | version: '2.0' 2 | 3 | examples.mistral-with-items-static-list: 4 | description: Mistral's with-items with a static list 5 | type: direct 6 | input: 7 | - tempfile 8 | tasks: 9 | task1: 10 | with-items: i in [0, 1, 2, 3] 11 | action: core.local 12 | input: 13 | cmd: 'x=`cat <% $.tempfile %>`; y=`echo "$x * <% $.i %> % 2" | bc`; exit `echo $y`' 14 | -------------------------------------------------------------------------------- /tests/fixtures/pack/pristine_actions/mistral-retry-continue-and-break-on.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | name: mistral-retry-continue-and-break-on 3 | description: A sample workflow used to test the retry feature with both continue-on and break-on. 4 | pack: examples 5 | runner_type: mistral-v2 6 | entry_point: workflows/mistral-retry-continue-and-break-on.yaml 7 | enabled: true 8 | parameters: 9 | tempfile: 10 | type: string 11 | required: true 12 | message: 13 | type: string 14 | required: true 15 | -------------------------------------------------------------------------------- /tests/fixtures/pack/pristine_actions/mistral-fail-retry-continue-and-break-on.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | name: mistral-fail-retry-continue-and-break-on 3 | description: A sample workflow used to test the retry feature with both continue-on and break-on. 4 | pack: examples 5 | runner_type: mistral-v2 6 | entry_point: workflows/mistral-fail-retry-continue-and-break-on.yaml 7 | enabled: true 8 | parameters: 9 | tempfile: 10 | type: string 11 | required: true 12 | message: 13 | type: string 14 | required: true 15 | -------------------------------------------------------------------------------- /tests/fixtures/mistral/boolean_publish_parameters.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | version: '2.0' 3 | 4 | st2cicd.boolean_publish_parameters: 5 | type: direct 6 | output: 7 | continue: '{{ _.continue }}' 8 | tasks: 9 | random_task_one: 10 | action: core.noop 11 | on-success: 12 | - publish_continue 13 | - publish_stop 14 | publish_continue: 15 | action: core.noop 16 | publish: 17 | continue: true 18 | publish_stop: 19 | action: core.noop 20 | publish: 21 | continue: false 22 | -------------------------------------------------------------------------------- /bin/orquestaconvert.sh: -------------------------------------------------------------------------------- 1 | SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )" 2 | ORQUESTACONVERT_DIR="${SCRIPT_DIR}/.." 3 | VIRTUALENV_DIR="${ORQUESTACONVERT_DIR}/virtualenv" 4 | 5 | # create virtualenv if it doesn't exit 6 | test -d "$VIRTUALENV_DIR" || VIRTUALENV_DIR=$VIRTUALENV_DIR make -C $ORQUESTACONVERT_DIR requirements 7 | 8 | # activate the virtualenv 9 | source ${VIRTUALENV_DIR}/bin/activate 10 | 11 | # run the script, forwarding all arguments passed to this shell script 12 | ${ORQUESTACONVERT_DIR}/orquestaconvert/client.py "$@" 13 | -------------------------------------------------------------------------------- /tests/fixtures/orquesta/output_test.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | version: '1.0' 3 | description: > 4 | workflow that contains an output stanza 5 | input: 6 | - cmd 7 | - filename: 8 | output: 9 | - stdout: '{{ ctx().stdout }}' 10 | - stderr: '{{ ctx().stderr }}' 11 | - filename: '{{ ctx().filename }}' 12 | tasks: 13 | execute: 14 | action: core.local 15 | input: 16 | cmd: '{{ ctx().cmd }}' 17 | next: 18 | - when: '{{ succeeded() }}' 19 | publish: 20 | - stdout: '{{ result().stdout }}' 21 | - stderr: '{{ result().stderr }}' 22 | -------------------------------------------------------------------------------- /tests/fixtures/pack/pristine_actions/workflows/mistral-with-items-yaml-list.yaml: -------------------------------------------------------------------------------- 1 | version: '2.0' 2 | 3 | examples.mistral-with-items-list: 4 | description: Mistral's with-items with a YAML list 5 | type: direct 6 | input: 7 | - tempfile 8 | tasks: 9 | task1: 10 | with-items: 11 | - i in <% [0, 1, 2, 3] %> 12 | - j in <% [4, 5, 6, 7] %> 13 | - k in <% [8, 9] %> 14 | action: core.local 15 | input: 16 | cmd: 'echo "<% $.i %><% $.j %><% $.k %>"' 17 | -------------------------------------------------------------------------------- /bin/orquestaconvert-pack.sh: -------------------------------------------------------------------------------- 1 | SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )" 2 | ORQUESTACONVERT_DIR="${SCRIPT_DIR}/.." 3 | VIRTUALENV_DIR="${ORQUESTACONVERT_DIR}/virtualenv" 4 | 5 | # create virtualenv if it doesn't exit 6 | test -d "$VIRTUALENV_DIR" || VIRTUALENV_DIR=$VIRTUALENV_DIR make -C $ORQUESTACONVERT_DIR requirements 7 | 8 | # activate the virtualenv 9 | source ${VIRTUALENV_DIR}/bin/activate 10 | 11 | # run the script, forwarding all arguments passed to this shell script 12 | ${ORQUESTACONVERT_DIR}/orquestaconvert/pack_client.py "$@" 13 | exit $? 14 | -------------------------------------------------------------------------------- /tests/fixtures/pack/o_actions/workflows/mistral-with-items-yaql.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | version: '1.0' 3 | description: > 4 | A sample workflow that demonstrates how to repeat a task 5 | multiple times with different inputs. 6 | input: 7 | - yaql_cmds 8 | output: 9 | - result: <% ctx().result %> 10 | tasks: 11 | repeat: 12 | with: 13 | items: cmd in <% ctx().yaql_cmds %> 14 | action: core.local cmd=<% item(cmd) %> jinja_cmd={{ item(cmd) }} 15 | next: 16 | - when: <% succeeded() %> 17 | publish: 18 | - result: <% result().select(ctx().stdout) %> 19 | -------------------------------------------------------------------------------- /tests/fixtures/pack/pristine_actions/workflows/mistral-with-items-mixed-list.yaml: -------------------------------------------------------------------------------- 1 | version: '2.0' 2 | 3 | examples.mistral-with-items-list: 4 | description: Mistral's with-items with a mixed YAQL/Jinja list 5 | type: direct 6 | input: 7 | - tempfile 8 | tasks: 9 | task1: 10 | with-items: 11 | - i in <% [0, 1, 2, 3] %> 12 | - j in {{ [4, 5, 6, 7] }} 13 | - k in <% [8, 9] %> 14 | action: core.local 15 | input: 16 | cmd: 'echo "<% $.i %>{{ _.j }}<% $.k %>"' 17 | -------------------------------------------------------------------------------- /tests/fixtures/pack/o_actions/workflows/mistral-with-items-jinja.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | version: '1.0' 3 | description: > 4 | A sample workflow that demonstrates how to repeat a task 5 | multiple times with different inputs. 6 | input: 7 | - jinja_cmds 8 | output: 9 | - result: <% ctx().result %> 10 | tasks: 11 | repeat: 12 | with: 13 | items: cmd in {{ ctx().jinja_cmds }} 14 | action: core.local 15 | input: 16 | cmd: '{{ item(cmd) }}' 17 | next: 18 | - when: <% succeeded() %> 19 | publish: 20 | - result: "{{ result() | map(attribute='stdout') | list }}" 21 | -------------------------------------------------------------------------------- /tests/fixtures/pack/o_actions/workflows/mistral-test-cancel.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | version: '1.0' 3 | description: A sample workflow used to test the cancellation feature. 4 | input: 5 | - tempfile 6 | - message 7 | tasks: 8 | task1: 9 | action: core.local 10 | input: 11 | cmd: while [ -e '{{ ctx().tempfile }}' ]; do sleep 0.1; done 12 | timeout: 300 13 | next: 14 | - when: <% succeeded() %> 15 | publish: 16 | - var1: <% ctx().message %> 17 | do: 18 | - task2 19 | task2: 20 | action: core.noop 21 | input: 22 | cmd: echo "{{ ctx().var1 }}" 23 | -------------------------------------------------------------------------------- /tests/fixtures/mistral/int_publish_parameters.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | version: '2.0' 3 | 4 | st2cicd.int_publish_parameters: 5 | type: direct 6 | output: 7 | continue: '{{ _.continue }}' 8 | tasks: 9 | random_task_one: 10 | action: core.noop 11 | publish: 12 | continue: 0 13 | on-success: 14 | - publish_one_hundred 15 | publish_one_hundred: 16 | action: core.noop 17 | publish: 18 | continue: 100 19 | on-success: 20 | - publish_negative_one 21 | publish_negative_one: 22 | action: core.noop 23 | publish: 24 | continue: -1 25 | -------------------------------------------------------------------------------- /tests/fixtures/mistral/output_test.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | version: '2.0' 3 | 4 | test.workflow_with_output: 5 | description: > 6 | workflow that contains an output stanza 7 | type: direct 8 | input: 9 | - cmd 10 | - filename: null 11 | 12 | output: 13 | stdout: "{{ _.stdout }}" 14 | stderr: "{{ _.stderr }}" 15 | filename: "{{ _.filename }}" 16 | 17 | tasks: 18 | execute: 19 | action: core.local 20 | input: 21 | cmd: "{{ _.cmd }}" 22 | publish: 23 | stdout: "{{ task('execute').result.stdout }}" 24 | stderr: "{{ task('execute').result.stderr }}" 25 | -------------------------------------------------------------------------------- /tests/fixtures/orquesta/boolean_publish_parameters.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | version: '1.0' 3 | output: 4 | - continue: '{{ ctx().continue }}' 5 | tasks: 6 | random_task_one: 7 | action: core.noop 8 | next: 9 | - when: '{{ succeeded() }}' 10 | do: 11 | - publish_continue 12 | - publish_stop 13 | publish_continue: 14 | action: core.noop 15 | next: 16 | - when: '{{ succeeded() }}' 17 | publish: 18 | - continue: true 19 | publish_stop: 20 | action: core.noop 21 | next: 22 | - when: '{{ succeeded() }}' 23 | publish: 24 | - continue: false 25 | -------------------------------------------------------------------------------- /tests/fixtures/mistral/nasa_apod_twitter_post_yaql.yaml: -------------------------------------------------------------------------------- 1 | version: '2.0' 2 | 3 | tutorial.nasa_apod_twitter_post: 4 | # don't put type in here until the following is merged: https://github.com/StackStorm/orchestra/pull/71 5 | #type: direct 6 | input: 7 | - extra_message 8 | 9 | tasks: 10 | get_apod_url: 11 | action: tutorial.nasa_apod 12 | publish: 13 | apod_url: <% task(get_apod_url).result.result.url %> 14 | on-success: 15 | - post_to_twitter 16 | 17 | post_to_twitter: 18 | action: twitter.update_status 19 | input: 20 | status: <% $.extra_message %> <% $.apod_url %> 21 | -------------------------------------------------------------------------------- /tests/fixtures/mistral/nasa_apod_twitter_post.yaml: -------------------------------------------------------------------------------- 1 | version: '2.0' 2 | 3 | tutorial.nasa_apod_twitter_post: 4 | # don't put type in here until the following is merged: https://github.com/StackStorm/orchestra/pull/71 5 | #type: direct 6 | input: 7 | - extra_message 8 | 9 | tasks: 10 | get_apod_url: 11 | action: tutorial.nasa_apod 12 | publish: 13 | apod_url: "{{ task('get_apod_url').result.result.url }}" 14 | on-success: 15 | - post_to_twitter 16 | 17 | post_to_twitter: 18 | action: twitter.update_status 19 | input: 20 | status: "{{ _.extra_message }} {{ _.apod_url }}" 21 | -------------------------------------------------------------------------------- /tests/fixtures/pack/pristine_actions/workflows/mistral-with-items-yaql.yaml: -------------------------------------------------------------------------------- 1 | version: '2.0' 2 | 3 | examples.mistral-with-items-yaql: 4 | description: > 5 | A sample workflow that demonstrates how to repeat a task 6 | multiple times with different inputs. 7 | type: direct 8 | input: 9 | - yaql_cmds 10 | output: 11 | result: <% $.result %> 12 | tasks: 13 | repeat: 14 | with-items: cmd in <% $.yaql_cmds %> 15 | action: core.local cmd=<% $.cmd %> jinja_cmd={{ _.cmd }} 16 | publish: 17 | result: <% task(repeat).result.select($.stdout) %> 18 | -------------------------------------------------------------------------------- /tests/fixtures/pack/o_actions/workflows/mistral-retry.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | version: '1.0' 3 | description: A sample workflow used to test the retry feature. 4 | input: 5 | - tempfile 6 | - message 7 | tasks: 8 | task1: 9 | action: core.local 10 | retry: 11 | count: 15 12 | delay: 5 13 | input: 14 | cmd: while [ -e '{{ $.tempfile }}' ]; do sleep 0.1; done 15 | timeout: 300 16 | next: 17 | - when: <% succeeded() %> 18 | publish: 19 | - var1: <% ctx().message %> 20 | do: 21 | - task2 22 | task2: 23 | action: core.local 24 | input: 25 | cmd: echo "{{ $.var1 }}" 26 | -------------------------------------------------------------------------------- /tests/fixtures/pack/pristine_actions/workflows/mistral-with-items-jinja.yaml: -------------------------------------------------------------------------------- 1 | version: '2.0' 2 | 3 | examples.mistral-jinja-repeat-with-items: 4 | description: > 5 | A sample workflow that demonstrates how to repeat a task 6 | multiple times with different inputs. 7 | type: direct 8 | input: 9 | - jinja_cmds 10 | output: 11 | result: <% $.result %> 12 | tasks: 13 | repeat: 14 | with-items: "cmd in {{ _.jinja_cmds }}" 15 | action: core.local 16 | input: 17 | cmd: "{{ _.cmd }}" 18 | publish: 19 | result: "{{ task('repeat').result | map(attribute='stdout') | list }}" 20 | -------------------------------------------------------------------------------- /tests/fixtures/orquesta/int_publish_parameters.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | version: '1.0' 3 | output: 4 | - continue: '{{ ctx().continue }}' 5 | tasks: 6 | random_task_one: 7 | action: core.noop 8 | next: 9 | - when: '{{ succeeded() }}' 10 | publish: 11 | - continue: 0 12 | do: 13 | - publish_one_hundred 14 | publish_one_hundred: 15 | action: core.noop 16 | next: 17 | - when: '{{ succeeded() }}' 18 | publish: 19 | - continue: 100 20 | do: 21 | - publish_negative_one 22 | publish_negative_one: 23 | action: core.noop 24 | next: 25 | - when: '{{ succeeded() }}' 26 | publish: 27 | - continue: -1 28 | -------------------------------------------------------------------------------- /orquestaconvert/specs/mistral/v2/types.py: -------------------------------------------------------------------------------- 1 | # Copyright 2019 Extreme Networks, Inc. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | WORKFLOW_TYPE = {"enum": ["reverse", "direct"]} 16 | -------------------------------------------------------------------------------- /tests/fixtures/pack/pristine_actions/workflows/mistral-test-cancel.yaml: -------------------------------------------------------------------------------- 1 | version: '2.0' 2 | 3 | examples.mistral-test-cancel: 4 | description: A sample workflow used to test the cancellation feature. 5 | type: direct 6 | input: 7 | - tempfile 8 | - message 9 | tasks: 10 | task1: 11 | action: core.local 12 | input: 13 | cmd: "while [ -e '{{ _.tempfile }}' ]; do sleep 0.1; done" 14 | timeout: 300 15 | publish: 16 | var1: <% $.message %> 17 | on-success: 18 | - task2 19 | task2: 20 | action: std.noop 21 | input: 22 | cmd: echo "{{ _.var1 }}" 23 | -------------------------------------------------------------------------------- /tests/fixtures/pack/o_actions/workflows/mistral-retry-break-on.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | version: '1.0' 3 | description: > 4 | A sample workflow that demonstrates retry with count, delay, and break-on. 5 | tasks: 6 | init: 7 | action: core.local cmd="rm -f /tmp/done" 8 | next: 9 | - when: <% succeeded() %> 10 | do: 11 | - test_error_undo_retry 12 | test_error_undo_retry: 13 | action: core.local cmd="echo 'Do something useful here.';" 14 | retry: 15 | count: 30 16 | delay: 1 17 | when: <% failed() and not (ctx().bar = 'BREAK') %> 18 | next: 19 | - when: <% succeeded() %> 20 | do: 21 | - delete_file 22 | delete_file: 23 | action: core.local cmd="rm -f /tmp/done" 24 | -------------------------------------------------------------------------------- /tests/fixtures/pack/o_actions/workflows/mistral-retry-continue-on.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | version: '1.0' 3 | description: > 4 | A sample workflow that demonstrates retry with count, delay, and continue-on. 5 | tasks: 6 | init: 7 | action: core.local cmd="rm -f /tmp/done" 8 | next: 9 | - when: <% succeeded() %> 10 | do: 11 | - test_error_undo_retry 12 | test_error_undo_retry: 13 | action: core.local cmd="echo 'Do something useful here.';" 14 | retry: 15 | count: 30 16 | delay: 1 17 | when: <% succeeded() and (ctx().foo = 'continue') %> 18 | next: 19 | - when: <% succeeded() %> 20 | do: 21 | - delete_file 22 | delete_file: 23 | action: core.local cmd="rm -f /tmp/done" 24 | -------------------------------------------------------------------------------- /doc/expressions.md: -------------------------------------------------------------------------------- 1 | # Converting Expressions from Mistral -> Orchestra 2 | 3 | ## Jinja 4 | 5 | * `_.xxx` becomes `ctx().xxx` 6 | 7 | ## YAQL 8 | 9 | * `$.xxx` becomes `ctx().xxx` (how does this affect the `when` context for `$.` ?) 10 | 11 | ## Common 12 | 13 | * `task('xxx').result` becomes `result()` , but only if the current task is `xxx` 14 | * **TODO** How do we handle `task('yyy').result` when referencing a task outside of itself 15 | * **TODO** How do we handle `st2kv.x.y.z` and `| decrypt_kv` 16 | * **TODO** How do we handle other custom Jinja/YAQL filters https://docs.stackstorm.com/reference/jinja.html#custom-jinja-filters 17 | * **TODO** Are custom filters going to be supported in the `| custom_filter` syntax, or just the `custom_filter()` syntax? 18 | -------------------------------------------------------------------------------- /tests/fixtures/pack/pristine_actions/workflows/mistral-retry.yaml: -------------------------------------------------------------------------------- 1 | version: '2.0' 2 | 3 | examples.mistral-retry: 4 | description: A sample workflow used to test the retry feature. 5 | type: direct 6 | input: 7 | - tempfile 8 | - message 9 | tasks: 10 | task1: 11 | action: core.local 12 | retry: 13 | delay: 5 14 | count: 15 15 | input: 16 | cmd: "while [ -e '{{ $.tempfile }}' ]; do sleep 0.1; done" 17 | timeout: 300 18 | publish: 19 | var1: <% $.message %> 20 | on-success: 21 | - task2 22 | task2: 23 | action: core.local 24 | input: 25 | cmd: echo "{{ $.var1 }}" 26 | -------------------------------------------------------------------------------- /tests/fixtures/pack/o_actions/workflows/mistral-with-items-concurrency.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | version: '1.0' 3 | description: > 4 | A sample workflow that demonstrates how to set the concurrency option 5 | in with-items to throttle the number of action executions that get 6 | run simultaneously. Currently in this release, the concurrency option 7 | does not work with YAQL expression. 8 | input: 9 | - cmd 10 | - count 11 | output: 12 | - result: <% ctx().result %> 13 | tasks: 14 | repeat: 15 | with: 16 | items: i in <% list(range(0, ctx().count)) %> 17 | concurrency: 2 18 | action: core.local 19 | input: 20 | cmd: <% ctx().cmd %>; sleep 3 21 | next: 22 | - when: <% succeeded() %> 23 | publish: 24 | - result: <% result().select(ctx().stdout) %> 25 | -------------------------------------------------------------------------------- /orquestaconvert/specs/mistral/v2/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2019 Extreme Networks, Inc. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | from orquestaconvert.specs.mistral.v2 import base as mistral_v2_base 16 | 17 | 18 | VERSION = mistral_v2_base.Spec.get_version() 19 | -------------------------------------------------------------------------------- /tests/fixtures/pack/o_actions/workflows/mistral-with-items-concurrency-str.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | version: '1.0' 3 | description: > 4 | A sample workflow that demonstrates how to set the concurrency option 5 | in with-items to throttle the number of action executions that get 6 | run simultaneously. Currently in this release, the concurrency option 7 | does not work with YAQL expression. 8 | input: 9 | - cmd 10 | - count 11 | output: 12 | - result: <% ctx().result %> 13 | tasks: 14 | repeat: 15 | with: 16 | items: i in <% list(range(0, ctx().count)) %> 17 | concurrency: '2' 18 | action: core.local 19 | input: 20 | cmd: <% ctx().cmd %>; sleep 3 21 | next: 22 | - when: <% succeeded() %> 23 | publish: 24 | - result: <% result().select(ctx().stdout) %> 25 | -------------------------------------------------------------------------------- /tests/fixtures/pack/pristine_actions/workflows/mistral-retry-break-on.yaml: -------------------------------------------------------------------------------- 1 | version: '2.0' 2 | 3 | examples.mistral-retry-break-on: 4 | description: > 5 | A sample workflow that demonstrates retry with count, delay, and break-on. 6 | type: direct 7 | tasks: 8 | init: 9 | action: core.local cmd="rm -f /tmp/done" 10 | on-success: 11 | - test-error-undo-retry 12 | test-error-undo-retry: 13 | retry: 14 | count: 30 15 | delay: 1 16 | break-on: <% $.bar = 'BREAK' %> 17 | action: core.local cmd="echo 'Do something useful here.';" 18 | on-success: 19 | - delete-file 20 | delete-file: 21 | action: core.local cmd="rm -f /tmp/done" 22 | -------------------------------------------------------------------------------- /tests/fixtures/pack/o_actions/workflows/mistral-with-items-concurrency-jinja.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | version: '1.0' 3 | description: > 4 | A sample workflow that demonstrates how to set the concurrency option 5 | in with-items to throttle the number of action executions that get 6 | run simultaneously. Currently in this release, the concurrency option 7 | does not work with YAQL expression. 8 | input: 9 | - cmd 10 | - count 11 | output: 12 | - result: <% ctx().result %> 13 | tasks: 14 | repeat: 15 | with: 16 | items: i in {{ range(0, ctx().count) }} 17 | concurrency: '{{ ctx().count }}' 18 | action: core.local 19 | input: 20 | cmd: '{{ ctx().cmd }}; sleep 3' 21 | next: 22 | - when: <% succeeded() %> 23 | publish: 24 | - result: '{{ result()|selectattr("stdout") }}' 25 | -------------------------------------------------------------------------------- /tests/fixtures/pack/o_actions/workflows/mistral-with-items-concurrency-yaql.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | version: '1.0' 3 | description: > 4 | A sample workflow that demonstrates how to set the concurrency option 5 | in with-items to throttle the number of action executions that get 6 | run simultaneously. Currently in this release, the concurrency option 7 | does not work with YAQL expression. 8 | input: 9 | - cmd 10 | - count 11 | output: 12 | - result: <% ctx().result %> 13 | tasks: 14 | repeat: 15 | with: 16 | items: i in <% list(range(0, ctx().count)) %> 17 | concurrency: <% ctx().count %> 18 | action: core.local 19 | input: 20 | cmd: <% ctx().cmd %>; sleep 3 21 | next: 22 | - when: <% succeeded() %> 23 | publish: 24 | - result: <% result().select(ctx().stdout) %> 25 | -------------------------------------------------------------------------------- /tests/fixtures/pack/pristine_actions/workflows/mistral-retry-continue-on.yaml: -------------------------------------------------------------------------------- 1 | version: '2.0' 2 | 3 | examples.mistral-retry-continue-on: 4 | description: > 5 | A sample workflow that demonstrates retry with count, delay, and continue-on. 6 | type: direct 7 | tasks: 8 | init: 9 | action: core.local cmd="rm -f /tmp/done" 10 | on-success: 11 | - test-error-undo-retry 12 | test-error-undo-retry: 13 | retry: 14 | count: 30 15 | delay: 1 16 | continue-on: <% $.foo = 'continue' %> 17 | action: core.local cmd="echo 'Do something useful here.';" 18 | on-success: 19 | - delete-file 20 | delete-file: 21 | action: core.local cmd="rm -f /tmp/done" 22 | -------------------------------------------------------------------------------- /tests/fixtures/mistral/complex_publish_conditions.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | version: '2.0' 3 | 4 | tests.complex_publish_conditions: 5 | type: direct 6 | input: 7 | - var1: initial_value_1 8 | - var2: initial_value_2 9 | - var3: initial_value_3 10 | output: 11 | var1: '{{ _.var1 }}' 12 | var2: '{{ _.var2 }}' 13 | var3: '{{ _.var3 }}' 14 | tasks: 15 | first_task: 16 | action: core.noop 17 | publish: 18 | var1: value_1_from_first_task 19 | publish-on-error: 20 | var2: value_2_from_first_task 21 | on-success: 22 | - success_task 23 | on-error: 24 | - error_task 25 | on-complete: 26 | - complete_task 27 | success_task: 28 | action: core.noop 29 | error_task: 30 | action: core.noop 31 | complete_task: 32 | action: core.noop 33 | -------------------------------------------------------------------------------- /tests/fixtures/pack/pristine_actions/workflows/mistral-with-items-concurrency.yaml: -------------------------------------------------------------------------------- 1 | version: '2.0' 2 | 3 | examples.mistral-with-items-concurrency: 4 | description: > 5 | A sample workflow that demonstrates how to set the concurrency option 6 | in with-items to throttle the number of action executions that get 7 | run simultaneously. Currently in this release, the concurrency option 8 | does not work with YAQL expression. 9 | type: direct 10 | input: 11 | - cmd 12 | - count 13 | output: 14 | result: <% $.result %> 15 | tasks: 16 | repeat: 17 | with-items: i in <% list(range(0, $.count)) %> 18 | concurrency: 2 19 | action: core.local 20 | input: 21 | cmd: "<% $.cmd %>; sleep 3" 22 | publish: 23 | result: <% task(repeat).result.select($.stdout) %> 24 | -------------------------------------------------------------------------------- /tests/fixtures/orquesta/retry-with-break-on.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | version: '1.0' 3 | description: > 4 | A sample workflow that demonstrates how to handle rollback and retry on error. In this example, the workflow will error and then continue to retry until the file /tmp/done exists. A parallel task will wait for some time before creating the 5 | file. When completed, /tmp/done will be deleted. 6 | tasks: 7 | init: 8 | action: core.local cmd="rm -f /tmp/done" 9 | next: 10 | - when: '{{ succeeded() }}' 11 | do: 12 | - test_error_undo_retry 13 | test_error_undo_retry: 14 | action: core.local cmd="echo 'Do something useful here.';" 15 | retry: 16 | count: 30 17 | delay: 1 18 | when: <% ctx().bar != 'BREAK' %> 19 | next: 20 | - when: '{{ succeeded() }}' 21 | do: 22 | - delete_file 23 | delete_file: 24 | action: core.local cmd="rm -f /tmp/done" 25 | -------------------------------------------------------------------------------- /tests/fixtures/pack/pristine_actions/workflows/mistral-with-items-concurrency-str.yaml: -------------------------------------------------------------------------------- 1 | version: '2.0' 2 | 3 | examples.mistral-with-items-concurrency-str: 4 | description: > 5 | A sample workflow that demonstrates how to set the concurrency option 6 | in with-items to throttle the number of action executions that get 7 | run simultaneously. Currently in this release, the concurrency option 8 | does not work with YAQL expression. 9 | type: direct 10 | input: 11 | - cmd 12 | - count 13 | output: 14 | result: <% $.result %> 15 | tasks: 16 | repeat: 17 | with-items: i in <% list(range(0, $.count)) %> 18 | concurrency: '2' 19 | action: core.local 20 | input: 21 | cmd: "<% $.cmd %>; sleep 3" 22 | publish: 23 | result: <% task(repeat).result.select($.stdout) %> 24 | -------------------------------------------------------------------------------- /tests/fixtures/orquesta/retry-with-continue-on.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | version: '1.0' 3 | description: > 4 | A sample workflow that demonstrates how to handle rollback and retry on error. In this example, the workflow will error and then continue to retry until the file /tmp/done exists. A parallel task will wait for some time before creating the 5 | file. When completed, /tmp/done will be deleted. 6 | tasks: 7 | init: 8 | action: core.local cmd="rm -f /tmp/done" 9 | next: 10 | - when: '{{ succeeded() }}' 11 | do: 12 | - test_error_undo_retry 13 | test_error_undo_retry: 14 | action: core.local cmd="echo 'Do something useful here.';" 15 | retry: 16 | count: 30 17 | delay: 1 18 | when: <% ctx().foo = 'continue' %> 19 | next: 20 | - when: '{{ succeeded() }}' 21 | do: 22 | - delete_file 23 | delete_file: 24 | action: core.local cmd="rm -f /tmp/done" 25 | -------------------------------------------------------------------------------- /tests/fixtures/pack/pristine_actions/workflows/mistral-with-items-concurrency-yaql.yaml: -------------------------------------------------------------------------------- 1 | version: '2.0' 2 | 3 | examples.mistral-with-items-concurrency-yaql: 4 | description: > 5 | A sample workflow that demonstrates how to set the concurrency option 6 | in with-items to throttle the number of action executions that get 7 | run simultaneously. Currently in this release, the concurrency option 8 | does not work with YAQL expression. 9 | type: direct 10 | input: 11 | - cmd 12 | - count 13 | output: 14 | result: <% $.result %> 15 | tasks: 16 | repeat: 17 | with-items: i in <% list(range(0, $.count)) %> 18 | concurrency: <% $.count %> 19 | action: core.local 20 | input: 21 | cmd: "<% $.cmd %>; sleep 3" 22 | publish: 23 | result: <% task(repeat).result.select($.stdout) %> 24 | -------------------------------------------------------------------------------- /tests/fixtures/pack/pristine_actions/workflows/mistral-with-items-concurrency-jinja.yaml: -------------------------------------------------------------------------------- 1 | version: '2.0' 2 | 3 | examples.mistral-with-items-concurrency-jinja: 4 | description: > 5 | A sample workflow that demonstrates how to set the concurrency option 6 | in with-items to throttle the number of action executions that get 7 | run simultaneously. Currently in this release, the concurrency option 8 | does not work with YAQL expression. 9 | type: direct 10 | input: 11 | - cmd 12 | - count 13 | output: 14 | result: <% $.result %> 15 | tasks: 16 | repeat: 17 | with-items: 'i in {{ range(0, _.count) }}' 18 | concurrency: '{{ _.count }}' 19 | action: core.local 20 | input: 21 | cmd: '{{ _.cmd }}; sleep 3' 22 | publish: 23 | result: '{{ task(repeat).result|selectattr("stdout") }}' 24 | -------------------------------------------------------------------------------- /tests/fixtures/orquesta/complex_publish_conditions.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | version: '1.0' 3 | input: 4 | - var1: initial_value_1 5 | - var2: initial_value_2 6 | - var3: initial_value_3 7 | output: 8 | - var1: '{{ _.var1 }}' 9 | - var2: '{{ _.var2 }}' 10 | - var3: '{{ _.var3 }}' 11 | tasks: 12 | first_task: 13 | action: core.noop 14 | next: 15 | - when: <% succeeded() %> 16 | publish: 17 | - var1: value_1_from_first_task 18 | do: 19 | - success_task 20 | - when: <% failed() %> 21 | publish: 22 | - var2: value_2_from_first_task 23 | do: 24 | - error_task 25 | - publish: 26 | - var1: value_1_from_first_task 27 | - var2: value_2_from_first_task 28 | do: 29 | - complete_task 30 | success_task: 31 | action: core.noop 32 | error_task: 33 | action: core.noop 34 | complete_task: 35 | action: core.noop 36 | -------------------------------------------------------------------------------- /tests/fixtures/pack/o_actions/workflows/mistral-retry-continue-and-break-on.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | version: '1.0' 3 | description: > 4 | A sample workflow that demonstrates how to handle rollback and retry on error. In this example, the workflow will error and then continue to retry until the file /tmp/done exists. A parallel task will wait for some time before creating the 5 | file. When completed, /tmp/done will be deleted. 6 | tasks: 7 | init: 8 | action: core.local cmd="rm -f /tmp/done" 9 | next: 10 | - when: <% succeeded() %> 11 | do: 12 | - test_error_undo_retry 13 | test_error_undo_retry: 14 | action: core.local cmd="echo 'Do something useful here.';" 15 | retry: 16 | count: 30 17 | delay: 1 18 | when: <% (succeeded() and (ctx().foo = 'continue')) or (failed() and not (ctx().bar = 'BREAK')) %> 19 | next: 20 | - when: <% succeeded() %> 21 | do: 22 | - delete_file 23 | delete_file: 24 | action: core.local cmd="rm -f /tmp/done" 25 | -------------------------------------------------------------------------------- /tests/unit/test_utils_type_utils.py: -------------------------------------------------------------------------------- 1 | # Licensed under the Apache License, Version 2.0 (the "License"); 2 | # you may not use this file except in compliance with the License. 3 | # You may obtain a copy of the License at 4 | # 5 | # http://www.apache.org/licenses/LICENSE-2.0 6 | # 7 | # Unless required by applicable law or agreed to in writing, software 8 | # distributed under the License is distributed on an "AS IS" BASIS, 9 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 10 | # See the License for the specific language governing permissions and 11 | # limitations under the License. 12 | 13 | import ruamel.yaml 14 | 15 | from orquestaconvert.utils import type_utils 16 | 17 | from tests import base_test_case 18 | 19 | 20 | class TestTypeUtils(base_test_case.BaseTestCase): 21 | __test__ = True 22 | 23 | def test_dict_types(self): 24 | self.assertIsInstance({}, type_utils.dict_types) 25 | self.assertIsInstance(ruamel.yaml.comments.CommentedMap(), type_utils.dict_types) 26 | -------------------------------------------------------------------------------- /tests/fixtures/mistral/emptywee_test.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | version: '2.0' 3 | 4 | c_int.wf_test_join: 5 | input: 6 | - x 7 | - y 8 | 9 | tasks: 10 | setup_task: 11 | action: core.noop 12 | on-success: 13 | - validate_X 14 | - validate_Y 15 | 16 | validate_X: 17 | with-items: ix in <% $.x %> 18 | action: core.local 19 | input: 20 | cmd: 'test <% str($.ix)%> == "ok"' 21 | on-success: 22 | - wait_validations 23 | on-error: 24 | - wait_validations 25 | 26 | validate_Y: 27 | with-items: iy in <% $.y %> 28 | action: core.local 29 | input: 30 | cmd: 'test <% str($.iy) %> == "ok"' 31 | on-success: 32 | - wait_validations 33 | on-error: 34 | - wait_validations 35 | 36 | wait_validations: 37 | join: all 38 | timeout: 600 39 | action: core.noop 40 | on-success: 41 | - do_it: '<% len($.x) or len($.y) %>' 42 | 43 | do_it: 44 | action: core.noop 45 | -------------------------------------------------------------------------------- /tests/unit/test_utils_task_utils.py: -------------------------------------------------------------------------------- 1 | # Licensed under the Apache License, Version 2.0 (the "License"); 2 | # you may not use this file except in compliance with the License. 3 | # You may obtain a copy of the License at 4 | # 5 | # http://www.apache.org/licenses/LICENSE-2.0 6 | # 7 | # Unless required by applicable law or agreed to in writing, software 8 | # distributed under the License is distributed on an "AS IS" BASIS, 9 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 10 | # See the License for the specific language governing permissions and 11 | # limitations under the License. 12 | 13 | from orquestaconvert.utils import task_utils 14 | 15 | from tests import base_test_case 16 | 17 | 18 | class TestTaskUtils(base_test_case.BaseTestCase): 19 | __test__ = True 20 | 21 | def test_convert_dashes_to_underscores(self): 22 | self.assertEqual(task_utils.translate_task_name('foo-bar-baz'), 'foo_bar_baz') 23 | self.assertEqual(task_utils.translate_task_name('baz_foo'), 'baz_foo') 24 | -------------------------------------------------------------------------------- /tests/fixtures/orquesta/emptywee_test.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | version: '1.0' 3 | input: 4 | - x 5 | - y 6 | tasks: 7 | setup_task: 8 | action: core.noop 9 | next: 10 | - when: '{{ succeeded() }}' 11 | do: 12 | - validate_X 13 | - validate_Y 14 | validate_X: 15 | action: core.local 16 | input: 17 | cmd: test <% str(ctx().ix)%> == "ok" 18 | next: 19 | - when: '{{ succeeded() }}' 20 | do: 21 | - wait_validations 22 | - when: '{{ failed() }}' 23 | do: 24 | - wait_validations 25 | validate_Y: 26 | action: core.local 27 | input: 28 | cmd: test <% str(ctx().iy) %> == "ok" 29 | next: 30 | - when: '{{ succeeded() }}' 31 | do: 32 | - wait_validations 33 | - when: '{{ failed() }}' 34 | do: 35 | - wait_validations 36 | wait_validations: 37 | action: core.noop 38 | join: all 39 | next: 40 | - when: <% succeeded() and (len(ctx().x) or len(ctx().y)) %> 41 | do: 42 | - do_it 43 | do_it: 44 | action: core.noop 45 | -------------------------------------------------------------------------------- /tests/fixtures/pack/pristine_actions/workflows/mistral-retry-continue-and-break-on.yaml: -------------------------------------------------------------------------------- 1 | version: '2.0' 2 | 3 | examples.mistral-retry-continue-and-break-on: 4 | description: > 5 | A sample workflow that demonstrates how to handle rollback and retry on error. 6 | In this example, the workflow will error and then continue to retry until the file 7 | /tmp/done exists. A parallel task will wait for some time before creating the 8 | file. When completed, /tmp/done will be deleted. 9 | type: direct 10 | tasks: 11 | init: 12 | action: core.local cmd="rm -f /tmp/done" 13 | on-success: 14 | - test-error-undo-retry 15 | test-error-undo-retry: 16 | retry: 17 | count: 30 18 | delay: 1 19 | continue-on: <% $.foo = 'continue' %> 20 | break-on: <% $.bar = 'BREAK' %> 21 | action: core.local cmd="echo 'Do something useful here.';" 22 | on-success: 23 | - delete-file 24 | delete-file: 25 | action: core.local cmd="rm -f /tmp/done" 26 | -------------------------------------------------------------------------------- /tests/fixtures/pack/pristine_actions/workflows/mistral-fail-retry-continue-and-break-on.yaml: -------------------------------------------------------------------------------- 1 | version: '2.0' 2 | 3 | examples.mistral-fail-retry-continue-and-break-on: 4 | description: > 5 | A sample workflow that demonstrates how to handle rollback and retry on error. 6 | In this example, the workflow will error and then continue to retry until the file 7 | /tmp/done exists. A parallel task will wait for some time before creating the 8 | file. When completed, /tmp/done will be deleted. 9 | type: direct 10 | tasks: 11 | init: 12 | action: core.local cmd="rm -f /tmp/done" 13 | on-success: 14 | - test-error-undo-retry 15 | test-error-undo-retry: 16 | retry: 17 | count: 30 18 | delay: 1 19 | continue-on: <% $.foo = 'continue' %> 20 | break-on: '{{ _.bar = "BREAK" }}' 21 | action: core.local cmd="echo 'Do something useful here.';" 22 | on-success: 23 | - delete-file 24 | delete-file: 25 | action: core.local cmd="rm -f /tmp/done" 26 | -------------------------------------------------------------------------------- /tests/fixtures/mistral/immediately_referenced_context_variables.yaml: -------------------------------------------------------------------------------- 1 | version: "2.0" 2 | 3 | test_workflow: 4 | type: direct 5 | description: Test 6 | input: 7 | - uuid 8 | tasks: 9 | task_1: 10 | action: pack.action_1 11 | # returns result as dict. 12 | input: 13 | foreign_id: <% $.uuid %> 14 | publish: 15 | operations: <% task(task_1).result.result['operations'] + 1 %> 16 | namespace: <% task(task_1).result.result['namespace'] %> 17 | on-success: 18 | - task_1: <% ($.operations) %> 19 | - task_2: <% len($.operations) = "len function" %> 20 | - task_3: <% $.operations * 4 > 0 %> 21 | - task_3: <% 0 < 4 * $.operations %> 22 | - task_4: <% $.operations|length %> 23 | - task_5: <% 5 + $.operations * 4 %> 24 | - task_6: <% ctx(operations) %> 25 | - task_7: "{{ _.operations }}" 26 | - task_8: <% asdfctx(operations) %> 27 | - task_9: <% $.operations_NOT %> 28 | task_2: 29 | action: core.noop 30 | task_3: 31 | action: core.noop 32 | task_4: 33 | action: core.noop 34 | task_5: 35 | action: core.noop 36 | task_6: 37 | action: core.noop 38 | task_7: 39 | action: core.noop 40 | task_8: 41 | action: core.noop 42 | -------------------------------------------------------------------------------- /orquestaconvert/specs/mistral/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2019 Extreme Networks, Inc. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | from orquestaconvert.specs.mistral import v2 as mistral_v2_specs 16 | from orquestaconvert.specs.mistral.v2 import tasks as task_models 17 | from orquestaconvert.specs.mistral.v2 import workflows as workflow_models 18 | 19 | 20 | VERSION = mistral_v2_specs.VERSION 21 | deserialize = workflow_models.deserialize 22 | instantiate = workflow_models.instantiate 23 | TaskDefaultsSpec = task_models.TaskDefaultsSpec 24 | TaskSpec = task_models.TaskSpec 25 | WorkflowSpec = workflow_models.WorkflowSpec 26 | 27 | __all__ = [ 28 | instantiate.__name__, 29 | deserialize.__name__, 30 | WorkflowSpec.__name__, 31 | TaskDefaultsSpec.__name__, 32 | TaskSpec.__name__, 33 | ] 34 | -------------------------------------------------------------------------------- /tests/fixtures/pack/pristine_actions/workflows/mistral-transition-expressions.yaml: -------------------------------------------------------------------------------- 1 | version: '2.0' 2 | 3 | examples.mistral-transition-expressions: 4 | description: > 5 | Test the conversion of task transition expressions in conjunction with 6 | the publish attribute 7 | type: direct 8 | input: 9 | - tempfile 10 | - message 11 | tasks: 12 | task1: 13 | action: core.noop 14 | publish: 15 | published_variable_1: Simple string value 16 | on-success: 17 | - notify_start 18 | - yaql_transition_expression: <% $.tempfile != null %> 19 | - jinja_transition_expression: "{{ _.tempfile = null }}" 20 | on-complete: 21 | - simple_complete_task 22 | - yaql_complete_task: <% $.yaql_expression %> 23 | - jinja_complete_task: '{{ _.jinja_expression }}' 24 | notify_start: 25 | action: core.noop 26 | yaql_transition_expression: 27 | action: core.noop 28 | jinja_transition_expression: 29 | action: core.noop 30 | simple_complete_task: 31 | action: core.noop 32 | yaql_complete_task: 33 | action: core.noop 34 | jinja_complete_task: 35 | action: core.noop 36 | -------------------------------------------------------------------------------- /orquestaconvert/expressions/yaql.py: -------------------------------------------------------------------------------- 1 | # Licensed under the Apache License, Version 2.0 (the "License"); 2 | # you may not use this file except in compliance with the License. 3 | # You may obtain a copy of the License at 4 | # 5 | # http://www.apache.org/licenses/LICENSE-2.0 6 | # 7 | # Unless required by applicable law or agreed to in writing, software 8 | # distributed under the License is distributed on an "AS IS" BASIS, 9 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 10 | # See the License for the specific language governing permissions and 11 | # limitations under the License. 12 | 13 | import re 14 | 15 | from orquestaconvert.expressions import base as expr_base 16 | 17 | # <% xxx %> -> xxx 18 | UNWRAP_REGEX = r"(<%)(.*)(%>)" 19 | UNWRAP_PATTERN = re.compile(UNWRAP_REGEX) 20 | 21 | # $. -> ctx(). 22 | CONTEXT_VARS_REGEX = r"(\$\.([\w]+))" 23 | CONTEXT_VARS_PATTERN = re.compile(CONTEXT_VARS_REGEX) 24 | 25 | 26 | class YaqlExpressionConverter(expr_base.BaseExpressionConverter): 27 | 28 | @classmethod 29 | def wrap_expression(cls, expr): 30 | return "<% " + expr + " %>" 31 | 32 | @classmethod 33 | def unwrap_expression(cls, expr): 34 | return UNWRAP_PATTERN.sub(cls._replace_unwrap, expr) 35 | 36 | @classmethod 37 | def convert_context_vars(cls, expr, **kwargs): 38 | _replace_vars = cls._get_replace_vars(**kwargs) 39 | return CONTEXT_VARS_PATTERN.sub(_replace_vars, expr) 40 | -------------------------------------------------------------------------------- /orquestaconvert/expressions/jinja.py: -------------------------------------------------------------------------------- 1 | # Licensed under the Apache License, Version 2.0 (the "License"); 2 | # you may not use this file except in compliance with the License. 3 | # You may obtain a copy of the License at 4 | # 5 | # http://www.apache.org/licenses/LICENSE-2.0 6 | # 7 | # Unless required by applicable law or agreed to in writing, software 8 | # distributed under the License is distributed on an "AS IS" BASIS, 9 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 10 | # See the License for the specific language governing permissions and 11 | # limitations under the License. 12 | 13 | import re 14 | 15 | from orquestaconvert.expressions import base as expr_base 16 | 17 | # {{ xxx }} -> xxx 18 | UNWRAP_REGEX = r"({{)(.*)(}})" 19 | UNWRAP_PATTERN = re.compile(UNWRAP_REGEX) 20 | 21 | # _. -> ctx(). 22 | CONTEXT_VARS_REGEX = r"\b(_\.([\w]+))" 23 | CONTEXT_VARS_PATTERN = re.compile(CONTEXT_VARS_REGEX) 24 | 25 | 26 | class JinjaExpressionConverter(expr_base.BaseExpressionConverter): 27 | 28 | @classmethod 29 | def wrap_expression(cls, expr): 30 | return "{{ " + expr + " }}" 31 | 32 | @classmethod 33 | def unwrap_expression(cls, expr): 34 | return UNWRAP_PATTERN.sub(cls._replace_unwrap, expr) 35 | 36 | @classmethod 37 | def convert_context_vars(cls, expr, **kwargs): 38 | _replace_vars = cls._get_replace_vars(**kwargs) 39 | return CONTEXT_VARS_PATTERN.sub(_replace_vars, expr) 40 | -------------------------------------------------------------------------------- /tests/fixtures/pack/o_actions/workflows/mistral-transition-expressions.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | version: '1.0' 3 | description: > 4 | Test the conversion of task transition expressions in conjunction with 5 | the publish attribute 6 | input: 7 | - tempfile 8 | - message 9 | tasks: 10 | task1: 11 | action: core.noop 12 | next: 13 | - when: <% succeeded() %> 14 | publish: 15 | - published_variable_1: Simple string value 16 | do: 17 | - notify_start 18 | - when: <% succeeded() and (ctx().tempfile != null) %> 19 | publish: 20 | - published_variable_1: Simple string value 21 | do: 22 | - yaql_transition_expression 23 | - when: '{{ succeeded() and (ctx().tempfile = null) }}' 24 | publish: 25 | - published_variable_1: Simple string value 26 | do: 27 | - jinja_transition_expression 28 | - publish: 29 | - published_variable_1: Simple string value 30 | do: 31 | - simple_complete_task 32 | - when: <% ctx().yaql_expression %> 33 | publish: 34 | - published_variable_1: Simple string value 35 | do: 36 | - yaql_complete_task 37 | - when: '{{ ctx().jinja_expression }}' 38 | publish: 39 | - published_variable_1: Simple string value 40 | do: 41 | - jinja_complete_task 42 | notify_start: 43 | action: core.noop 44 | yaql_transition_expression: 45 | action: core.noop 46 | jinja_transition_expression: 47 | action: core.noop 48 | simple_complete_task: 49 | action: core.noop 50 | yaql_complete_task: 51 | action: core.noop 52 | jinja_complete_task: 53 | action: core.noop 54 | -------------------------------------------------------------------------------- /orquestaconvert/utils/yaml_utils.py: -------------------------------------------------------------------------------- 1 | # Licensed under the Apache License, Version 2.0 (the "License"); 2 | # you may not use this file except in compliance with the License. 3 | # You may obtain a copy of the License at 4 | # 5 | # http://www.apache.org/licenses/LICENSE-2.0 6 | # 7 | # Unless required by applicable law or agreed to in writing, software 8 | # distributed under the License is distributed on an "AS IS" BASIS, 9 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 10 | # See the License for the specific language governing permissions and 11 | # limitations under the License. 12 | 13 | import ruamel.yaml 14 | import ruamel.yaml.comments 15 | import six 16 | import yaml 17 | import yamlloader 18 | 19 | 20 | def yaml_to_obj(stream): 21 | return yaml.load(stream, Loader=yamlloader.ordereddict.CSafeLoader) 22 | 23 | 24 | def read_yaml(yaml_filename): 25 | # parse data in a format that preserves ordering 26 | with open(yaml_filename, 'r') as stream: 27 | # safe load with ordered dicts 28 | ruamel_data = ruamel.yaml.round_trip_load(stream) 29 | 30 | # parse YAML into a dict 31 | with open(yaml_filename, 'r') as stream: 32 | data = yaml_to_obj(stream) 33 | 34 | return (data, ruamel_data) 35 | 36 | 37 | def obj_to_yaml(obj, indent=2): 38 | # use this different library because PyYAML doesn't handle indenting properly 39 | # rt = round-trip 40 | ruyaml = ruamel.yaml.YAML(typ='rt') 41 | ruyaml.explicit_start = True 42 | # this crazyness basically sets indents to 'indent' 43 | # 'sequence' is always supposed to be 'offset' + 2 44 | ruyaml.indent(mapping=indent, sequence=(indent + 2), offset=indent) 45 | # prevent line-wrap 46 | ruyaml.width = 99999999999 47 | stream = six.StringIO() 48 | ruyaml.dump(obj, stream) 49 | return stream.getvalue() 50 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | .eggs/ 17 | lib/ 18 | lib64/ 19 | parts/ 20 | sdist/ 21 | var/ 22 | wheels/ 23 | *.egg-info/ 24 | .installed.cfg 25 | *.egg 26 | MANIFEST 27 | 28 | # PyInstaller 29 | # Usually these files are written by a python script from a template 30 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 31 | *.manifest 32 | *.spec 33 | 34 | # Installer logs 35 | pip-log.txt 36 | pip-delete-this-directory.txt 37 | 38 | # Unit test / coverage reports 39 | htmlcov/ 40 | .tox/ 41 | .coverage 42 | .coverage.* 43 | .cache 44 | nosetests.xml 45 | coverage.xml 46 | *.cover 47 | .hypothesis/ 48 | .pytest_cache/ 49 | 50 | # Translations 51 | *.mo 52 | *.pot 53 | 54 | # Django stuff: 55 | *.log 56 | local_settings.py 57 | db.sqlite3 58 | 59 | # Flask stuff: 60 | instance/ 61 | .webassets-cache 62 | 63 | # Scrapy stuff: 64 | .scrapy 65 | 66 | # Sphinx documentation 67 | docs/_build/ 68 | 69 | # PyBuilder 70 | target/ 71 | 72 | # Jupyter Notebook 73 | .ipynb_checkpoints 74 | 75 | # pyenv 76 | .python-version 77 | 78 | # celery beat schedule file 79 | celerybeat-schedule 80 | 81 | # SageMath parsed files 82 | *.sage.py 83 | 84 | # Environments 85 | .env 86 | .venv 87 | env/ 88 | venv/ 89 | ENV/ 90 | env.bak/ 91 | venv.bak/ 92 | 93 | # Spyder project settings 94 | .spyderproject 95 | .spyproject 96 | 97 | # Rope project settings 98 | .ropeproject 99 | 100 | # mkdocs documentation 101 | /site 102 | 103 | # mypy 104 | .mypy_cache/ 105 | 106 | # virtualenv 107 | virtualenv/ 108 | 109 | # ignore HTML test coverage results 110 | cover/ -------------------------------------------------------------------------------- /orquestaconvert/specs/mistral/v2/base.py: -------------------------------------------------------------------------------- 1 | # Copyright 2019 Extreme Networks, Inc. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | import logging 16 | 17 | from orquesta.specs import base as spec_base 18 | 19 | 20 | LOG = logging.getLogger(__name__) 21 | 22 | 23 | class Spec(spec_base.Spec): 24 | _catalog = "mistral" 25 | 26 | _version = "2.0" 27 | 28 | _meta_schema = { 29 | "type": "object", 30 | "properties": {"version": {"enum": [_version, float(_version)]}}, 31 | } 32 | 33 | def get_spec_path(self, prop_name, parent=None): 34 | if parent: 35 | return parent.get("spec_path") + "." + prop_name 36 | elif not parent and self.name: 37 | return self.name + "." + prop_name 38 | else: 39 | return prop_name 40 | 41 | 42 | class MappingSpec(spec_base.MappingSpec): 43 | _catalog = "mistral" 44 | 45 | _version = "2.0" 46 | 47 | _meta_schema = { 48 | "type": "object", 49 | "properties": {"version": {"enum": [_version, float(_version)]}}, 50 | } 51 | 52 | 53 | class SequenceSpec(spec_base.SequenceSpec): 54 | _catalog = "mistral" 55 | 56 | _version = "2.0" 57 | 58 | _meta_schema = { 59 | "type": "object", 60 | "properties": {"version": {"enum": [_version, float(_version)]}}, 61 | } 62 | -------------------------------------------------------------------------------- /lint-configs/python/.pylintrc: -------------------------------------------------------------------------------- 1 | [MESSAGES CONTROL] 2 | # C0111 Missing docstring 3 | # I0011 Warning locally suppressed using disable-msg 4 | # I0012 Warning locally suppressed using disable-msg 5 | # W0704 Except doesn't do anything Used when an except clause does nothing but "pass" and there is no "else" clause 6 | # W0142 Used * or * magic* Used when a function or method is called using *args or **kwargs to dispatch arguments. 7 | # W0212 Access to a protected member %s of a client class 8 | # W0232 Class has no __init__ method Used when a class has no __init__ method, neither its parent classes. 9 | # W0613 Unused argument %r Used when a function or method argument is not used. 10 | # W0702 No exception's type specified Used when an except clause doesn't specify exceptions type to catch. 11 | # R0201 Method could be a function 12 | # W0614 Unused import XYZ from wildcard import 13 | # R0914 Too many local variables 14 | # R0912 Too many branches 15 | # R0915 Too many statements 16 | # R0913 Too many arguments 17 | # R0904 Too many public methods 18 | # E0211: Method has no argument 19 | # E1128: Assigning to function call which only returns None Used when an assignment is done on a function call but the inferred function returns nothing but None. 20 | # E1129: Context manager ‘%s’ doesn’t implement __enter__ and __exit__. Used when an instance in a with statement doesn’t implement the context manager protocol(__enter__/__exit__). 21 | disable=C0103,C0111,I0011,I0012,W0704,W0142,W0212,W0232,W0613,W0702,R0201,W0614,R0914,R0912,R0915,R0913,R0904,R0801,not-context-manager,assignment-from-none 22 | 23 | [TYPECHECK] 24 | # Note: This modules are manipulated during the runtime so we can't detect all the properties during 25 | # static analysis 26 | ignored-modules=distutils,eventlet.green.subprocess,six,six.moves 27 | 28 | [FORMAT] 29 | max-line-length=100 30 | max-module-lines=1000 31 | indent-string=' ' 32 | 33 | [REPORTS] 34 | output-format=parseable -------------------------------------------------------------------------------- /tests/fixtures/pack/o_actions/workflows/mistral-publish-only-transitions.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | version: '1.0' 3 | description: Test the conversion of publish-only task transitions 4 | input: 5 | - tempfile 6 | - message 7 | output: 8 | - init_result_yaql: <% ctx().used_init_result %> 9 | - init_result_jinja: '{{ ctx().used_init_result }}' 10 | - used_simple_result: <% ctx().used_simple_result %> 11 | - used_yaql_result: <% ctx().used_yaql_result %> 12 | - used_jinja_result: '{{ ctx().used_jinja_result }}' 13 | tasks: 14 | init: 15 | action: core.noop 16 | next: 17 | - when: <% succeeded() %> 18 | publish: 19 | - used_init_result: SUCCESS! 20 | do: 21 | - simple_task_transition 22 | - simple_task_transition_unused_publish 23 | - when: <% succeeded() and (ctx().yaql_expression) %> 24 | publish: 25 | - used_init_result: SUCCESS! 26 | do: 27 | - yaql_task 28 | - yaql_task_unused_publish 29 | - when: '{{ succeeded() and (ctx().jinja_expression) }}' 30 | publish: 31 | - used_init_result: SUCCESS! 32 | do: 33 | - jinja_task 34 | - jinja_task_unused_publish 35 | simple_task_transition: 36 | action: core.noop 37 | next: 38 | - when: <% succeeded() %> 39 | publish: 40 | - used_simple_result: This simple result should be in the converted workflow 41 | simple_task_transition_unused_publish: 42 | action: core.noop 43 | yaql_task: 44 | action: core.noop 45 | next: 46 | - when: <% succeeded() %> 47 | publish: 48 | - used_yaql_result: This YAQL result should be converted 49 | yaql_task_unused_publish: 50 | action: core.noop 51 | jinja_task: 52 | action: core.noop 53 | next: 54 | - when: <% succeeded() %> 55 | publish: 56 | - used_jinja_result: This Jinja result should be converted 57 | jinja_task_unused_publish: 58 | action: core.noop 59 | -------------------------------------------------------------------------------- /tests/fixtures/pack/pristine_actions/workflows/mistral-publish-only-transitions.yaml: -------------------------------------------------------------------------------- 1 | version: '2.0' 2 | 3 | examples.mistral-publish-only-transitions: 4 | description: Test the conversion of publish-only task transitions 5 | type: direct 6 | input: 7 | - tempfile 8 | - message 9 | output: 10 | init_result_yaql: <% $.used_init_result %> 11 | init_result_jinja: '{{ _.used_init_result }}' 12 | used_simple_result: <% $.used_simple_result %> 13 | used_yaql_result: <% $.used_yaql_result %> 14 | used_jinja_result: '{{ _.used_jinja_result }}' 15 | tasks: 16 | init: 17 | action: core.noop 18 | publish: 19 | used_init_result: SUCCESS! 20 | on-success: 21 | - simple_task_transition 22 | - simple_task_transition_unused_publish 23 | - yaql_task: <% $.yaql_expression %> 24 | - yaql_task_unused_publish: <% $.yaql_expression %> 25 | - jinja_task: '{{ _.jinja_expression }}' 26 | - jinja_task_unused_publish: '{{ _.jinja_expression }}' 27 | 28 | simple_task_transition: 29 | action: core.noop 30 | publish: 31 | used_simple_result: This simple result should be in the converted workflow 32 | simple_task_transition_unused_publish: 33 | action: core.noop 34 | publish: 35 | unused_simple_result: This simple result should be removed 36 | 37 | yaql_task: 38 | action: core.noop 39 | publish: 40 | used_yaql_result: This YAQL result should be converted 41 | yaql_task_unused_publish: 42 | action: core.noop 43 | publish: 44 | unused_yaql_result: This YAQL result should be removed 45 | 46 | jinja_task: 47 | action: core.noop 48 | publish: 49 | used_jinja_result: This Jinja result should be converted 50 | jinja_task_unused_publish: 51 | action: core.noop 52 | publish: 53 | unused_jinja_result: This should also be removed 54 | -------------------------------------------------------------------------------- /orquestaconvert/specs/mistral/v2/policies.py: -------------------------------------------------------------------------------- 1 | # Copyright 2019 Extreme Networks, Inc. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | import logging 16 | 17 | from orquesta.specs import types as spec_types 18 | from orquestaconvert.specs.mistral.v2 import base as mistral_v2_spec_base 19 | 20 | 21 | LOG = logging.getLogger(__name__) 22 | 23 | 24 | KEEP_RESULT_SCHEMA = spec_types.STRING_OR_BOOLEAN 25 | WAIT_BEFORE_SCHEMA = spec_types.STRING_OR_POSITIVE_INTEGER 26 | WAIT_AFTER_SCHEMA = spec_types.STRING_OR_POSITIVE_INTEGER 27 | TIMEOUT_SCHEMA = spec_types.STRING_OR_POSITIVE_INTEGER 28 | PAUSE_BEFORE_SCHEMA = spec_types.STRING_OR_BOOLEAN 29 | CONCURRENCY_SCHEMA = spec_types.STRING_OR_POSITIVE_INTEGER 30 | SAFE_RERUN_SCHEMA = spec_types.STRING_OR_BOOLEAN 31 | TARGET_SCHEMA = spec_types.NONEMPTY_STRING 32 | 33 | 34 | class RetrySpec(mistral_v2_spec_base.Spec): 35 | _schema = { 36 | "type": "object", 37 | "properties": { 38 | "count": spec_types.STRING_OR_POSITIVE_INTEGER, 39 | "break-on": spec_types.NONEMPTY_STRING, 40 | "continue-on": spec_types.NONEMPTY_STRING, 41 | "delay": spec_types.STRING_OR_POSITIVE_INTEGER, 42 | }, 43 | "required": ["delay", "count"], 44 | "additionalProperties": False, 45 | } 46 | 47 | 48 | class PoliciesSpec(mistral_v2_spec_base.Spec): 49 | _schema = { 50 | "type": "object", 51 | "properties": { 52 | "retry": RetrySpec, 53 | "wait-before": WAIT_BEFORE_SCHEMA, 54 | "wait-after": WAIT_AFTER_SCHEMA, 55 | "timeout": TIMEOUT_SCHEMA, 56 | "pause-before": PAUSE_BEFORE_SCHEMA, 57 | "concurrency": CONCURRENCY_SCHEMA, 58 | }, 59 | "additionalProperties": False, 60 | } 61 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | from __future__ import absolute_import 16 | 17 | import dist_utils 18 | import os.path 19 | import setuptools 20 | 21 | MODULE_NAME = 'orquestaconvert' 22 | BASE_DIR = os.path.dirname(os.path.abspath(__file__)) 23 | REQUIREMENTS_FILE = os.path.join(BASE_DIR, 'requirements.txt') 24 | INIT_FILE = os.path.join(BASE_DIR, MODULE_NAME + '/__init__.py') 25 | 26 | install_reqs, dep_links = dist_utils.fetch_requirements(REQUIREMENTS_FILE) 27 | 28 | with open("README.md", "r") as fh: 29 | long_description = fh.read() 30 | 31 | dist_utils.apply_vagrant_workaround() 32 | 33 | setuptools.setup( 34 | name=MODULE_NAME, 35 | version=dist_utils.get_version_string(INIT_FILE), 36 | description='Tool to convert OpenStack Mistral workflows to StackStorm Orquesta workflows', 37 | long_description=long_description, 38 | long_description_content_type="text/markdown", 39 | author='StackStorm', 40 | author_email='info@stackstorm.com', 41 | url='https://stackstorm.com/', 42 | install_requires=install_reqs, 43 | dependency_links=dep_links, 44 | test_suite=MODULE_NAME, 45 | zip_safe=False, 46 | include_package_data=True, 47 | packages=setuptools.find_packages(exclude=['setuptools', 'tests']), 48 | scripts=[ 49 | 'bin/orquestaconvert.sh', 50 | 'bin/orquestaconvert-pack.sh', 51 | ], 52 | classifiers=[ 53 | 'Development Status :: 3 - Alpha', 54 | 'License :: OSI Approved :: Apache Software License', 55 | 'Programming Language :: Python', 56 | 'Programming Language :: Python :: 3', 57 | 'Programming Language :: Python :: 3.5', 58 | 'Programming Language :: Python :: 3.6', 59 | 'Programming Language :: Python :: 3.7', 60 | 'Programming Language :: Python :: 3.8', 61 | 'Intended Audience :: Developers', 62 | 'Environment :: Console', 63 | ], 64 | ) 65 | -------------------------------------------------------------------------------- /tests/unit/test_client.py: -------------------------------------------------------------------------------- 1 | # Licensed under the Apache License, Version 2.0 (the "License"); 2 | # you may not use this file except in compliance with the License. 3 | # You may obtain a copy of the License at 4 | # 5 | # http://www.apache.org/licenses/LICENSE-2.0 6 | # 7 | # Unless required by applicable law or agreed to in writing, software 8 | # distributed under the License is distributed on an "AS IS" BASIS, 9 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 10 | # See the License for the specific language governing permissions and 11 | # limitations under the License. 12 | 13 | import mock 14 | 15 | from orquestaconvert import client 16 | 17 | from tests import base_test_case 18 | 19 | 20 | class TestClient(base_test_case.BaseCLITestCase): 21 | __test__ = True 22 | 23 | def setUp(self): 24 | super(TestClient, self).setUp() 25 | self.client = client.Client() 26 | self.maxDiff = 20000 27 | 28 | def _validate_args(self, args, filename, expected_filename=None): 29 | # path to fixture file 30 | fixture_path = self.get_fixture_path('mistral/' + filename) 31 | 32 | # run 33 | exit_status = client.Client().run(args + [fixture_path], self.stdout) 34 | self.assertEqual(exit_status, 0) 35 | 36 | # read expected data 37 | if not expected_filename: 38 | expected_filename = filename 39 | expected = self.get_fixture_content('orquesta/' + expected_filename) 40 | 41 | # compare 42 | self.stdout.seek(0) 43 | stdout = self.stdout.read() 44 | self.assertMultiLineEqual(stdout, expected) 45 | 46 | def test_run(self): 47 | self._validate_args([], 'nasa_apod_twitter_post.yaml') 48 | 49 | def test_run_jinja(self): 50 | self._validate_args(['-e', 'jinja'], 'nasa_apod_twitter_post.yaml') 51 | 52 | def test_run_yaql(self): 53 | self._validate_args(['-e', 'yaql'], 54 | 'nasa_apod_twitter_post_yaql.yaml', 55 | 'nasa_apod_twitter_post_yaql.yaml') 56 | 57 | def test_run_expression_jinja(self): 58 | self._validate_args(['--expression', 'jinja'], 'nasa_apod_twitter_post.yaml') 59 | 60 | def test_run_expression_yaql(self): 61 | self._validate_args(['--expression', 'yaql'], 62 | 'nasa_apod_twitter_post_yaql.yaml', 63 | 'nasa_apod_twitter_post_yaql.yaml') 64 | 65 | def test_validate_workflow_spec_raises(self): 66 | wf_spec = mock.MagicMock() 67 | wf_spec.inspect_syntax.return_value = "some error string" 68 | 69 | with self.assertRaises(ValueError): 70 | self.client.validate_workflow_spec(wf_spec) 71 | -------------------------------------------------------------------------------- /tests/unit/test_utils_yaml_utils.py: -------------------------------------------------------------------------------- 1 | # Licensed under the Apache License, Version 2.0 (the "License"); 2 | # you may not use this file except in compliance with the License. 3 | # You may obtain a copy of the License at 4 | # 5 | # http://www.apache.org/licenses/LICENSE-2.0 6 | # 7 | # Unless required by applicable law or agreed to in writing, software 8 | # distributed under the License is distributed on an "AS IS" BASIS, 9 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 10 | # See the License for the specific language governing permissions and 11 | # limitations under the License. 12 | 13 | import ruamel.yaml 14 | 15 | from orquestaconvert.utils import yaml_utils 16 | 17 | from tests import base_test_case 18 | 19 | OrderedMap = ruamel.yaml.comments.CommentedMap 20 | 21 | 22 | class TestYamlUtils(base_test_case.BaseTestCase): 23 | __test__ = True 24 | 25 | def test_yaml_to_obj(self): 26 | yaml_str = ("test_dict:\n" 27 | " a: b\n" 28 | "\n" 29 | "test_list:\n" 30 | " - x\n" 31 | " - y\n") 32 | result = yaml_utils.yaml_to_obj(yaml_str) 33 | self.assertEqual(result, OrderedMap([ 34 | ('test_dict', OrderedMap([ 35 | ('a', 'b') 36 | ])), 37 | ('test_list', ['x', 'y']) 38 | ])) 39 | 40 | def test_read_yaml(self): 41 | fixture_path = self.get_fixture_path('yaml/simple.yaml') 42 | data, ruamel_data = yaml_utils.read_yaml(fixture_path) 43 | self.assertEqual(data, {'test_dict': {'a': True}}) 44 | self.assertEqual(data, OrderedMap([ 45 | ('test_dict', OrderedMap([ 46 | ('a', True) 47 | ])) 48 | ])) 49 | 50 | def test_obj_to_yaml(self): 51 | ruamel_data = OrderedMap([ 52 | ('test_dict', OrderedMap([ 53 | ('a', True) 54 | ])) 55 | ]) 56 | result = yaml_utils.obj_to_yaml(ruamel_data) 57 | self.assertEqual(result, self.get_fixture_content('yaml/simple.yaml')) 58 | 59 | def test_obj_to_yaml_dict(self): 60 | ruamel_data = {'test_dict': {'a': True}} 61 | result = yaml_utils.obj_to_yaml(ruamel_data) 62 | self.assertEqual(result, self.get_fixture_content('yaml/simple.yaml')) 63 | 64 | def test_obj_to_yaml_no_line_wrap(self): 65 | ruamel_data = { 66 | 'key': ("this is some super super super long line that would normally cause" 67 | " a line wrap when converting from dict to yaml. instead we don't want" 68 | " to do that and just keep this one crazy long line.") 69 | } 70 | result = yaml_utils.obj_to_yaml(ruamel_data) 71 | self.assertEqual(result, self.get_fixture_content('yaml/no_line_wrap.yaml')) 72 | -------------------------------------------------------------------------------- /tests/unit/test_expressions_base.py: -------------------------------------------------------------------------------- 1 | # Licensed under the Apache License, Version 2.0 (the "License"); 2 | # you may not use this file except in compliance with the License. 3 | # You may obtain a copy of the License at 4 | # 5 | # http://www.apache.org/licenses/LICENSE-2.0 6 | # 7 | # Unless required by applicable law or agreed to in writing, software 8 | # distributed under the License is distributed on an "AS IS" BASIS, 9 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 10 | # See the License for the specific language governing permissions and 11 | # limitations under the License. 12 | 13 | from orquestaconvert.expressions import base as expr_base 14 | 15 | from tests import base_test_case 16 | 17 | 18 | class AbstractBaseExpressionsTestCase(base_test_case.BaseTestCase): 19 | __test__ = True 20 | 21 | def test_wrap_expression_raises(self): 22 | with self.assertRaises(NotImplementedError): 23 | expr_base.AbstractBaseExpressionConverter.wrap_expression('junk') 24 | 25 | def test_unwrap_expression_raises(self): 26 | with self.assertRaises(NotImplementedError): 27 | expr_base.AbstractBaseExpressionConverter.unwrap_expression('junk') 28 | 29 | def test_convert_string_raises(self): 30 | with self.assertRaises(NotImplementedError): 31 | expr_base.AbstractBaseExpressionConverter.convert_string('junk') 32 | 33 | def test_convert_context_vars_raises(self): 34 | with self.assertRaises(NotImplementedError): 35 | expr_base.AbstractBaseExpressionConverter.convert_context_vars('junk') 36 | 37 | def test_convert_task_result_raises(self): 38 | with self.assertRaises(NotImplementedError): 39 | expr_base.AbstractBaseExpressionConverter.convert_task_result('junk') 40 | 41 | def test_convert_st2kv(self): 42 | with self.assertRaises(NotImplementedError): 43 | expr_base.AbstractBaseExpressionConverter.convert_st2kv('junk') 44 | 45 | def test_convert_st2_execution_id(self): 46 | with self.assertRaises(NotImplementedError): 47 | expr_base.AbstractBaseExpressionConverter.convert_st2_execution_id('junk') 48 | 49 | def test_convert_st2_api_url(self): 50 | with self.assertRaises(NotImplementedError): 51 | expr_base.AbstractBaseExpressionConverter.convert_st2_api_url('junk') 52 | 53 | 54 | class BaseExpressionsTestCase(base_test_case.BaseTestCase): 55 | __test__ = True 56 | 57 | def test_wrap_expression_raises(self): 58 | with self.assertRaises(NotImplementedError): 59 | expr_base.BaseExpressionConverter.wrap_expression('junk') 60 | 61 | def test_unwrap_expression_raises(self): 62 | with self.assertRaises(NotImplementedError): 63 | expr_base.BaseExpressionConverter.unwrap_expression('junk') 64 | 65 | def test_convert_static_string(self): 66 | with self.assertRaises(NotImplementedError): 67 | expr_base.BaseExpressionConverter.convert_string('junk') 68 | 69 | def test_convert_static_context_vars_raises(self): 70 | with self.assertRaises(NotImplementedError): 71 | expr_base.BaseExpressionConverter.convert_context_vars('junk') 72 | -------------------------------------------------------------------------------- /tests/fixtures/orquesta/immediately_referenced_context_variables.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | version: '1.0' 3 | description: Test 4 | input: 5 | - uuid 6 | tasks: 7 | task_1: 8 | action: pack.action_1 9 | input: 10 | foreign_id: <% ctx().uuid %> 11 | next: 12 | - when: <% succeeded() and (result().result['operations'] + 1) %> 13 | publish: 14 | - operations: <% result().result['operations'] + 1 %> 15 | - namespace: <% result().result['namespace'] %> 16 | do: 17 | - task_1 18 | - when: <% succeeded() and (len(result().result['operations'] + 1) = "len function") %> 19 | publish: 20 | - operations: <% result().result['operations'] + 1 %> 21 | - namespace: <% result().result['namespace'] %> 22 | do: 23 | - task_2 24 | - when: <% succeeded() and ((result().result['operations'] + 1) * 4 > 0) %> 25 | publish: 26 | - operations: <% result().result['operations'] + 1 %> 27 | - namespace: <% result().result['namespace'] %> 28 | do: 29 | - task_3 30 | - when: <% succeeded() and (0 < 4 * (result().result['operations'] + 1)) %> 31 | publish: 32 | - operations: <% result().result['operations'] + 1 %> 33 | - namespace: <% result().result['namespace'] %> 34 | do: 35 | - task_3 36 | - when: <% succeeded() and ((result().result['operations'] + 1)|length) %> 37 | publish: 38 | - operations: <% result().result['operations'] + 1 %> 39 | - namespace: <% result().result['namespace'] %> 40 | do: 41 | - task_4 42 | - when: <% succeeded() and (5 + (result().result['operations'] + 1) * 4) %> 43 | publish: 44 | - operations: <% result().result['operations'] + 1 %> 45 | - namespace: <% result().result['namespace'] %> 46 | do: 47 | - task_5 48 | - when: <% succeeded() and (result().result['operations'] + 1) %> 49 | publish: 50 | - operations: <% result().result['operations'] + 1 %> 51 | - namespace: <% result().result['namespace'] %> 52 | do: 53 | - task_6 54 | - when: '{{ succeeded() and (ctx().operations) }}' 55 | publish: 56 | - operations: <% result().result['operations'] + 1 %> 57 | - namespace: <% result().result['namespace'] %> 58 | do: 59 | - task_7 60 | - when: <% succeeded() and (asdfctx(operations)) %> 61 | publish: 62 | - operations: <% result().result['operations'] + 1 %> 63 | - namespace: <% result().result['namespace'] %> 64 | do: 65 | - task_8 66 | - when: <% succeeded() and (ctx().operations_NOT) %> 67 | publish: 68 | - operations: <% result().result['operations'] + 1 %> 69 | - namespace: <% result().result['namespace'] %> 70 | do: 71 | - task_9 72 | task_2: 73 | action: core.noop 74 | task_3: 75 | action: core.noop 76 | task_4: 77 | action: core.noop 78 | task_5: 79 | action: core.noop 80 | task_6: 81 | action: core.noop 82 | task_7: 83 | action: core.noop 84 | task_8: 85 | action: core.noop 86 | -------------------------------------------------------------------------------- /tests/unit/test_pack_client.py: -------------------------------------------------------------------------------- 1 | # Licensed under the Apache License, Version 2.0 (the "License"); 2 | # you may not use this file except in compliance with the License. 3 | # You may obtain a copy of the License at 4 | # 5 | # http://www.apache.org/licenses/LICENSE-2.0 6 | # 7 | # Unless required by applicable law or agreed to in writing, software 8 | # distributed under the License is distributed on an "AS IS" BASIS, 9 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 10 | # See the License for the specific language governing permissions and 11 | # limitations under the License. 12 | 13 | from __future__ import print_function 14 | 15 | import mock 16 | 17 | from orquestaconvert import pack_client 18 | 19 | from tests import base_test_case 20 | 21 | 22 | class PackClientTestCase(base_test_case.BasePackClientRunTestCase): 23 | __test__ = True 24 | 25 | def setUp(self): 26 | super(PackClientTestCase, self).setUp() 27 | 28 | self.client = mock.MagicMock() 29 | self.client.run = mock.MagicMock(return_value=0) 30 | self.pack_client = pack_client.PackClient() 31 | 32 | def test_get_mistral_workflow_files_in_p_dir(self): 33 | workflows = self.pack_client.get_workflow_files('mistral-v2', self.p_actions_dir) 34 | self.assertEqual(workflows, self.pristine_action_wfs) 35 | 36 | def test_get_orquesta_workflow_files_in_p_dir(self): 37 | workflows = self.pack_client.get_workflow_files('orquesta', self.p_actions_dir) 38 | self.assertEqual(workflows, {}) 39 | 40 | def test_get_orquesta_workflow_files_in_o_dir(self): 41 | workflows = self.pack_client.get_workflow_files('orquesta', self.o_actions_dir) 42 | self.assertEqual(self.orquesta_action_wfs, workflows) 43 | 44 | def test_validate_nothing(self): 45 | args = ['--validate', '--actions-dir={}'.format(self.m_actions_dir)] 46 | result = self.pack_client.run(args, self.stdout, client=self.client) 47 | 48 | self.assertEqual(result, 0) 49 | 50 | self.assertEqual(self.client.run.call_count, 0) 51 | 52 | def test_validate_orquesta(self): 53 | args = ['--validate', '--actions-dir={}'.format(self.o_actions_dir)] 54 | result = self.pack_client.run(args, self.stdout, client=self.client) 55 | 56 | self.assertEqual(result, 0) 57 | 58 | self.assertEqual(self.client.run.call_count, len(self.action_passing_files)) 59 | 60 | calls = [ 61 | mock.call(['--validate', wf], self.stdout) 62 | for wf in self.orquesta_action_wfs.values() 63 | ] 64 | self.client.run.assert_has_calls(calls, any_order=True) 65 | 66 | def test_convert_pack(self): 67 | args = ['-e', 'yaql', '--actions-dir={}'.format(self.m_actions_dir)] 68 | result = self.pack_client.run(args, self.stdout, client=self.client) 69 | 70 | self.assertEqual(result, 0) 71 | 72 | self.assertEqual(self.client.run.call_count, len(self.action_files)) 73 | 74 | expected = [ 75 | ['-e', 'yaql', wf] 76 | for wf in self.action_wfs.values() 77 | ] 78 | self.assertItemsEqual([call_args[0][0] for call_args in self.client.run.call_args_list], 79 | expected) 80 | -------------------------------------------------------------------------------- /tests/integration/test_end_to_end.py: -------------------------------------------------------------------------------- 1 | # Licensed under the Apache License, Version 2.0 (the "License"); 2 | # you may not use this file except in compliance with the License. 3 | # You may obtain a copy of the License at 4 | # 5 | # http://www.apache.org/licenses/LICENSE-2.0 6 | # 7 | # Unless required by applicable law or agreed to in writing, software 8 | # distributed under the License is distributed on an "AS IS" BASIS, 9 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 10 | # See the License for the specific language governing permissions and 11 | # limitations under the License. 12 | 13 | import six 14 | import warnings 15 | 16 | from orquestaconvert import client 17 | 18 | from tests import base_test_case 19 | 20 | 21 | class TestEndToEnd(base_test_case.BaseTestCase): 22 | __test__ = True 23 | 24 | def setUp(self): 25 | super(TestEndToEnd, self).setUp() 26 | self.maxDiff = 20000 27 | self.client = client.Client() 28 | parser = self.client.parser() 29 | # Ensure that the client has an args attribute 30 | self.client.args = parser.parse_args(['file.yaml']) 31 | 32 | def e2e_from_file(self, m_wf_filename, o_wf_filename=None): 33 | if o_wf_filename is None: 34 | o_wf_filename = m_wf_filename 35 | fixture_path = self.get_fixture_path('mistral/' + m_wf_filename) 36 | result = self.client.convert_file(fixture_path) 37 | expected = self.get_fixture_content('orquesta/' + o_wf_filename) 38 | self.assertMultiLineEqual(result, expected) 39 | 40 | def test_e2e_force_option(self): 41 | parser = self.client.parser() 42 | self.client.args = parser.parse_args(['--force', 'file.yaml']) 43 | self.e2e_from_file('unsupported_attributes.yaml', 44 | '../broken/unsupported_attributes.yaml') 45 | 46 | def test_e2e_nasa_apod_twitter_post(self): 47 | self.e2e_from_file('nasa_apod_twitter_post.yaml') 48 | 49 | def test_e2e_emptywee_test(self): 50 | with self.assertRaises(NotImplementedError): 51 | self.e2e_from_file('emptywee_test.yaml') 52 | 53 | def test_e2e_output_test(self): 54 | self.e2e_from_file('output_test.yaml') 55 | 56 | def test_e2e_transition_strings_test(self): 57 | self.e2e_from_file('transition_strings.yaml') 58 | 59 | def test_e2e_boolean_publish_parameters(self): 60 | self.e2e_from_file('boolean_publish_parameters.yaml') 61 | 62 | def test_e2e_null_publish_parameters(self): 63 | self.e2e_from_file('null_publish_parameters.yaml') 64 | 65 | def test_e2e_int_publish_parameters(self): 66 | self.e2e_from_file('int_publish_parameters.yaml') 67 | 68 | def test_e2e_convert_dashes_in_task_names_to_underscores(self): 69 | self.e2e_from_file('dashes_in_task_names.yaml') 70 | 71 | def test_e2e_immediately_referenced_context_variables(self): 72 | with warnings.catch_warnings(record=True) as ws: 73 | self.e2e_from_file('immediately_referenced_context_variables.yaml') 74 | 75 | self.assertGreater(len(ws), 0) 76 | expected_warning = ( 77 | "The transition \"{{ succeeded() and (ctx().operations) }}\" " 78 | "in task_1 references the 'operations' context variable, which " 79 | "is published in the same transition. You will need to " 80 | "manually convert the operations expression in the transition.") 81 | if six.PY2: 82 | self.assertEqual(expected_warning, ws[0].message.message) 83 | else: 84 | self.assertEqual(expected_warning, str(ws[0].message)) 85 | -------------------------------------------------------------------------------- /tests/unit/test_expressions_mixed.py: -------------------------------------------------------------------------------- 1 | # Licensed under the Apache License, Version 2.0 (the "License"); 2 | # you may not use this file except in compliance with the License. 3 | # You may obtain a copy of the License at 4 | # 5 | # http://www.apache.org/licenses/LICENSE-2.0 6 | # 7 | # Unless required by applicable law or agreed to in writing, software 8 | # distributed under the License is distributed on an "AS IS" BASIS, 9 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 10 | # See the License for the specific language governing permissions and 11 | # limitations under the License. 12 | 13 | import re 14 | 15 | from orquestaconvert.expressions import mixed 16 | 17 | from tests import base_test_case 18 | 19 | 20 | class TestExpressionsMixed(base_test_case.BaseTestCase): 21 | __test__ = True 22 | 23 | def test_convert_string(self): 24 | s = "ABC {{ _.test }}" 25 | result = mixed.MixedExpressionConverter.convert(s) 26 | self.assertEqual(result, "ABC {{ ctx().test }}") 27 | 28 | def test_convert_string_items(self): 29 | s = "i in {{ _.test1 }}<% $.test2 %>{{ _.test3 }}<% $.test4 %>" 30 | result = mixed.MixedExpressionConverter.convert(s, item_vars=['test1', 'test2']) 31 | expected = "i in {{ item(test1) }}<% item(test2) %>{{ ctx().test3 }}<% ctx().test4 %>" 32 | self.assertEqual(result, expected) 33 | 34 | def test_convert_dict(self): 35 | data = { 36 | "test1": "FOO {{ _.bar }}", 37 | "test2": "FOOBAR <% $.baz %>", 38 | } 39 | expected = { 40 | "test1": "FOO {{ ctx().bar }}", 41 | "test2": "FOOBAR <% ctx().baz %>", 42 | } 43 | result = mixed.MixedExpressionConverter.convert(data) 44 | self.assertEqual(expected, result) 45 | 46 | def test_convert_list(self): 47 | data = [ 48 | "FOO {{ _.bar }}", 49 | "FOOBAR <% $.baz %>", 50 | ] 51 | expected = [ 52 | "FOO {{ ctx().bar }}", 53 | "FOOBAR <% ctx().baz %>", 54 | ] 55 | result = mixed.MixedExpressionConverter.convert(data) 56 | self.assertEqual(expected, result) 57 | 58 | def test_convert_dict_of_lists_of_dicts(self): 59 | data = { 60 | "list1": [ 61 | { 62 | "key1": "FOO1 {{ _.bar }}", 63 | "key2": "FOO2 {{ _.baz }}", 64 | }, 65 | { 66 | "key3": "FOO3 <% $.car %>", 67 | "key4": "FOO4 <% $.caz %>", 68 | }, 69 | ], 70 | "int1": 1, 71 | "bool1": False, 72 | "null1": None, 73 | } 74 | expected = { 75 | "list1": [ 76 | { 77 | "key1": "FOO1 {{ ctx().bar }}", 78 | "key2": "FOO2 {{ ctx().baz }}", 79 | }, 80 | { 81 | "key3": "FOO3 <% ctx().car %>", 82 | "key4": "FOO4 <% ctx().caz %>", 83 | }, 84 | ], 85 | "int1": 1, 86 | "bool1": False, 87 | "null1": None, 88 | } 89 | result = mixed.MixedExpressionConverter.convert(data) 90 | self.assertItemsEqual(expected.keys(), result.keys()) 91 | self.assertItemsEqual(expected['list1'][0].keys(), result['list1'][0].keys()) 92 | self.assertItemsEqual(expected['list1'][0].values(), result['list1'][0].values()) 93 | self.assertEqual(expected['int1'], result['int1']) 94 | self.assertEqual(expected['bool1'], result['bool1']) 95 | self.assertEqual(expected['null1'], result['null1']) 96 | 97 | def test_convert_expr_obj_raises_warning(self): 98 | expr_obj = object() 99 | expected_warning_regex = re.compile( 100 | r"Could not recognize expression ''; " 101 | r"results may not be accurate.") 102 | with self.assertWarnsRegex(SyntaxWarning, expected_warning_regex): 103 | result = mixed.MixedExpressionConverter.convert(expr_obj) 104 | self.assertEqual(result, expr_obj) 105 | -------------------------------------------------------------------------------- /orquestaconvert/specs/mistral/v2/workflows.py: -------------------------------------------------------------------------------- 1 | # Copyright 2019 Extreme Networks, Inc. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | import logging 16 | import six 17 | 18 | from orquesta import exceptions as exc 19 | from orquesta.expressions import base as expr_base 20 | from orquesta.specs import types as spec_types 21 | from orquesta.utils import dictionary as dict_util 22 | from orquestaconvert.specs.mistral.v2 import base as mistral_spec_base 23 | from orquestaconvert.specs.mistral.v2 import tasks as task_models 24 | from orquestaconvert.specs.mistral.v2 import types as mistral_spec_types 25 | 26 | 27 | LOG = logging.getLogger(__name__) 28 | 29 | 30 | def instantiate(definition): 31 | definition.pop("version", None) 32 | 33 | if len(definition.keys()) > 1: 34 | raise ValueError("Workflow definition contains more than one workflow.") 35 | 36 | wf_name, wf_spec = list(definition.items())[0] 37 | 38 | return WorkflowSpec(wf_spec, name=wf_name) 39 | 40 | 41 | def deserialize(data): 42 | return WorkflowSpec.deserialize(data) 43 | 44 | 45 | class WorkflowSpec(mistral_spec_base.Spec): 46 | _schema = { 47 | "type": "object", 48 | "properties": { 49 | "type": mistral_spec_types.WORKFLOW_TYPE, 50 | "vars": spec_types.NONEMPTY_DICT, 51 | "input": spec_types.UNIQUE_STRING_OR_ONE_KEY_DICT_LIST, 52 | "output": spec_types.NONEMPTY_DICT, 53 | "output-on-error": spec_types.NONEMPTY_DICT, 54 | "task-defaults": task_models.TaskDefaultsSpec, 55 | "tasks": task_models.TaskMappingSpec, 56 | }, 57 | "required": ["tasks"], 58 | "additionalProperties": False, 59 | } 60 | 61 | _context_evaluation_sequence = ["input", "vars", "tasks", "output"] 62 | 63 | _context_inputs = ["input", "vars"] 64 | 65 | def render_input(self, runtime_inputs, in_ctx=None): 66 | input_specs = getattr(self, "input") or [] 67 | default_inputs = dict([list(i.items())[0] for i in input_specs if isinstance(i, dict)]) 68 | merged_inputs = dict_util.merge_dicts(default_inputs, runtime_inputs, True) 69 | rendered_inputs = {} 70 | errors = [] 71 | 72 | try: 73 | rendered_inputs = expr_base.evaluate(merged_inputs, {}) 74 | except exc.ExpressionEvaluationException as e: 75 | errors.append(str(e)) 76 | 77 | return rendered_inputs, errors 78 | 79 | def render_vars(self, in_ctx): 80 | vars_specs = getattr(self, "vars") or {} 81 | rendered_vars = {} 82 | errors = [] 83 | 84 | try: 85 | rendered_vars = expr_base.evaluate(vars_specs, in_ctx) 86 | except exc.ExpressionEvaluationException as e: 87 | errors.append(str(e)) 88 | 89 | return rendered_vars, errors 90 | 91 | def render_output(self, in_ctx): 92 | output_specs = getattr(self, "output") or {} 93 | rendered_outputs = {} 94 | errors = [] 95 | 96 | try: 97 | rendered_outputs = { 98 | var_name: expr_base.evaluate(var_expr, in_ctx) 99 | for var_name, var_expr in six.iteritems(output_specs) 100 | } 101 | except exc.ExpressionEvaluationException as e: 102 | errors.append(str(e)) 103 | 104 | return rendered_outputs, errors 105 | 106 | 107 | class WorkbookSpec(mistral_spec_base.Spec): 108 | _schema = { 109 | "type": "object", 110 | "properties": { 111 | "workflows": { 112 | "type": "object", 113 | "minProperties": 1, 114 | "patternProperties": {r"^(?!version)\w+$": WorkflowSpec}, 115 | } 116 | }, 117 | "additionalProperties": False, 118 | } 119 | -------------------------------------------------------------------------------- /orquestaconvert/expressions/__init__.py: -------------------------------------------------------------------------------- 1 | # Licensed under the Apache License, Version 2.0 (the "License"); 2 | # you may not use this file except in compliance with the License. 3 | # You may obtain a copy of the License at 4 | # 5 | # http://www.apache.org/licenses/LICENSE-2.0 6 | # 7 | # Unless required by applicable law or agreed to in writing, software 8 | # distributed under the License is distributed on an "AS IS" BASIS, 9 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 10 | # See the License for the specific language governing permissions and 11 | # limitations under the License. 12 | 13 | import ruamel.yaml.comments 14 | import six 15 | import warnings 16 | 17 | import orquesta.expressions.base 18 | 19 | from orquestaconvert.expressions import jinja 20 | from orquestaconvert.expressions import yaql as yql 21 | from orquestaconvert.utils import type_utils 22 | 23 | 24 | class ExpressionConverter(object): 25 | 26 | @classmethod 27 | def convert(cls, expr, **kwargs): 28 | if (isinstance(expr, type_utils.dict_types)): 29 | return cls.convert_dict(expr, **kwargs) 30 | elif isinstance(expr, list): 31 | return cls.convert_list(expr, **kwargs) 32 | elif isinstance(expr, six.string_types): 33 | return cls.convert_string(expr, **kwargs) 34 | elif isinstance(expr, bool): 35 | return expr 36 | elif expr is None: 37 | # Note: The only point to explicitly converting None is to skip 38 | # emitting the warning. Otherwise we would remove this elif 39 | # and let the function return the input. Also, ruamel doesn't 40 | # explicitly serialize null to None, it just leaves the key 41 | # blank (in other words, it "implicitly" serializes it). 42 | # Example: 43 | # 44 | # next: 45 | # - when: ... 46 | # publish: 47 | # - continue: 48 | # ... 49 | # 50 | # See this link for more information: 51 | # https://bitbucket.org/ruamel/yaml/issues/169/roundtripdumper-dumps-null-values 52 | return None 53 | elif isinstance(expr, int): 54 | return expr 55 | else: 56 | warnings.warn("Could not recognize expression '{}'; results may not " 57 | "be accurate.".format(expr), SyntaxWarning) 58 | return expr 59 | 60 | @classmethod 61 | def expression_type(cls, expr): 62 | for name, evaluator in six.iteritems(orquesta.expressions.base.get_evaluators()): 63 | if evaluator.has_expressions(str(expr)): 64 | return name 65 | return None 66 | 67 | @classmethod 68 | def get_converter(cls, expr): 69 | expr_type = cls.expression_type(expr) 70 | if expr_type == 'jinja': 71 | return jinja.JinjaExpressionConverter 72 | elif expr_type == 'yaql': 73 | return yql.YaqlExpressionConverter 74 | return None 75 | 76 | @classmethod 77 | def unwrap_expression(cls, expr): 78 | converter = cls.get_converter(expr) 79 | if converter: 80 | return converter.unwrap_expression(expr) 81 | # this isn't a Jinja or YAQL expression, so return the raw string 82 | return expr 83 | 84 | @classmethod 85 | def convert_string(cls, expr, **kwargs): 86 | # - task('xxx').result -> result() 87 | # if 'xxx' != current task name, error 88 | # - _. -> ctx(). 89 | # - $. -> ctx(). 90 | # - st2kv. -> st2kv('xxx') 91 | # others? 92 | converter = cls.get_converter(expr) 93 | if converter: 94 | return converter.convert_string(expr, **kwargs) 95 | # this isn't a Jinja or YAQL expression, so return the raw string 96 | return expr 97 | 98 | @classmethod 99 | def convert_dict(cls, expr_dict, **kwargs): 100 | converted = ruamel.yaml.comments.CommentedMap() 101 | for k, expr in six.iteritems(expr_dict): 102 | converted[cls.convert(k, **kwargs)] = cls.convert(expr, **kwargs) 103 | return converted 104 | 105 | @classmethod 106 | def convert_list(cls, expr_list, **kwargs): 107 | return [cls.convert(expr, **kwargs) for expr in expr_list] 108 | -------------------------------------------------------------------------------- /dist_utils.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | from __future__ import absolute_import 16 | 17 | import os 18 | import re 19 | import sys 20 | 21 | from distutils import version 22 | 23 | GET_PIP = 'curl https://bootstrap.pypa.io/get-pip.py | python' 24 | 25 | try: 26 | import pip 27 | except ImportError as e: 28 | print('Failed to import pip: %s' % (str(e))) 29 | print('') 30 | print('Download pip:\n%s' % (GET_PIP)) 31 | sys.exit(1) 32 | 33 | __all__ = [ 34 | 'check_pip_version', 35 | 'fetch_requirements', 36 | 'apply_vagrant_workaround', 37 | 'get_version_string', 38 | 'parse_version_string' 39 | ] 40 | 41 | 42 | def check_pip_version(min_version='6.0.0'): 43 | """Ensure that a minimum supported version of pip is installed. 44 | 45 | """ 46 | if version.StrictVersion(pip.__version__) < version.StrictVersion(min_version): 47 | print("Upgrade pip, your version '{0}' " 48 | "is outdated. Minimum required version is '{1}':\n{2}".format(pip.__version__, 49 | min_version, 50 | GET_PIP)) 51 | sys.exit(1) 52 | 53 | 54 | def fetch_requirements(requirements_file_path): 55 | """Return a list of requirements and links by parsing the provided requirements file. 56 | 57 | """ 58 | links = [] 59 | reqs = [] 60 | 61 | def _get_link(line): 62 | vcs_prefixes = ['git+', 'svn+', 'hg+', 'bzr+'] 63 | 64 | for vcs_prefix in vcs_prefixes: 65 | if line.startswith(vcs_prefix) or line.startswith('-e %s' % (vcs_prefix)): 66 | req_name = re.findall('.*#egg=(.+)([&|@]).*$', line) 67 | 68 | if not req_name: 69 | req_name = re.findall('.*#egg=(.+?)$', line) 70 | else: 71 | req_name = req_name[0] 72 | 73 | if not req_name: 74 | raise ValueError('Line "%s" is missing "#egg="' % (line)) 75 | 76 | link = line.replace('-e ', '').strip() 77 | return link, req_name[0] 78 | 79 | return None, None 80 | 81 | with open(requirements_file_path, 'r') as fp: 82 | for line in fp.readlines(): 83 | line = line.strip() 84 | 85 | if line.startswith('#') or not line: 86 | continue 87 | 88 | link, req_name = _get_link(line=line) 89 | 90 | if link: 91 | links.append(link) 92 | else: 93 | req_name = line 94 | 95 | if ';' in req_name: 96 | req_name = req_name.split(';')[0].strip() 97 | 98 | reqs.append(req_name) 99 | 100 | return (reqs, links) 101 | 102 | 103 | def apply_vagrant_workaround(): 104 | """Function which detects if the script is being executed inside vagrant 105 | 106 | Function which detects if the script is being executed inside vagrant and if it is, it deletes 107 | "os.link" attribute. 108 | Note: Without this workaround, setup.py sdist will fail when running inside a shared directory 109 | (nfs / virtualbox shared folders). 110 | """ 111 | if os.environ.get('USER', None) == 'vagrant': 112 | del os.link 113 | 114 | 115 | def get_version_string(init_file): 116 | """Read __version__ string for an init file. 117 | 118 | """ 119 | 120 | with open(init_file, 'r') as fp: 121 | content = fp.read() 122 | version_match = re.search(r"^__version__ = ['\"]([^'\"]*)['\"]", 123 | content, re.M) 124 | if version_match: 125 | return version_match.group(1) 126 | 127 | raise RuntimeError('Unable to find version string in %s.' % (init_file)) 128 | 129 | 130 | # alias for get_version_string 131 | parse_version_string = get_version_string 132 | -------------------------------------------------------------------------------- /orquestaconvert/client.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | import argparse 16 | import sys 17 | 18 | from orquesta.specs.native.v1 import models as native_v1_models 19 | from orquestaconvert.specs.mistral.v2 import workflows as mistral_workflow 20 | from orquestaconvert.utils import yaml_utils 21 | from orquestaconvert.workflows import base as workflows_base 22 | 23 | 24 | class Client(object): 25 | 26 | def parser(self): 27 | parser = argparse.ArgumentParser(description='Convert Mistral workflows to Orquesta') 28 | parser.add_argument('-v', '--verbose', default=False, action='store_true', 29 | help='Print success message when validating, otherwise ignored') 30 | parser.add_argument('-e', '--expressions', 31 | choices=['jinja', 'yaql'], 32 | default='jinja', 33 | help=('Type of expressions to use when inserting new expressions' 34 | ' (things like task transitions in the "when" clause.')) 35 | parser.add_argument('--force', default=False, action='store_true', 36 | help='Include unsupported attributes in the generated outputs') 37 | parser.add_argument('--validate', default=False, action='store_true', 38 | help='Validate the Orquesta workflow') 39 | parser.add_argument('filename', metavar='FILENAME', nargs=1, 40 | help='Path to the Mistral Workflow YAML file to convert') 41 | return parser 42 | 43 | def validate_workflow_spec(self, wf_spec): 44 | result = wf_spec.inspect_syntax() 45 | if result: 46 | raise ValueError(result) 47 | 48 | def convert_file(self, filename, expr_type=None): 49 | # parse the Mistral workflow from file 50 | mistral_wf_data, mistral_wf_data_ruamel = yaml_utils.read_yaml(filename) 51 | 52 | # validate the Mistral workflow before we start 53 | mistral_wf_spec = mistral_workflow.instantiate(mistral_wf_data) 54 | self.validate_workflow_spec(mistral_wf_spec) 55 | 56 | # convert Mistral -> Orquesta 57 | mistral_wf = mistral_wf_data_ruamel[mistral_wf_spec.name] 58 | workflow_converter = workflows_base.WorkflowConverter() 59 | orquesta_wf_data_ruamel = workflow_converter.convert(mistral_wf, expr_type, 60 | force=self.args.force) 61 | orquesta_wf_data_str = yaml_utils.obj_to_yaml(orquesta_wf_data_ruamel) 62 | orquesta_wf_data = yaml_utils.yaml_to_obj(orquesta_wf_data_str) 63 | 64 | # validate we've generated a proper Orquesta workflow 65 | orquesta_wf_spec = native_v1_models.instantiate(orquesta_wf_data) 66 | if not self.args.force: 67 | self.validate_workflow_spec(orquesta_wf_spec) 68 | 69 | # write out the new Orquesta workflow to a YAML string 70 | return yaml_utils.obj_to_yaml(orquesta_wf_data_ruamel) 71 | 72 | def validate_file(self, filename): 73 | # parse the Orquesta workflow from file 74 | orquesta_wf_data, orquesta_wf_data_ruamel = yaml_utils.read_yaml(filename) 75 | 76 | # validate the Orquesta workflow 77 | orquesta_wf_spec = native_v1_models.instantiate(orquesta_wf_data) 78 | self.validate_workflow_spec(orquesta_wf_spec) 79 | 80 | if self.args.verbose: 81 | print("Successfully validated workflow from {}".format(filename)) 82 | 83 | def run(self, argv, output_stream): 84 | # Write the file to the output_stream 85 | self.args = self.parser().parse_args(argv) 86 | expr_type = self.args.expressions 87 | if self.args.validate: 88 | for f in self.args.filename: 89 | self.validate_file(f) 90 | else: 91 | for f in self.args.filename: 92 | output_stream.write(self.convert_file(f, expr_type)) 93 | return 0 94 | 95 | 96 | if __name__ == '__main__': 97 | # Write the converted workflow to stdout 98 | sys.exit(Client().run(sys.argv[1:], sys.stdout)) 99 | -------------------------------------------------------------------------------- /orquestaconvert/expressions/mixed.py: -------------------------------------------------------------------------------- 1 | # Licensed under the Apache License, Version 2.0 (the "License"); 2 | # you may not use this file except in compliance with the License. 3 | # You may obtain a copy of the License at 4 | # 5 | # http://www.apache.org/licenses/LICENSE-2.0 6 | # 7 | # Unless required by applicable law or agreed to in writing, software 8 | # distributed under the License is distributed on an "AS IS" BASIS, 9 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 10 | # See the License for the specific language governing permissions and 11 | # limitations under the License. 12 | 13 | import re 14 | import six 15 | import warnings 16 | 17 | import ruamel.yaml.comments 18 | 19 | from orquestaconvert import expressions 20 | from orquestaconvert.expressions import base as expr_base 21 | from orquestaconvert.utils import type_utils 22 | 23 | 24 | # These regexes match Jinja and YAQL expressions, respectively. They are used 25 | # to pick out expressions from strings that contain both types of expressions, 26 | # which we call 'mixed' expressions, in attributes like task actions: 27 | # 28 | # tasks: 29 | # - do_thing: 30 | # with-item: target_host in {{ _.target_hosts }} 31 | # action: ping <% $.ping_flags %> {{ _.target_host }} 32 | # 33 | # Each expression needs to be converted to use the ctx() and item() accessors 34 | # in Orquesta: 35 | # 36 | # tasks: 37 | # - do_thing: 38 | # action: ping <% ctx().ping_flags %> {{ item(target_host) }} 39 | # 40 | JINJA_EXPR_RGX = re.compile(r'(?P(?:{{)\s*.+?\s*(?:}}))') 41 | YAQL_EXPR_RGX = re.compile(r'(?P(?:<%)\s*.+?\s*(?:%>))') 42 | 43 | 44 | class MixedExpressionConverter(expr_base.BaseExpressionConverter): 45 | '''Converter is used to convert all expressions 46 | 47 | Mixed expressions are strings that possibly contain both Jinja and YAQL 48 | expressions. This converter is used to convert all expressions, regardless 49 | of their type. 50 | ''' 51 | 52 | @classmethod 53 | def convert(cls, expr, **kwargs): 54 | if (isinstance(expr, type_utils.dict_types)): 55 | return cls.convert_dict(expr, **kwargs) 56 | elif isinstance(expr, list): 57 | return cls.convert_list(expr, **kwargs) 58 | elif isinstance(expr, six.string_types): 59 | return cls.convert_string(expr, **kwargs) 60 | elif isinstance(expr, bool): 61 | return expr 62 | elif expr is None: 63 | # Note: The only point to explicitly converting None is to skip 64 | # emitting the warning. Otherwise we would remove this elif 65 | # and let the function return the input. Also, ruamel doesn't 66 | # explicitly serialize null to None, it just leaves the key 67 | # blank (in other words, it "implicitly" serializes it). 68 | # Example: 69 | # 70 | # next: 71 | # - when: ... 72 | # publish: 73 | # - continue: 74 | # ... 75 | # 76 | # See this link for more information: 77 | # https://bitbucket.org/ruamel/yaml/issues/169/roundtripdumper-dumps-null-values 78 | return None 79 | elif isinstance(expr, int): 80 | return expr 81 | else: 82 | warnings.warn("Could not recognize expression '{}'; results may not " 83 | "be accurate.".format(expr), SyntaxWarning) 84 | return expr 85 | 86 | @classmethod 87 | def convert_string_containing_expressions(cls, match, **kwargs): 88 | expr = match.group('expr') 89 | converter = expressions.ExpressionConverter.get_converter(expr) 90 | if converter: 91 | expr = converter.unwrap_expression(expr) 92 | expr = converter.convert_string(expr, **kwargs) 93 | expr = converter.wrap_expression(expr) 94 | 95 | return expr 96 | 97 | @classmethod 98 | def convert_string(cls, expr, **kwargs): 99 | def _inner_convert_string(match): 100 | return cls.convert_string_containing_expressions(match, **kwargs) 101 | expr = JINJA_EXPR_RGX.sub(_inner_convert_string, expr) 102 | expr = YAQL_EXPR_RGX.sub(_inner_convert_string, expr) 103 | return expr 104 | 105 | @classmethod 106 | def convert_dict(cls, expr_dict, **kwargs): 107 | converted = ruamel.yaml.comments.CommentedMap() 108 | for k, expr in six.iteritems(expr_dict): 109 | converted[k] = cls.convert(expr, **kwargs) 110 | return converted 111 | 112 | @classmethod 113 | def convert_list(cls, expr_list, **kwargs): 114 | return [cls.convert(expr, **kwargs) for expr in expr_list] 115 | -------------------------------------------------------------------------------- /tests/unit/test_expressions_jinja.py: -------------------------------------------------------------------------------- 1 | # Licensed under the Apache License, Version 2.0 (the "License"); 2 | # you may not use this file except in compliance with the License. 3 | # You may obtain a copy of the License at 4 | # 5 | # http://www.apache.org/licenses/LICENSE-2.0 6 | # 7 | # Unless required by applicable law or agreed to in writing, software 8 | # distributed under the License is distributed on an "AS IS" BASIS, 9 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 10 | # See the License for the specific language governing permissions and 11 | # limitations under the License. 12 | 13 | from orquestaconvert.expressions import jinja 14 | 15 | from tests import base_test_case 16 | 17 | 18 | class TestExpressionsJinja(base_test_case.BaseTestCase): 19 | __test__ = True 20 | 21 | def test_unwrap_expression(self): 22 | expr = "{{ _.test }}" 23 | result = jinja.JinjaExpressionConverter.unwrap_expression(expr) 24 | self.assertEqual(result, "_.test") 25 | 26 | def test_unwrap_expression_nested(self): 27 | expr = "{{ _.test {{ abc }} }}" 28 | result = jinja.JinjaExpressionConverter.unwrap_expression(expr) 29 | self.assertEqual(result, "_.test {{ abc }}") 30 | 31 | def test_unwrap_expression_trim_spaces(self): 32 | expr = "{{ _.test }}" 33 | result = jinja.JinjaExpressionConverter.unwrap_expression(expr) 34 | self.assertEqual(result, "_.test") 35 | 36 | def test_convert_expression_jinja_context_vars(self): 37 | expr = "{{ _.test }}" 38 | result = jinja.JinjaExpressionConverter.convert_string(expr) 39 | self.assertEqual(result, "{{ ctx().test }}") 40 | 41 | def test_convert_expression_jinja_item_vars(self): 42 | expr = "{{ _.test }}" 43 | result = jinja.JinjaExpressionConverter.convert_string(expr, item_vars=['test']) 44 | self.assertEqual(result, "{{ item(test) }}") 45 | 46 | def test_convert_expression_jinja_context_and_item_vars(self): 47 | expr = "{{ _.test + _.test2 - _.long_var }}" 48 | result = jinja.JinjaExpressionConverter.convert_string(expr, item_vars=['test']) 49 | self.assertEqual(result, "{{ item(test) + ctx().test2 - ctx().long_var }}") 50 | 51 | def test_convert_expression_jinja_function_context_vars(self): 52 | expr = "{{ list(range(0, _.count)) }}" 53 | result = jinja.JinjaExpressionConverter.convert_string(expr) 54 | self.assertEqual(result, "{{ list(range(0, ctx().count)) }}") 55 | 56 | def test_convert_expression_jinja_complex_function_context_vars(self): 57 | expr = "{{ zip([0, 1, 2], [3, 4, 5], _.all_the_things) }}" 58 | result = jinja.JinjaExpressionConverter.convert_string(expr) 59 | self.assertEqual(result, "{{ zip([0, 1, 2], [3, 4, 5], ctx().all_the_things) }}") 60 | 61 | def test_convert_expression_jinja_context_vars_multiple(self): 62 | expr = "{{ _.test + _.other }}" 63 | result = jinja.JinjaExpressionConverter.convert_string(expr) 64 | self.assertEqual(result, "{{ ctx().test + ctx().other }}") 65 | 66 | def test_convert_expression_jinja_context_vars_with_underscore(self): 67 | expr = "{{ _.test_.other }}" 68 | result = jinja.JinjaExpressionConverter.convert_string(expr) 69 | self.assertEqual(result, "{{ ctx().test_.other }}") 70 | 71 | def test_convert_expression_jinja_task_result(self): 72 | expr = "{{ task('abc').result.result }}" 73 | result = jinja.JinjaExpressionConverter.convert_string(expr) 74 | self.assertEqual(result, "{{ result().result }}") 75 | 76 | def test_convert_expression_jinja_task_result_double_quotes(self): 77 | expr = '{{ task("abc").result.double_quote }}' 78 | result = jinja.JinjaExpressionConverter.convert_string(expr) 79 | self.assertEqual(result, "{{ result().double_quote }}") 80 | 81 | def test_convert_expression_jinja_st2kv(self): 82 | expr = '{{ st2kv.system.test.kv }}' 83 | result = jinja.JinjaExpressionConverter.convert_string(expr) 84 | self.assertEqual(result, "{{ st2kv('system.test.kv') }}") 85 | 86 | def test_convert_expression_jinja_st2kv_user(self): 87 | expr = '{{ st2kv.user.test.kv }}' 88 | result = jinja.JinjaExpressionConverter.convert_string(expr) 89 | self.assertEqual(result, "{{ st2kv('user.test.kv') }}") 90 | 91 | def test_convert_expression_jinja_st2_execution_id(self): 92 | expr = '{{ env().st2_execution_id }}' 93 | result = jinja.JinjaExpressionConverter.convert_string(expr) 94 | self.assertEqual(result, "{{ ctx().st2.action_execution_id }}") 95 | 96 | def test_convert_expression_jinja_st2_api_url(self): 97 | expr = '{{ env().st2_action_api_url }}' 98 | result = jinja.JinjaExpressionConverter.convert_string(expr) 99 | self.assertEqual(result, "{{ ctx().st2.api_url }}") 100 | -------------------------------------------------------------------------------- /orquestaconvert/expressions/base.py: -------------------------------------------------------------------------------- 1 | # Licensed under the Apache License, Version 2.0 (the "License"); 2 | # you may not use this file except in compliance with the License. 3 | # You may obtain a copy of the License at 4 | # 5 | # http://www.apache.org/licenses/LICENSE-2.0 6 | # 7 | # Unless required by applicable law or agreed to in writing, software 8 | # distributed under the License is distributed on an "AS IS" BASIS, 9 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 10 | # See the License for the specific language governing permissions and 11 | # limitations under the License. 12 | 13 | import abc 14 | import re 15 | import six 16 | 17 | # task('xxx').result -> result() 18 | # if 'xxx' != current task name, error 19 | TASK_RESULT_REGEX = r"(task\([\"\']?\w+[\"\']?\).result)" 20 | TASK_RESULT_PATTERN = re.compile(TASK_RESULT_REGEX) 21 | 22 | # - st2kv. -> st2kv('xxx') 23 | ST2KV_REGEX = r"(st2kv\.([\w\.]+))" 24 | ST2KV_PATTERN = re.compile(ST2KV_REGEX) 25 | 26 | # env().st2_execution_id -> ctx().st2.action_execution_id 27 | ST2_EXECUTION_ID_REGEX = r"\benv\(\)\.st2_execution_id\b" 28 | ST2_EXECUTION_ID_PATTERN = re.compile(ST2_EXECUTION_ID_REGEX) 29 | 30 | # env().st2_action_api_url -> ctx().st2.api_url 31 | ST2_API_URL_REGEX = r"\benv\(\).st2_action_api_url\b" 32 | ST2_API_URL_PATTERN = re.compile(ST2_API_URL_REGEX) 33 | 34 | 35 | @six.add_metaclass(abc.ABCMeta) 36 | class AbstractBaseExpressionConverter(object): 37 | 38 | @classmethod 39 | @abc.abstractmethod 40 | def wrap_expression(cls, expr): 41 | raise NotImplementedError() 42 | 43 | @classmethod 44 | @abc.abstractmethod 45 | def unwrap_expression(cls, expr): 46 | raise NotImplementedError() 47 | 48 | @classmethod 49 | @abc.abstractmethod 50 | def convert_string(self, expr, **kwargs): 51 | raise NotImplementedError() 52 | 53 | @classmethod 54 | @abc.abstractmethod 55 | def convert_context_vars(self, expr, **kwargs): 56 | raise NotImplementedError() 57 | 58 | @classmethod 59 | @abc.abstractmethod 60 | def convert_task_result(self, expr, **kwargs): 61 | raise NotImplementedError() 62 | 63 | @classmethod 64 | @abc.abstractmethod 65 | def convert_st2kv(self, expr, **kwargs): 66 | raise NotImplementedError() 67 | 68 | @classmethod 69 | @abc.abstractmethod 70 | def convert_st2_execution_id(self, expr, **kwargs): 71 | raise NotImplementedError() 72 | 73 | @classmethod 74 | @abc.abstractmethod 75 | def convert_st2_api_url(self, expr, **kwargs): 76 | raise NotImplementedError() 77 | 78 | 79 | class BaseExpressionConverter(AbstractBaseExpressionConverter): 80 | 81 | @classmethod 82 | def _replace_unwrap(cls, match): 83 | return match.group(2).strip() 84 | 85 | @classmethod 86 | def convert_string(cls, expr, **kwargs): 87 | # others? 88 | expr = cls.convert_context_vars(expr, **kwargs) 89 | expr = cls.convert_task_result(expr) 90 | expr = cls.convert_st2kv(expr) 91 | expr = cls.convert_st2_execution_id(expr) 92 | expr = cls.convert_st2_api_url(expr) 93 | return expr 94 | 95 | @classmethod 96 | def _replace_item_vars(cls, match): 97 | return "item(" + match.group(2) + ")" 98 | 99 | @classmethod 100 | def _replace_context_vars(cls, match): 101 | return "ctx()." + match.group(2) 102 | 103 | @classmethod 104 | def _get_replace_vars(cls, **kwargs): 105 | item_vars = kwargs.get('item_vars') or [] 106 | 107 | def _inner_replace_vars(match): 108 | if match and match.group(2) in item_vars: 109 | return cls._replace_item_vars(match) 110 | else: 111 | return cls._replace_context_vars(match) 112 | 113 | return _inner_replace_vars 114 | 115 | @classmethod 116 | def _replace_task_result(cls, match): 117 | return "result()" 118 | 119 | @classmethod 120 | def convert_task_result(cls, expr): 121 | # TODO(nmaludy) error if task name is not the same task in this context 122 | return TASK_RESULT_PATTERN.sub(cls._replace_task_result, expr) 123 | 124 | @classmethod 125 | def _replace_st2kv(cls, match): 126 | return "st2kv('" + match.group(2) + "')" 127 | 128 | @classmethod 129 | def convert_st2kv(cls, expr): 130 | return ST2KV_PATTERN.sub(cls._replace_st2kv, expr) 131 | 132 | @classmethod 133 | def _replace_st2_execution_id(cls, match): 134 | return "ctx().st2.action_execution_id" 135 | 136 | @classmethod 137 | def convert_st2_execution_id(cls, expr): 138 | return ST2_EXECUTION_ID_PATTERN.sub(cls._replace_st2_execution_id, expr) 139 | 140 | @classmethod 141 | def _replace_st2_api_url(cls, match): 142 | return "ctx().st2.api_url" 143 | 144 | @classmethod 145 | def convert_st2_api_url(cls, expr): 146 | return ST2_API_URL_PATTERN.sub(cls._replace_st2_api_url, expr) 147 | -------------------------------------------------------------------------------- /tests/unit/test_expressions_yaql.py: -------------------------------------------------------------------------------- 1 | # Licensed under the Apache License, Version 2.0 (the "License"); 2 | # you may not use this file except in compliance with the License. 3 | # You may obtain a copy of the License at 4 | # 5 | # http://www.apache.org/licenses/LICENSE-2.0 6 | # 7 | # Unless required by applicable law or agreed to in writing, software 8 | # distributed under the License is distributed on an "AS IS" BASIS, 9 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 10 | # See the License for the specific language governing permissions and 11 | # limitations under the License. 12 | 13 | from orquestaconvert.expressions import yaql as yql 14 | 15 | from tests import base_test_case 16 | 17 | 18 | class TestExpressionsYaql(base_test_case.BaseTestCase): 19 | __test__ = True 20 | 21 | def test_unwrap_expression(self): 22 | expr = "<% $.test %>" 23 | result = yql.YaqlExpressionConverter.unwrap_expression(expr) 24 | self.assertEqual(result, "$.test") 25 | 26 | def test_unwrap_expression_nested(self): 27 | expr = "<% $.test <% abc %> %>" 28 | result = yql.YaqlExpressionConverter.unwrap_expression(expr) 29 | self.assertEqual(result, "$.test <% abc %>") 30 | 31 | def test_unwrap_expression_trim_spaces(self): 32 | expr = "<% $.test %>" 33 | result = yql.YaqlExpressionConverter.unwrap_expression(expr) 34 | self.assertEqual(result, "$.test") 35 | 36 | def test_convert_expression_yaql_context_vars(self): 37 | expr = "<% $.test %>" 38 | result = yql.YaqlExpressionConverter.convert_string(expr) 39 | self.assertEqual(result, "<% ctx().test %>") 40 | 41 | def test_convert_expression_yaql_item_vars(self): 42 | expr = "<% $.test %>" 43 | result = yql.YaqlExpressionConverter.convert_string(expr, item_vars=['test']) 44 | self.assertEqual(result, "<% item(test) %>") 45 | 46 | def test_convert_expression_yaql_context_and_item_vars(self): 47 | expr = "<% $.test + $.test2 - $.long_var %>" 48 | result = yql.YaqlExpressionConverter.convert_string(expr, item_vars=['test']) 49 | self.assertEqual(result, "<% item(test) + ctx().test2 - ctx().long_var %>") 50 | 51 | def test_convert_expression_yaql_function_context_vars(self): 52 | expr = "<% list(range(0, $.count)) %>" 53 | result = yql.YaqlExpressionConverter.convert_string(expr) 54 | self.assertEqual(result, "<% list(range(0, ctx().count)) %>") 55 | 56 | def test_convert_expression_yaql_complex_function_context_vars(self): 57 | expr = "<% zip([0, 1, 2], [3, 4, 5], $.all_the_things) %>" 58 | result = yql.YaqlExpressionConverter.convert_string(expr) 59 | self.assertEqual(result, "<% zip([0, 1, 2], [3, 4, 5], ctx().all_the_things) %>") 60 | 61 | def test_convert_expression_yaql_context_vars_multiple(self): 62 | expr = "<% $.test + $.other %>" 63 | result = yql.YaqlExpressionConverter.convert_string(expr) 64 | self.assertEqual(result, "<% ctx().test + ctx().other %>") 65 | 66 | def test_convert_expression_yaql_context_vars_with_underscore(self): 67 | expr = "<% $.test_.other %>" 68 | result = yql.YaqlExpressionConverter.convert_string(expr) 69 | self.assertEqual(result, "<% ctx().test_.other %>") 70 | 71 | def test_convert_expression_yaql_task_result(self): 72 | expr = "<% task(abc).result.result %>" 73 | result = yql.YaqlExpressionConverter.convert_string(expr) 74 | self.assertEqual(result, "<% result().result %>") 75 | 76 | def test_convert_expression_yaql_task_result_single_quote(self): 77 | expr = "<% task('abc').result.result %>" 78 | result = yql.YaqlExpressionConverter.convert_string(expr) 79 | self.assertEqual(result, "<% result().result %>") 80 | 81 | def test_convert_expression_yaql_task_result_double_quotes(self): 82 | expr = '<% task("abc").result.double_quote %>' 83 | result = yql.YaqlExpressionConverter.convert_string(expr) 84 | self.assertEqual(result, "<% result().double_quote %>") 85 | 86 | def test_convert_expression_yaql_st2kv(self): 87 | expr = '<% st2kv.system.test.kv %>' 88 | result = yql.YaqlExpressionConverter.convert_string(expr) 89 | self.assertEqual(result, "<% st2kv('system.test.kv') %>") 90 | 91 | def test_convert_expression_yaql_st2kv_user(self): 92 | expr = '<% st2kv.user.test.kv %>' 93 | result = yql.YaqlExpressionConverter.convert_string(expr) 94 | self.assertEqual(result, "<% st2kv('user.test.kv') %>") 95 | 96 | def test_convert_expression_yaql_st2_execution_id(self): 97 | expr = '<% env().st2_execution_id %>' 98 | result = yql.YaqlExpressionConverter.convert_string(expr) 99 | self.assertEqual(result, "<% ctx().st2.action_execution_id %>") 100 | 101 | def test_convert_expression_yaql_st2_api_url(self): 102 | expr = '<% env().st2_action_api_url %>' 103 | result = yql.YaqlExpressionConverter.convert_string(expr) 104 | self.assertEqual(result, "<% ctx().st2.api_url %>") 105 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | [![Build Status](https://circleci.com/gh/StackStorm/orquestaconvert.svg?style=shield&circle-token=:circle-token)](https://circleci.com/gh/StackStorm/orquestaconvert) [![codecov](https://codecov.io/gh/StackStorm/orquestaconvert/branch/master/graph/badge.svg)](https://codecov.io/gh/StackStorm/orquestaconvert) [![License](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://opensource.org/licenses/Apache-2.0) 2 | 3 | # orquestaconvert 4 | 5 | Converts Mistral workflows into Orquesta workflows 6 | 7 | # Usage 8 | 9 | ## Getting Started 10 | 11 | The script automatically sets up a `virtualenv` (if it doesn't exist) that contains all of the necessary depedencies to run. 12 | 13 | Simply run the command for the first time and everything will get setup for you: 14 | 15 | ``` shell 16 | git clone https://github.com/StackStorm/orquestaconvert.git 17 | cd orquestaconvert 18 | ./bin/orquestaconvert.sh --help 19 | ``` 20 | 21 | ## `orquestaconvert.sh` - convert a single workflow and print to stdout 22 | 23 | You must specify one or more workflow YAML files to convert as the last arguments to the script. 24 | 25 | There are also some options you can use: 26 | 27 | - `-e ` - Type of expressions (YAQL or Jinja) to use when inserting new expressions (such as task transitions in the `when` clause) 28 | - `--force` - Forces the script to convert and print the workflow even if it does not successfully validate against the Orquesta YAML schema. 29 | - `--validate` - Runs just the validation portion of the script, very useful to validate workflows you partially converted with `--force` then finished conversion by hand. 30 | 31 | ### Examples 32 | 33 | #### Convert a single workflow 34 | 35 | Convert the `nasa_apod_twitter_post.yaml` workflow from Mistral to Orquesta, using Jinja expressions (the default) in task transition conditions. 36 | 37 | ```shell 38 | ./bin/orquestaconvert.sh ./tests/fixtures/mistral/nasa_apod_twitter_post.yaml 39 | ``` 40 | 41 | #### Convert a workflow, output YAQL expressions 42 | 43 | Convert the workflow, using YAQL expressions for new task transition conditions, and skips Orquesta workflow validation. Note that this may generate a workflow that is *neither* a valid Mistral *nor* a valid Orquesta workflow. 44 | 45 | ```shell 46 | ./bin/orquestaconvert.sh -e yaql --force ./tests/fixtures/mistral/nasa_apod_twitter_post.yaml 47 | ``` 48 | 49 | #### Validate an Orquesta workflow 50 | 51 | Run Orquesta YAML schema validation on the file. Returns 0 on successful validation, nonzero on unsuccessful validation. Also use the `--verbose` option to explitly print the validation results for the file. 52 | 53 | ```shell 54 | ./bin/orquestaconvert.sh --validate ./tests/fixtures/orquesta/nasa_apod_twitter_post.yaml 55 | ``` 56 | 57 | ## `orquestaconvert-pack.sh` - convert all Mistral workflows in a pack 58 | 59 | This script scans a pack for all action metadata files and attempts to convert all Mistral workflows to Orquesta and/or validates all Orquesta workflows in a pack using the `orquestaconvert.sh` script. This script passes all unrecognized arguments to `orquestaconvert.sh`, so all actions you can do on one workflow with that script, you can do to the entire pack by using this script. 60 | 61 | You must either run this command from the base directory of a pack or you must specify the directory that contains action metadata files with the `--actions-dir` option. 62 | 63 | Recognized options are: 64 | 65 | - `--list-workflows ` - List all workflows of the specified type (must either be `action-chain` for ActionChain, `mistral-v2` for Mistral, or `orquesta` or `orchestra` for Orquesta workflows) 66 | - `--actions-dir ` - Specifies the directory to scan and convert 67 | 68 | ### Examples 69 | 70 | #### Convert all workflows in a pack 71 | 72 | Attempts to convert all workflows from Mistral to Orquesta, using Jinja expressions in new task transitions (Jinja is the default). 73 | 74 | ```shell 75 | ./bin/orquestaconvert-pack.sh 76 | ``` 77 | 78 | #### List Mistral workflows in a pack 79 | 80 | Lists remaining Mistral workflows. 81 | 82 | ```shell 83 | ./bin/orquestaconvert-pack.sh --list-workflows mistral-v2 84 | ``` 85 | 86 | #### Convert all workflows in a pack, outputting YAQL expressions 87 | 88 | Converts all Mistral workflows (using YAQL expressions when generating task transition conditions) in `mypack/actions` to Orquesta and skips validation. Note that using this option may create workflows that are *neither* valid as Mistral *nor* Orquesta workflows. 89 | 90 | ```shell 91 | ./bin/orquestaconvert-pack.sh --expressions yaql --force --action-dir mypack/actions 92 | ``` 93 | 94 | #### Validate all Orquesta workflows in a pack 95 | 96 | Explicitly rints the validation results for all Orquesta workflows. 97 | 98 | ```shell 99 | ./bin/orquestaconvert-pack.sh --validate --verbose 100 | ``` 101 | 102 | # Features 103 | 104 | * Converts `direct` Mistral Workflows into Orquesta Workflows (general structure) 105 | * Handles `input`, `output`, `tasks` 106 | * For each task, `action`, `input`, `publish`, `on-success`, `on-error`, and `on-complete` are all converted 107 | * Converts _simple_ Jinja and YAQL expressions 108 | * Converts `task()`, `st2kv`, `_.xxx` / `$.xxx`, etc in Jinja and YAQL expressions 109 | * Converts `with-items` and `concurrency` attributes 110 | * Converts most `retry` attributes, including `continue-on` and _simple_ `break-on` expressions 111 | 112 | # Limitations 113 | 114 | * Does not convert `{% %}` Jinja expressions 115 | * Does not convert complex Jinja / YAQL expressions 116 | * Does not convert `retry` specifications with complex Jinja expressions for `continue-on` and `break-on` 117 | - use the `--force` flag to forcibly convert those workflows and convert those expressions manually 118 | * Does not convert `reverse` type workflows 119 | * Does not convert workbooks (multiple workflows in the same file 120 | * Does not convert `task('xxx')` references to non-local tasks, the current task is always assumed. 121 | * Does not convert workflows with an `output-on-error` stanza 122 | * Does not convert workflows if the tasks contain one or more of the following attributes 123 | - `keep-result` 124 | - `pause-before` 125 | - `safe-rerun` 126 | - `target` 127 | - `timeout` 128 | - `wait-after` 129 | - `wait-before` 130 | - `workflow` 131 | 132 | 133 | # Development 134 | 135 | Donated to the StackStorm project by [Encore Technologies](https://encore.tech) 136 | -------------------------------------------------------------------------------- /tests/base_test_case.py: -------------------------------------------------------------------------------- 1 | # Licensed under the Apache License, Version 2.0 (the "License"); 2 | # you may not use this file except in compliance with the License. 3 | # You may obtain a copy of the License at 4 | # 5 | # http://www.apache.org/licenses/LICENSE-2.0 6 | # 7 | # Unless required by applicable law or agreed to in writing, software 8 | # distributed under the License is distributed on an "AS IS" BASIS, 9 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 10 | # See the License for the specific language governing permissions and 11 | # limitations under the License. 12 | 13 | import hashlib 14 | import json 15 | import os 16 | import shutil 17 | import six 18 | import sys 19 | import unittest2 20 | import yaml 21 | 22 | 23 | # The pristine actions directory - this is not touched, simply copied as the 24 | # Mistral actions directory 25 | P_ACTIONS_DIR = os.path.join('tests', 'fixtures', 'pack', 'pristine_actions') 26 | # The Orquesta actions directory - this is not touched, this is used to compare 27 | # autoconverted results against - autoconverted files should match files in 28 | # this directory 29 | O_ACTIONS_DIR = os.path.join('tests', 'fixtures', 'pack', 'o_actions') 30 | # The Mistral actions directory - this is a copy of the pristine actions 31 | # directory, and files within get mangled by the autoconvert script, then 32 | # compared against files in the Orquesta actions directory 33 | M_ACTIONS_DIR = os.path.join('tests', 'fixtures', 'pack', 'actions') 34 | 35 | # These files should fail autoconversion 36 | ACTION_FAILING_FILES = [ 37 | 'mistral-fail-retry-continue-and-break-on.yaml', 38 | ] 39 | # All of these files should be successfully converted 40 | ACTION_PASSING_FILES = [ 41 | 'mistral-test-cancel.yaml', 42 | 'mistral-with-items-jinja.yaml', 43 | 'mistral-with-items-static-list.yaml', 44 | 'mistral-with-items-mixed-list.yaml', 45 | 'mistral-with-items-yaql.yaml', 46 | 'mistral-with-items-yaml-list.yaml', 47 | 'mistral-with-items-yaql-list.yaml', 48 | 'mistral-with-items-concurrency.yaml', 49 | 'mistral-with-items-concurrency-str.yaml', 50 | 'mistral-with-items-concurrency-jinja.yaml', 51 | 'mistral-with-items-concurrency-yaql.yaml', 52 | 'mistral-transition-expressions.yaml', 53 | 'mistral-publish-only-transitions.yaml', 54 | 'mistral-retry.yaml', 55 | 'mistral-retry-continue-on.yaml', 56 | 'mistral-retry-break-on.yaml', 57 | 'mistral-retry-continue-and-break-on.yaml', 58 | ] 59 | 60 | 61 | class BaseTestCase(unittest2.TestCase): 62 | __test__ = False 63 | 64 | def get_fixture_content(self, filename): 65 | """Return raw fixture content for the provided fixture path. 66 | 67 | :param fixture_path: Fixture path relative to the tests/fixtures/ directory. 68 | :type fixture_path: ``str`` 69 | """ 70 | fixture_path = self.get_fixture_path(filename) 71 | with open(fixture_path, 'r') as fp: 72 | content = fp.read() 73 | return content 74 | 75 | def get_fixture_path(self, filename): 76 | base_path = self._get_base_path() 77 | fixtures_path = os.path.join(base_path, 'fixtures') 78 | return os.path.join(fixtures_path, filename) 79 | 80 | def _get_base_path(self): 81 | base_path = os.path.dirname(__file__) 82 | return os.path.abspath(base_path) 83 | 84 | def load_yaml(self, filename): 85 | return yaml.safe_load(self.get_fixture_content(filename)) 86 | 87 | def load_json(self, filename): 88 | return json.loads(self.get_fixture_content(filename)) 89 | 90 | 91 | class BaseCLITestCase(BaseTestCase): 92 | capture_output = True # if True, stdout and stderr are saved to self.stdout and self.stderr 93 | 94 | stdout = six.moves.StringIO() 95 | stderr = six.moves.StringIO() 96 | 97 | def setup_captures(self): 98 | if self.capture_output: 99 | # Make sure we reset it for each test class instance 100 | self.stdout = six.moves.StringIO() 101 | self.stderr = six.moves.StringIO() 102 | 103 | sys.stdout = self.stdout 104 | sys.stderr = self.stderr 105 | 106 | def reset_captures(self): 107 | if self.capture_output: 108 | # Reset to original stdout and stderr. 109 | sys.stdout = sys.__stdout__ 110 | sys.stderr = sys.__stderr__ 111 | 112 | def setUp(self): 113 | super(BaseCLITestCase, self).setUp() 114 | self.setup_captures() 115 | 116 | def tearDown(self): 117 | super(BaseCLITestCase, self).tearDown() 118 | self.reset_captures() 119 | 120 | 121 | class BasePackClientRunTestCase(BaseCLITestCase): 122 | def setUp(self): 123 | super(BasePackClientRunTestCase, self).setUp() 124 | 125 | # The pristine actions directory - this is not touched 126 | self.p_actions_dir = P_ACTIONS_DIR 127 | # The Orquesta actions directory - this is not touched 128 | self.o_actions_dir = O_ACTIONS_DIR 129 | # The Mistral actions directory - this is a copy of pristine actions 130 | self.m_actions_dir = M_ACTIONS_DIR 131 | 132 | self.action_failing_files = ACTION_FAILING_FILES 133 | self.action_passing_files = ACTION_PASSING_FILES 134 | self.action_files = self.action_failing_files + self.action_passing_files 135 | 136 | self.p_wfs_dir = os.path.join(self.p_actions_dir, 'workflows') 137 | self.o_wfs_dir = os.path.join(self.o_actions_dir, 'workflows') 138 | self.m_wfs_dir = os.path.join(self.m_actions_dir, 'workflows') 139 | 140 | self.pristine_action_wfs = { 141 | os.path.join(self.p_actions_dir, af): os.path.join(self.p_wfs_dir, af) 142 | for af in self.action_files 143 | } 144 | self.mistral_action_wfs = { 145 | os.path.join(self.m_actions_dir, af): os.path.join(self.m_wfs_dir, af) 146 | for af in self.action_failing_files 147 | } 148 | self.orquesta_action_wfs = { 149 | os.path.join(self.o_actions_dir, af): os.path.join(self.o_wfs_dir, af) 150 | for af in self.action_passing_files 151 | } 152 | self.action_wfs = { 153 | os.path.join(self.m_actions_dir, af): os.path.join(self.m_wfs_dir, af) 154 | for af in (self.action_failing_files + self.action_passing_files) 155 | } 156 | 157 | self.maxDiff = None 158 | 159 | if os.path.isdir(self.m_actions_dir): 160 | shutil.rmtree(self.m_actions_dir) 161 | 162 | shutil.copytree(self.p_actions_dir, self.m_actions_dir) 163 | 164 | def tearDown(self): 165 | super(BasePackClientRunTestCase, self).tearDown() 166 | 167 | if os.path.isdir(self.m_actions_dir): 168 | shutil.rmtree(self.m_actions_dir) 169 | 170 | def _hash_directory(self, directory, files): 171 | '''Hash files in a directory for comparison, returns a dictinary of hashes 172 | 173 | ''' 174 | dirhash = {} 175 | for dirf in files: 176 | with open(os.path.join(directory, dirf), 'r') as f: 177 | fdata = f.read() 178 | if not six.PY2: 179 | fdata = fdata.encode('utf-8') 180 | dirhash[dirf] = hashlib.sha256(fdata).hexdigest() 181 | return dirhash 182 | -------------------------------------------------------------------------------- /tests/unit/test_expressions.py: -------------------------------------------------------------------------------- 1 | # Licensed under the Apache License, Version 2.0 (the "License"); 2 | # you may not use this file except in compliance with the License. 3 | # You may obtain a copy of the License at 4 | # 5 | # http://www.apache.org/licenses/LICENSE-2.0 6 | # 7 | # Unless required by applicable law or agreed to in writing, software 8 | # distributed under the License is distributed on an "AS IS" BASIS, 9 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 10 | # See the License for the specific language governing permissions and 11 | # limitations under the License. 12 | 13 | import re 14 | 15 | from orquestaconvert import expressions 16 | from orquestaconvert.expressions import jinja 17 | from orquestaconvert.expressions import yaql as yql 18 | 19 | from tests import base_test_case 20 | 21 | 22 | class TestExpressions(base_test_case.BaseTestCase): 23 | __test__ = True 24 | 25 | def test_convert_expr_bool(self): 26 | expr_bool = True 27 | result = expressions.ExpressionConverter.convert(expr_bool) 28 | self.assertEqual(result, True) 29 | expr_bool = False 30 | result = expressions.ExpressionConverter.convert(expr_bool) 31 | self.assertEqual(result, False) 32 | 33 | def test_convert_expr_null(self): 34 | expr_none = None 35 | result = expressions.ExpressionConverter.convert(expr_none) 36 | self.assertEqual(result, None) 37 | 38 | def test_convert_expr_int(self): 39 | expr_int = 0 40 | result = expressions.ExpressionConverter.convert(expr_int) 41 | self.assertEqual(result, 0) 42 | expr_int = 100 43 | result = expressions.ExpressionConverter.convert(expr_int) 44 | self.assertEqual(result, 100) 45 | expr_int = -1 46 | result = expressions.ExpressionConverter.convert(expr_int) 47 | self.assertEqual(result, -1) 48 | 49 | def test_convert_expr_obj_with_warning(self): 50 | expr_obj = object() 51 | expected_warning_regex = re.compile( 52 | r"Could not recognize expression ''; " 53 | r"results may not be accurate.") 54 | with self.assertWarnsRegex(SyntaxWarning, expected_warning_regex): 55 | result = expressions.ExpressionConverter.convert(expr_obj) 56 | self.assertEqual(result, expr_obj) 57 | 58 | def test_convert_expr_dict(self): 59 | expr_dict = {"test": "{{ _.value }}"} 60 | result = expressions.ExpressionConverter.convert(expr_dict) 61 | self.assertEqual(result, {"test": "{{ ctx().value }}"}) 62 | 63 | def test_convert_expr_list(self): 64 | expr_list = ["test", "{{ _.value }}"] 65 | result = expressions.ExpressionConverter.convert(expr_list) 66 | self.assertEqual(result, ["test", "{{ ctx().value }}"]) 67 | 68 | def test_convert_expr_string(self): 69 | expr_dict = "{{ _.value }}" 70 | result = expressions.ExpressionConverter.convert(expr_dict) 71 | self.assertEqual(result, "{{ ctx().value }}") 72 | 73 | def test_convert_no_expression(self): 74 | expr = "data" 75 | result = expressions.ExpressionConverter.convert(expr) 76 | self.assertEqual(result, "data") 77 | 78 | def test_expression_type_jinja(self): 79 | expr = "{{ _.test }}" 80 | result = expressions.ExpressionConverter.expression_type(expr) 81 | self.assertEqual(result, 'jinja') 82 | 83 | def test_expression_type_yaql(self): 84 | expr = "<% $.test %>" 85 | result = expressions.ExpressionConverter.expression_type(expr) 86 | self.assertEqual(result, 'yaql') 87 | 88 | def test_expression_type_none(self): 89 | expr = "test" 90 | result = expressions.ExpressionConverter.expression_type(expr) 91 | self.assertIsNone(result) 92 | 93 | def test_get_converter_jinja(self): 94 | expr = "{{ _.test }}" 95 | result = expressions.ExpressionConverter.get_converter(expr) 96 | self.assertIs(result, jinja.JinjaExpressionConverter) 97 | 98 | def test_get_converter_yaql(self): 99 | expr = "<% $.test %>" 100 | result = expressions.ExpressionConverter.get_converter(expr) 101 | self.assertIs(result, yql.YaqlExpressionConverter) 102 | 103 | def test_get_converter_none(self): 104 | expr = "test" 105 | result = expressions.ExpressionConverter.get_converter(expr) 106 | self.assertIsNone(result) 107 | 108 | def test_unwrap_expression_jinja(self): 109 | expr = "{{ _.test }}" 110 | result = expressions.ExpressionConverter.unwrap_expression(expr) 111 | self.assertEqual(result, '_.test') 112 | 113 | def test_unwrap_expression_yaql(self): 114 | expr = "<% $.test %>" 115 | result = expressions.ExpressionConverter.unwrap_expression(expr) 116 | self.assertEqual(result, '$.test') 117 | 118 | def test_unwrap_expression_none(self): 119 | expr = "test" 120 | result = expressions.ExpressionConverter.unwrap_expression(expr) 121 | self.assertEqual(result, 'test') 122 | 123 | def test_convert_dict(self): 124 | expr = { 125 | "jinja_str": "{{ _.test_jinja }}", 126 | "yaql_str": "<% $.test_yaql %>", 127 | } 128 | result = expressions.ExpressionConverter.convert_dict(expr) 129 | self.assertEqual(result, { 130 | "jinja_str": "{{ ctx().test_jinja }}", 131 | "yaql_str": "<% ctx().test_yaql %>", 132 | }) 133 | 134 | def test_convert_dict_nested_list(self): 135 | expr = { 136 | "expr_list": 137 | [ 138 | "{{ _.a }}", 139 | "<% $.a %>", 140 | ] 141 | } 142 | result = expressions.ExpressionConverter.convert_dict(expr) 143 | self.assertEqual(result, { 144 | "expr_list": 145 | [ 146 | "{{ ctx().a }}", 147 | "<% ctx().a %>", 148 | ] 149 | }) 150 | 151 | def test_convert_dict_nested_dict(self): 152 | expr = { 153 | "expr_dict": 154 | { 155 | "nested_jinja": "{{ _.a }}", 156 | "nested_yaql": "<% $.a %>", 157 | } 158 | } 159 | result = expressions.ExpressionConverter.convert_dict(expr) 160 | self.assertEqual(result, { 161 | "expr_dict": 162 | { 163 | "nested_jinja": "{{ ctx().a }}", 164 | "nested_yaql": "<% ctx().a %>", 165 | } 166 | }) 167 | 168 | def test_convert_list(self): 169 | expr = [ 170 | "{{ _.test_jinja }}", 171 | "<% $.test_yaql %>", 172 | ] 173 | result = expressions.ExpressionConverter.convert_list(expr) 174 | self.assertEqual(result, [ 175 | "{{ ctx().test_jinja }}", 176 | "<% ctx().test_yaql %>", 177 | ]) 178 | 179 | def test_convert_list_nested_list(self): 180 | expr = [ 181 | "{{ _.a }}", 182 | [ 183 | "<% $.a %>", 184 | ] 185 | ] 186 | result = expressions.ExpressionConverter.convert_list(expr) 187 | self.assertEqual(result, [ 188 | "{{ ctx().a }}", 189 | [ 190 | "<% ctx().a %>", 191 | ] 192 | ]) 193 | 194 | def test_convert_list_nested_dict(self): 195 | expr = [ 196 | { 197 | "nested_jinja": "{{ _.a }}", 198 | "nested_yaql": "<% $.a %>", 199 | } 200 | ] 201 | result = expressions.ExpressionConverter.convert_list(expr) 202 | self.assertEqual(result, [ 203 | { 204 | "nested_jinja": "{{ ctx().a }}", 205 | "nested_yaql": "<% ctx().a %>", 206 | } 207 | ]) 208 | 209 | def test_convert_string_other(self): 210 | expr = "test some raw string" 211 | result = expressions.ExpressionConverter.convert_string(expr) 212 | self.assertEqual(result, "test some raw string") 213 | 214 | def test_convert_string_jinja(self): 215 | expr = "{{ _.test }}" 216 | result = expressions.ExpressionConverter.convert_string(expr) 217 | self.assertEqual(result, "{{ ctx().test }}") 218 | 219 | def test_convert_string_yaql(self): 220 | expr = "<% $.test %>" 221 | result = expressions.ExpressionConverter.convert_string(expr) 222 | self.assertEqual(result, "<% ctx().test %>") 223 | -------------------------------------------------------------------------------- /orquestaconvert/pack_client.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | import argparse 16 | import glob 17 | import os 18 | import shutil 19 | import six 20 | import sys 21 | 22 | import yaml 23 | 24 | from orquestaconvert import client 25 | from orquestaconvert.utils import yaml_utils 26 | 27 | 28 | # The file extension to use for backup files - these files are backups of the 29 | # original workflow and action metadata files, and normally be removed upon 30 | # success or failure. However, sending a SIGINT or SIGHUP to the process may 31 | # leave these files behind. 32 | BACKUP_EXTENSION = 'orquestaconvert.bak.yaml' 33 | # The file extension to use for temporary files. These files are the converted 34 | # workflow and action metadata files, and should also be removed upon success 35 | # or failure. However, it may sometimes be useful to keep these files, 36 | # especially when using the --force option. 37 | TMP_EXTENSION = 'orquesta.temp.yaml' 38 | 39 | 40 | class PackClient(object): 41 | def parser(self): 42 | parser = argparse.ArgumentParser(description='Convert all Mistral workflows in a pack') 43 | parser.add_argument('--validate', default=False, action='store_true', 44 | help='Validate the Orquesta workflow') 45 | parser.add_argument('--list-workflows', dest='workflow_type', type=str, 46 | help='List Mistral workflows in the pack and exit') 47 | parser.add_argument('--actions-dir', dest='actions_directory', default=None, type=str, 48 | help='The action directory to convert') 49 | return parser 50 | 51 | def get_workflow_files(self, workflow_type, directory=None): 52 | action_directory = directory if directory else 'actions' 53 | glob_string = '{}/*.yaml'.format(action_directory) 54 | a_files = glob.glob(glob_string) 55 | mistral_workflows = {} 56 | for a_file in a_files: 57 | with open(a_file, 'r') as f: 58 | action_data = yaml.safe_load(f.read()) 59 | runner = action_data.get('runner_type') 60 | 61 | if runner == workflow_type: 62 | mistral_workflows[a_file] = os.path.join( 63 | action_directory, 64 | *os.path.split(action_data.get('entry_point'))) 65 | 66 | return mistral_workflows 67 | 68 | def run(self, argv, output_stream, client=None): 69 | self.args, args = self.parser().parse_known_args(argv) 70 | wf_type = self.args.workflow_type 71 | directory = self.args.actions_directory 72 | if wf_type: 73 | for action, workflow in self.get_workflow_files(wf_type, directory).items(): 74 | output_stream.write("{} --> {}\n".format(action, workflow)) 75 | return 0 76 | 77 | if self.args.validate: 78 | args.append('--validate') 79 | filenames = self.get_workflow_files('orquesta', directory).values() 80 | total = 0 81 | for f in filenames: 82 | fargs = list(args) 83 | fargs.append(f) 84 | total += client.run(fargs, output_stream) 85 | return total 86 | 87 | filenames = self.get_workflow_files('mistral-v2', directory) 88 | 89 | exceptions = {} 90 | for a_f, m_f in six.iteritems(filenames): 91 | fargs = list(args) # make a copy 92 | fargs.append(m_f) 93 | 94 | # Get the backup filenames 95 | m_f_backup = '{}.{}'.format(m_f, BACKUP_EXTENSION) 96 | a_f_backup = '{}.{}'.format(a_f, BACKUP_EXTENSION) 97 | o_f = '{}.{}'.format(m_f, TMP_EXTENSION) # Orquesta workflow file 98 | 99 | # Convert the workflow and save the YAML string in output 100 | output = six.moves.StringIO() 101 | # In this next block of code we are attempting to create a filesystem 102 | # transaction, where we either want to commit all of the changes to 103 | # disk or none of the changes. 104 | # Usually you want to minimize what you put into a try block, so you 105 | # can catch different errors and handle them differently. However, in 106 | # this case, there are so many file operations that can fail that it 107 | # ends up nesting try/except/else blocks very deep, and exception 108 | # handlers simply do cleanup and re-raise the exception. 109 | # So instead, we do all of our work in the try block, and handle 110 | # cleaning up after the different failure conditions in the except 111 | # block, and handle success conditions in the else block. 112 | try: 113 | client.run(fargs, output) 114 | 115 | # Write the workflow file 116 | with open(o_f, 'w') as o_file: 117 | o_file.write(str(output.getvalue())) 118 | 119 | # If the backup files already exist, they were created by a 120 | # previous run. In that case, we want to preserve the original 121 | # backup file, because it is more likely a valid Mistral 122 | # workflow than whatever the current workflow file is. 123 | if not os.path.isfile(m_f_backup): 124 | # Move the existing workflow file to a backup 125 | os.rename(m_f, m_f_backup) 126 | 127 | if not os.path.isfile(a_f_backup): 128 | # Move the existing metadata file to a backup 129 | shutil.copy2(a_f, a_f_backup) 130 | 131 | # Read the file into a dict, tweak the runner_type, and write 132 | # the dict back into a string 133 | action_data, action_data_ruamel = yaml_utils.read_yaml(a_f) 134 | action_data_ruamel['runner_type'] = 'orquesta' 135 | write_data = yaml_utils.obj_to_yaml(action_data_ruamel) 136 | 137 | # Promote/move the new workflow file into place 138 | shutil.move(o_f, m_f) 139 | 140 | # Rewrite the metadata file with the tweaked runner_rtype 141 | with open(a_f, 'w') as a_file: 142 | a_file.write(write_data) 143 | # SUCCESS! 144 | 145 | except Exception as e: 146 | # Anything could have happened, so we check for bad conditions 147 | 148 | # If we have a backup workflow file, revert it 149 | if os.path.isfile(m_f_backup): 150 | # Remove the converted workflow file 151 | if os.path.isfile(m_f): 152 | os.remove(m_f) 153 | # Move the backup file back 154 | os.rename(m_f_backup, m_f) 155 | 156 | # If we have a backup action metadata file 157 | if os.path.isfile(a_f_backup): 158 | # Remove the converted metadata file 159 | if os.path.isfile(a_f): 160 | os.remove(a_f) 161 | # Move the backup file back 162 | os.rename(a_f_backup, a_f) 163 | 164 | # If we ever support just Python 3, we can add the exception 165 | # directly to the dictionary value: 166 | # 167 | # .append({'file': m_f, 'exception': e}) 168 | # 169 | exceptions.setdefault(str(e), []).append(m_f) 170 | else: 171 | # Remove the backup files 172 | os.remove(m_f_backup) 173 | os.remove(a_f_backup) 174 | 175 | if exceptions: 176 | sys.stderr.write("ERROR: Unable to convert all Mistral workflows.\n") 177 | for e, wfs in exceptions.items(): 178 | sys.stderr.write("ISSUE: {}\n".format(e)) 179 | sys.stderr.write("Affected files:\n") 180 | for wf in wfs: 181 | sys.stderr.write(" - {}\n".format(wf)) 182 | sys.stderr.write("\n") 183 | 184 | return len(exceptions) 185 | 186 | 187 | if __name__ == '__main__': 188 | sys.exit(PackClient().run(sys.argv[1:], sys.stdout, client=client.Client())) 189 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | # Licensed under the Apache License, Version 2.0 (the "License"); 2 | # you may not use this file except in compliance with the License. 3 | # You may obtain a copy of the License at 4 | # 5 | # http://www.apache.org/licenses/LICENSE-2.0 6 | # 7 | # Unless required by applicable law or agreed to in writing, software 8 | # distributed under the License is distributed on an "AS IS" BASIS, 9 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 10 | # See the License for the specific language governing permissions and 11 | # limitations under the License. 12 | 13 | SHELL := /bin/bash 14 | 15 | ROOT_DIR := $(shell dirname $(realpath $(lastword $(MAKEFILE_LIST)))) 16 | PYMODULE_DIR := $(ROOT_DIR) 17 | PYMODULE_TESTS_DIR ?= $(PYMODULE_DIR)/tests 18 | PYMODULE_NAME = $(shell python $(PYMODULE_DIR)/setup.py --name ) 19 | YAML_FILES := $(shell git ls-files '*.yaml' '*.yml') 20 | JSON_FILES := $(shell git ls-files '*.json') 21 | PY_FILES := $(shell git ls-files '*.py') 22 | TEST_COVERAGE_DIR ?= $(ROOT_DIR)/cover 23 | NOSE_OPTS := -s -v --exe --rednose --immediate --with-coverage --cover-inclusive --cover-erase --cover-package=$(PYMODULE_NAME) 24 | 25 | # Detect the OS 26 | # https://stackoverflow.com/a/14777895 27 | ifeq ($(OS),Windows_NT) 28 | detected_OS := Windows 29 | else 30 | detected_OS := $(shell sh -c 'uname -s 2>/dev/null || echo no') 31 | endif 32 | 33 | # For a list of unames, see https://stackoverflow.com/a/14777895 34 | BSD_USERLANDS := Darwin FreeBSD NetBSD DragonFly 35 | # Note: GNU/kFreeBSD is the Debian GNU userland running on the BSD kernel 36 | GNU_USERLANDS := Linux GNU GNU/kFreeBSD 37 | 38 | # From https://stackoverflow.com/a/27335439 39 | # If the result of searching for the detected_OS value in BSD_USERLANDS is not 40 | # empty, then run the BSD sed block, run the same check for GNU_USERLANDS, or 41 | # if nothing matches, then we simply guess how to tell sed to modify files 42 | # in-place. 43 | ifneq ($(filter $(detected_OS),$(BSD_USERLANDS)),) 44 | # BSD sed requires an extension after the -i flag for the extension of the 45 | # backup file it creates (so...it's not actually "in-place"). Here we 46 | # disable that by passing in an empty string, and telling sed to go back 47 | # to parsing it's normal expressions by adding the -e flag. 48 | SED_INPLACE_FLAGS := -i'' -e 49 | else ifneq ($(filter $(detected_OS),$(GNU_USERLANDS)),) 50 | # Asking sed to replace files in-place is a little easier with GNU sed 51 | SED_INPLACE_FLAGS := -i 52 | else 53 | # All bets are off. We need to guess. 54 | SED_INPLACE_FLAGS := -i 55 | endif 56 | 57 | # Virtual Environment 58 | VIRTUALENV_DIR ?= $(ROOT_DIR)/virtualenv 59 | ORQUESTA_DIR ?= $(VIRTUALENV_DIR)/orquesta 60 | 61 | VIRTUALENV_FLAGS := --no-site-packages 62 | # https://stackoverflow.com/a/22105036/1134951 63 | PYV=$(shell python -c "import sys;t='{v[0]}.{v[1]}'.format(v=list(sys.version_info[:2]));sys.stdout.write(t)"); 64 | ifeq ($(PYV),2.7) 65 | VIRTUALENV_FLAGS += -p python2.7 66 | endif 67 | 68 | 69 | # Run all targets 70 | .PHONY: all 71 | all: requirements lint test # test-coveralls 72 | 73 | .PHONY: clean 74 | clean: .clean-virtualenv .clean-test-coverage .clean-pyc 75 | 76 | .PHONY: lint 77 | lint: requirements flake8 pylint json-lint yaml-lint 78 | 79 | .PHONY: flake8 80 | flake8: requirements .flake8 81 | 82 | .PHONY: pylint 83 | pylint: requirements .pylint 84 | 85 | .PHONY: json-lint 86 | pylint: requirements .json-lint 87 | 88 | .PHONY: yaml-lint 89 | pylint: requirements .yaml-lint 90 | 91 | .PHONY: test 92 | test: requirements .test 93 | 94 | .PHONY: test-coverage-html 95 | test-coverage-html: requirements .test-coverage-html 96 | 97 | .PHONY: test-coveralls 98 | test-coveralls: requirements .test-coveralls 99 | 100 | .PHONY: codecov 101 | codecov: requirements .codecov 102 | 103 | .PHONY: clean-test-coverage 104 | clean-test-coverage: .clean-test-coverage 105 | 106 | # list all makefile targets 107 | .PHONY: list 108 | list: 109 | @$(MAKE) -pRrq -f $(lastword $(MAKEFILE_LIST)) : 2>/dev/null | awk -v RS= -F: '/^# File/,/^# Finished Make data base/ {if ($$1 !~ "^[#.]") {print $$1}}' | sort | egrep -v -e '^[^[:alnum:]]' -e '^$@$$' | xargs 110 | 111 | 112 | .PHONY: .clean-pyc 113 | .clean-pyc: 114 | @echo "==================== clean *.pyc ====================" 115 | find $(ROOT_DIR) -name 'virtualenv' -prune -or -name '.git' -or -type f -name "*.pyc" -print | xargs --no-run-if-empty rm 116 | 117 | 118 | .PHONY: virtualenv 119 | virtualenv: $(VIRTUALENV_DIR)/bin/activate 120 | $(VIRTUALENV_DIR)/bin/activate: 121 | @echo "==================== virtualenv ====================" 122 | test -d $(VIRTUALENV_DIR) || virtualenv $(VIRTUALENV_FLAGS) $(VIRTUALENV_DIR) 123 | # Setup PYTHONPATH in bash activate script... 124 | # Delete existing entries (if any) 125 | sed $(SED_INPLACE_FLAGS) '/_OLD_PYTHONPATHp/d' $(VIRTUALENV_DIR)/bin/activate 126 | sed $(SED_INPLACE_FLAGS) '/PYTHONPATH=/d' $(VIRTUALENV_DIR)/bin/activate 127 | sed $(SED_INPLACE_FLAGS) '/export PYTHONPATH/d' $(VIRTUALENV_DIR)/bin/activate 128 | echo '_OLD_PYTHONPATH=$$PYTHONPATH' >> $(VIRTUALENV_DIR)/bin/activate 129 | echo 'PYTHONPATH=${ROOT_DIR}' >> $(VIRTUALENV_DIR)/bin/activate 130 | echo 'export PYTHONPATH' >> $(VIRTUALENV_DIR)/bin/activate 131 | touch $(VIRTUALENV_DIR)/bin/activate 132 | 133 | 134 | .PHONY: .clean-virtualenv 135 | .clean-virtualenv: 136 | @echo "==================== cleaning virtualenv ====================" 137 | rm -rf $(VIRTUALENV_DIR) 138 | 139 | 140 | .PHONY: requirements 141 | requirements: virtualenv 142 | @echo 143 | @echo "==================== requirements ====================" 144 | @echo 145 | . $(VIRTUALENV_DIR)/bin/activate; \ 146 | $(VIRTUALENV_DIR)/bin/pip install --upgrade pip; \ 147 | $(VIRTUALENV_DIR)/bin/pip install --cache-dir $(HOME)/.pip-cache -q -r $(PYMODULE_DIR)/requirements.txt -r $(PYMODULE_DIR)/requirements-test.txt -r $(PYMODULE_DIR)/requirements-orquesta.txt; 148 | 149 | 150 | .PHONY: update-orquesta-requirements 151 | update-orquesta-requirements: virtualenv 152 | test -d $(ORQUESTA_DIR) || git clone "https://github.com/StackStorm/orquesta" $(ORQUESTA_DIR) 153 | cp $(ORQUESTA_DIR)/requirements.txt requirements-orquesta.txt 154 | 155 | 156 | .PHONY: .flake8 157 | .flake8: 158 | @echo 159 | @echo "==================== flake8 ====================" 160 | @echo 161 | . $(VIRTUALENV_DIR)/bin/activate; \ 162 | for py in $(PY_FILES); do \ 163 | echo "Checking $$py"; \ 164 | flake8 --config $(PYMODULE_DIR)/lint-configs/python/.flake8 $$py || exit $?; \ 165 | done 166 | 167 | 168 | .PHONY: .pylint 169 | .pylint: 170 | @echo 171 | @echo "==================== pylint ====================" 172 | @echo 173 | . $(VIRTUALENV_DIR)/bin/activate; \ 174 | echo $(PY_FILES) | xargs python -m pylint -E --rcfile=$(PYMODULE_DIR)/lint-configs/python/.pylintrc && echo "--> No pylint issues found." || exit $? 175 | 176 | 177 | .PHONY: .json-lint 178 | .json-lint: 179 | @echo 180 | @echo "==================== json-lint ====================" 181 | @echo 182 | . $(VIRTUALENV_DIR)/bin/activate; \ 183 | for json in $(JSON_FILES); do \ 184 | echo "Checking $$json"; \ 185 | python -mjson.tool $$json > /dev/null || exit $?; \ 186 | done 187 | 188 | 189 | .PHONY: .yaml-lint 190 | .yaml-lint: 191 | @echo 192 | @echo "==================== yaml-lint ====================" 193 | @echo 194 | . $(VIRTUALENV_DIR)/bin/activate; \ 195 | for yaml in $(YAML_FILES); do \ 196 | echo "Checking $$yaml"; \ 197 | python -c "import yaml; yaml.safe_load(open('$$yaml', 'r'))" || exit $?; \ 198 | done 199 | 200 | 201 | .PHONY: .test 202 | .test: 203 | @echo 204 | @echo "==================== test ====================" 205 | @echo 206 | . $(VIRTUALENV_DIR)/bin/activate; \ 207 | if [ -d "$(PYMODULE_TESTS_DIR)" ]; then \ 208 | nosetests $(NOSE_OPTS) $(PYMODULE_TESTS_DIR) || exit $?; \ 209 | else \ 210 | echo "Tests directory not found: $(PYMODULE_TESTS_DIR)";\ 211 | fi; 212 | 213 | 214 | .PHONY: .test-coverage-html 215 | .test-coverage-html: 216 | @echo 217 | @echo "==================== test-coverage-html ====================" 218 | @echo 219 | . $(VIRTUALENV_DIR)/bin/activate; \ 220 | if [ -d "$(PYMODULE_TESTS_DIR)" ]; then \ 221 | nosetests $(NOSE_OPTS) --cover-html $(PYMODULE_TESTS_DIR) || exit $?; \ 222 | else \ 223 | echo "Tests directory not found: $(PYMODULE_TESTS_DIR)";\ 224 | fi; 225 | 226 | 227 | .PHONY: .test-test-coverage-html 228 | .test-test-coverage-html: 229 | @echo 230 | @echo "================== test-test-coverage-html ==================" 231 | @echo 232 | . $(VIRTUALENV_DIR)/bin/activate; \ 233 | if [ -d "$(PYMODULE_TESTS_DIR)" ]; then \ 234 | nosetests $(NOSE_OPTS),tests --cover-tests --cover-html $(PYMODULE_TESTS_DIR) || exit $?; \ 235 | else \ 236 | echo "Tests directory not found: $(PYMODULE_TESTS_DIR)";\ 237 | fi; 238 | 239 | 240 | .PHONY: .test-coveralls 241 | .test-coveralls: 242 | @echo 243 | @echo "==================== test-coveralls ====================" 244 | @echo 245 | . $(VIRTUALENV_DIR)/bin/activate; \ 246 | if [ ! -z "$$COVERALLS_REPO_TOKEN" ]; then \ 247 | coveralls; \ 248 | else \ 249 | echo "COVERALLS_REPO_TOKEN env variable is not set! Skipping test coverage submission to coveralls.io."; \ 250 | fi; 251 | 252 | 253 | .PHONY: .clean-test-coverage 254 | .clean-test-coverage: 255 | @echo 256 | @echo "==================== clean-test-coverage ====================" 257 | @echo 258 | rm -rf $(TEST_COVERAGE_DIR) 259 | rm -f $(PYMODULE_DIR)/.coverage 260 | 261 | .PHONY: .codecov 262 | .codecov: 263 | @echo 264 | @echo "==================== codecov ====================" 265 | @echo 266 | . $(VIRTUALENV_DIR)/bin/activate; \ 267 | codecov 268 | -------------------------------------------------------------------------------- /tests/integration/test_convert_pack.py: -------------------------------------------------------------------------------- 1 | # Licensed under the Apache License, Version 2.0 (the "License"); 2 | # you may not use this file except in compliance with the License. 3 | # You may obtain a copy of the License at 4 | # 5 | # http://www.apache.org/licenses/LICENSE-2.0 6 | # 7 | # Unless required by applicable law or agreed to in writing, software 8 | # distributed under the License is distributed on an "AS IS" BASIS, 9 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 10 | # See the License for the specific language governing permissions and 11 | # limitations under the License. 12 | 13 | from __future__ import print_function 14 | 15 | import filecmp 16 | import os 17 | import six 18 | import sys 19 | 20 | from orquestaconvert import client 21 | from orquestaconvert import pack_client 22 | 23 | from tests import base_test_case 24 | 25 | 26 | class PackClientRunTestCase(base_test_case.BasePackClientRunTestCase): 27 | __test__ = True 28 | 29 | def setUp(self): 30 | super(PackClientRunTestCase, self).setUp() 31 | 32 | self.client = client.Client() 33 | self.pack_client = pack_client.PackClient() 34 | 35 | def _validate_dirs(self, dir1, dir2): 36 | '''Make sure the directories are the same''' 37 | dirdiff = filecmp.dircmp(dir1, dir2) 38 | 39 | for diff_file in dirdiff.diff_files: 40 | sys.__stdout__.write(os.system("diff {} {}\n".format( 41 | os.path.join(self.m_actions_dir, diff_file), 42 | os.path.join(self.o_actions_dir, diff_file), 43 | ))) 44 | 45 | self.assertEqual(dirdiff.diff_files, []) 46 | self.assertEqual(dirdiff.funny_files, []) 47 | 48 | def test_list_mistral_workflows_in_pack(self): 49 | args = ['--list-workflows=mistral-v2', '--actions-dir={}'.format(self.m_actions_dir)] 50 | self.pack_client.run(args, self.stdout, client=self.client) 51 | 52 | out = self.stdout.getvalue() 53 | 54 | self.assertEqual(len([line for line in out.split('\n') if line]), len(self.action_files)) 55 | for action_file, wf_file in six.iteritems(self.action_wfs): 56 | self.assertIn('{action_file} --> {wf_file}\n' 57 | .format(action_file=action_file, wf_file=wf_file), 58 | out) 59 | self.assertEqual(self.stderr.getvalue(), '') 60 | 61 | self._validate_dirs(self.p_actions_dir, self.m_actions_dir) 62 | 63 | def test_list_orquesta_workflows_in_mistral_directory(self): 64 | args = ['--list-workflows=orquesta', '--actions-dir={}'.format(self.m_actions_dir)] 65 | self.pack_client.run(args, self.stdout, client=self.client) 66 | 67 | self.assertEqual(self.stdout.getvalue(), '') 68 | self.assertEqual(self.stderr.getvalue(), '') 69 | 70 | self._validate_dirs(self.p_actions_dir, self.m_actions_dir) 71 | 72 | def test_list_orquesta_workflows_in_pack(self): 73 | args = ['--list-workflows=orquesta', '--actions-dir={}'.format(self.o_actions_dir)] 74 | self.pack_client.run(args, self.stdout, client=self.client) 75 | 76 | out = self.stdout.getvalue() 77 | self.assertEqual(self.stderr.getvalue(), '') 78 | for action_file, wf_file in six.iteritems(self.orquesta_action_wfs): 79 | self.assertIn('{action_file} --> {wf_file}\n' 80 | .format(action_file=action_file, wf_file=wf_file), 81 | out) 82 | 83 | def test_partially_convert_pack(self): 84 | args = ['-e', 'yaql', '--actions-dir={}'.format(self.m_actions_dir)] 85 | result = self.pack_client.run(args, self.stdout, client=self.client) 86 | 87 | # Check the exit code 88 | self.assertEqual(1, result) 89 | 90 | # Check the stdout and stderr 91 | self.assertEqual(self.stdout.getvalue(), '') 92 | err = self.stderr.getvalue() 93 | self.assertIn("ERROR: Unable to convert all Mistral workflows.\n", err) 94 | self.assertIn( 95 | "Cannot convert continue-on (<% $.foo = 'continue' %>) and break-on " 96 | "({{{{ _.bar = \"BREAK\" }}}}) expressions that are different types in task " 97 | "'test-error-undo-retry'\n" 98 | "Affected files:\n" 99 | " - {m_wfs_dir}/mistral-fail-retry-continue-and-break-on.yaml\n" 100 | "\n".format(m_wfs_dir=self.m_wfs_dir), 101 | err) 102 | 103 | # Check the failing actions failed conversion 104 | for action in self.action_failing_files: 105 | m_actual_action = self.get_fixture_content('pack/actions/{}'.format(action)) 106 | m_expected_action = self.get_fixture_content('pack/pristine_actions/{}'.format(action)) 107 | 108 | self.assertEqual(m_expected_action, m_actual_action) 109 | 110 | m_actual_wf = self.get_fixture_content('pack/actions/workflows/{}'.format(action)) 111 | m_expected_wf = self.get_fixture_content('pack/pristine_actions/workflows/{}' 112 | .format(action)) 113 | 114 | self.assertEqual(m_expected_wf, m_actual_wf) 115 | 116 | # Check all of the remaining actions worked 117 | for o_action in self.action_passing_files: 118 | o_actual_action = self.get_fixture_content('pack/actions/{}'.format(o_action)) 119 | o_expected_action = self.get_fixture_content('pack/o_actions/{}'.format(o_action)) 120 | 121 | self.assertEqual(o_expected_action, o_actual_action) 122 | 123 | o_actual_wf = self.get_fixture_content('pack/actions/workflows/{}'.format(o_action)) 124 | o_expected_wf = self.get_fixture_content('pack/o_actions/workflows/{}'.format(o_action)) 125 | 126 | self.assertEqual(o_expected_wf, o_actual_wf) 127 | 128 | self._validate_dirs(self.m_actions_dir, self.o_actions_dir) 129 | 130 | def test_completely_convert_pack(self): 131 | for afile in self.action_failing_files: 132 | os.remove(os.path.join(self.m_actions_dir, afile)) 133 | os.remove(os.path.join(self.m_wfs_dir, afile)) 134 | 135 | args = ['-e', 'yaql', '--actions-dir={}'.format(self.m_actions_dir)] 136 | result = self.pack_client.run(args, self.stdout, client=self.client) 137 | 138 | self.assertEqual(0, result) 139 | 140 | self.assertEqual(self.stdout.getvalue(), '') 141 | self.assertEqual(self.stderr.getvalue(), '') 142 | 143 | for o_wf in self.action_passing_files: 144 | actual = self.get_fixture_content('pack/actions/workflows/{}'.format(o_wf)) 145 | expected = self.get_fixture_content('pack/o_actions/workflows/{}'.format(o_wf)) 146 | 147 | self.assertEqual(expected, actual) 148 | 149 | self._validate_dirs(self.m_actions_dir, self.o_actions_dir) 150 | 151 | def test_validate_nothing(self): 152 | args = ['--validate', '--actions-dir={}'.format(self.m_actions_dir)] 153 | result = self.pack_client.run(args, self.stdout, client=self.client) 154 | 155 | self.assertEqual(self.stdout.getvalue(), '') 156 | self.assertEqual(self.stderr.getvalue(), '') 157 | 158 | self.assertEqual(0, result) 159 | 160 | self._validate_dirs(self.p_actions_dir, self.m_actions_dir) 161 | 162 | def test_validate_verbose_nothing(self): 163 | args = ['--validate', '--verbose', '--actions-dir={}'.format(self.m_actions_dir)] 164 | result = self.pack_client.run(args, self.stdout, client=self.client) 165 | 166 | self.assertEqual(self.stdout.getvalue(), '') 167 | self.assertEqual(self.stderr.getvalue(), '') 168 | 169 | self.assertEqual(0, result) 170 | 171 | self._validate_dirs(self.p_actions_dir, self.m_actions_dir) 172 | 173 | def test_validate_orquesta(self): 174 | files = ['mistral-test-cancel.yaml', os.path.join('workflows', 'mistral-test-cancel.yaml')] 175 | before_dirhash = self._hash_directory(self.o_actions_dir, files) 176 | 177 | args = ['--validate', '--actions-dir={}'.format(self.o_actions_dir)] 178 | result = self.pack_client.run(args, self.stdout, client=self.client) 179 | 180 | after_dirhash = self._hash_directory(self.o_actions_dir, files) 181 | 182 | self.assertEqual(self.stdout.getvalue(), '') 183 | self.assertEqual(self.stderr.getvalue(), '') 184 | 185 | self.assertEqual(0, result) 186 | 187 | self.assertEqual(before_dirhash, after_dirhash) 188 | 189 | def test_validate_verbose_orquesta(self): 190 | files = ['mistral-test-cancel.yaml', os.path.join('workflows', 'mistral-test-cancel.yaml')] 191 | before_dirhash = self._hash_directory(self.o_actions_dir, files) 192 | 193 | args = ['--validate', '--verbose', '--actions-dir={}'.format(self.o_actions_dir)] 194 | result = self.pack_client.run(args, self.stdout, client=self.client) 195 | 196 | after_dirhash = self._hash_directory(self.o_actions_dir, files) 197 | 198 | self.assertEqual(0, result) 199 | 200 | self.assertEqual(self.stderr.getvalue(), '') 201 | 202 | out = self.stdout.getvalue() 203 | for action in self.action_passing_files: 204 | wf = os.path.join(self.o_wfs_dir, action) 205 | self.assertIn("Successfully validated workflow from {}\n".format(wf), out) 206 | 207 | self.assertEqual(before_dirhash, after_dirhash) 208 | 209 | def test_validate_partially_converted_pack(self): 210 | self.test_partially_convert_pack() 211 | 212 | self.setup_captures() 213 | 214 | args = ['--validate', '--actions-dir={}'.format(self.m_actions_dir)] 215 | result = self.pack_client.run(args, self.stdout, client=self.client) 216 | 217 | self.assertEqual(self.stdout.getvalue(), '') 218 | self.assertEqual(self.stderr.getvalue(), '') 219 | 220 | self.assertEqual(0, result) 221 | 222 | self._validate_dirs(self.m_actions_dir, self.o_actions_dir) 223 | 224 | def test_validate_verbose_partially_converted_pack(self): 225 | self.test_partially_convert_pack() 226 | 227 | self.setup_captures() 228 | 229 | args = ['--validate', '--verbose', '--actions-dir={}'.format(self.m_actions_dir)] 230 | result = self.pack_client.run(args, self.stdout, client=self.client) 231 | 232 | self.assertEqual(0, result) 233 | 234 | self.assertEqual(self.stderr.getvalue(), '') 235 | 236 | out = self.stdout.getvalue() 237 | for o_action in self.action_passing_files: 238 | o_wf = os.path.join(self.m_wfs_dir, o_action) 239 | self.assertIn("Successfully validated workflow from {}\n".format(o_wf), out) 240 | 241 | self._validate_dirs(self.m_actions_dir, self.o_actions_dir) 242 | -------------------------------------------------------------------------------- /orquestaconvert/specs/mistral/v2/tasks.py: -------------------------------------------------------------------------------- 1 | # Copyright 2019 Extreme Networks, Inc. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | import logging 16 | import re 17 | import six 18 | from six.moves import queue 19 | 20 | from orquesta import exceptions as exc 21 | from orquesta.expressions import base as expr_base 22 | from orquesta.specs import types as spec_types 23 | from orquesta.utils import dictionary as dict_util 24 | from orquesta.utils import jsonify as json_util 25 | from orquestaconvert.specs.mistral.v2 import base as mistral_spec_base 26 | from orquestaconvert.specs.mistral.v2 import policies as policy_models 27 | 28 | 29 | LOG = logging.getLogger(__name__) 30 | 31 | 32 | ON_CLAUSE_SCHEMA = { 33 | "oneOf": [spec_types.NONEMPTY_STRING, spec_types.UNIQUE_STRING_OR_ONE_KEY_DICT_LIST] 34 | } 35 | 36 | 37 | class TaskDefaultsSpec(mistral_spec_base.Spec): 38 | _schema = { 39 | "type": "object", 40 | "properties": { 41 | "concurrency": policy_models.CONCURRENCY_SCHEMA, 42 | "keep-result": policy_models.KEEP_RESULT_SCHEMA, 43 | "retry": policy_models.RetrySpec, 44 | "safe-rerun": policy_models.SAFE_RERUN_SCHEMA, 45 | "wait-before": policy_models.WAIT_BEFORE_SCHEMA, 46 | "wait-after": policy_models.WAIT_AFTER_SCHEMA, 47 | "pause-before": policy_models.PAUSE_BEFORE_SCHEMA, 48 | "target": policy_models.TARGET_SCHEMA, 49 | "timeout": policy_models.TIMEOUT_SCHEMA, 50 | "on-complete": ON_CLAUSE_SCHEMA, 51 | "on-success": ON_CLAUSE_SCHEMA, 52 | "on-error": ON_CLAUSE_SCHEMA, 53 | }, 54 | "additionalProperties": False, 55 | } 56 | 57 | 58 | class TaskSpec(mistral_spec_base.Spec): 59 | _schema = { 60 | "type": "object", 61 | "properties": { 62 | "join": {"oneOf": [{"enum": ["all"]}, spec_types.POSITIVE_INTEGER]}, 63 | "with-items": {"oneOf": [spec_types.NONEMPTY_STRING, spec_types.UNIQUE_STRING_LIST]}, 64 | "concurrency": policy_models.CONCURRENCY_SCHEMA, 65 | "action": spec_types.NONEMPTY_STRING, 66 | "workflow": spec_types.NONEMPTY_STRING, 67 | "input": spec_types.NONEMPTY_DICT, 68 | "publish": spec_types.NONEMPTY_DICT, 69 | "publish-on-error": spec_types.NONEMPTY_DICT, 70 | "keep-result": policy_models.KEEP_RESULT_SCHEMA, 71 | "retry": policy_models.RetrySpec, 72 | "safe-rerun": policy_models.SAFE_RERUN_SCHEMA, 73 | "wait-before": policy_models.WAIT_BEFORE_SCHEMA, 74 | "wait-after": policy_models.WAIT_AFTER_SCHEMA, 75 | "pause-before": policy_models.PAUSE_BEFORE_SCHEMA, 76 | "target": policy_models.TARGET_SCHEMA, 77 | "timeout": policy_models.TIMEOUT_SCHEMA, 78 | "on-complete": ON_CLAUSE_SCHEMA, 79 | "on-success": ON_CLAUSE_SCHEMA, 80 | "on-error": ON_CLAUSE_SCHEMA, 81 | }, 82 | "additionalProperties": False, 83 | "anyOf": [ 84 | {"not": {"type": "object", "required": ["action", "workflow"]}}, 85 | { 86 | "oneOf": [ 87 | {"type": "object", "required": ["action"]}, 88 | {"type": "object", "required": ["workflow"]}, 89 | ] 90 | }, 91 | ], 92 | } 93 | 94 | _context_evaluation_sequence = [ 95 | "join", 96 | "with-items", 97 | "concurrency", 98 | "action", 99 | "workflow", 100 | "input", 101 | "publish", 102 | "publish-on-error", 103 | "on-complete", 104 | "on-success", 105 | "on-error", 106 | ] 107 | 108 | _context_inputs = ["publish"] 109 | 110 | def has_items(self): 111 | return hasattr(self, "with-items") and getattr(self, "with-items", None) is not None 112 | 113 | def get_items_spec(self): 114 | return getattr(self, "with-items", None) 115 | 116 | def has_join(self): 117 | return hasattr(self, "join") and self.join 118 | 119 | def has_retry(self): 120 | return hasattr(self, "retry") and self.retry 121 | 122 | def render(self, in_ctx): 123 | action_specs = [] 124 | 125 | if self.has_items(): 126 | raise NotImplementedError("Task with items is not implemented.") 127 | 128 | action_spec = { 129 | "action": expr_base.evaluate(self.action, in_ctx), 130 | "input": expr_base.evaluate(getattr(self, "input", {}), in_ctx), 131 | } 132 | 133 | action_specs.append(action_spec) 134 | 135 | return self, action_specs 136 | 137 | def finalize_context(self, next_task_name, task_transition_meta, in_ctx): 138 | criteria = task_transition_meta[3].get("criteria") or [] 139 | expected_criteria_pattern = r"<\% task_status\(\w+\) in \['succeeded'\] \%>" 140 | new_ctx = {} 141 | errors = [] 142 | 143 | if not re.match(expected_criteria_pattern, criteria[0]): 144 | return in_ctx, new_ctx, errors 145 | 146 | task_publish_spec = getattr(self, "publish") or {} 147 | 148 | try: 149 | new_ctx = { 150 | var_name: expr_base.evaluate(var_expr, in_ctx) 151 | for var_name, var_expr in six.iteritems(task_publish_spec) 152 | } 153 | except exc.ExpressionEvaluationException as e: 154 | errors.append(str(e)) 155 | 156 | out_ctx = dict_util.merge_dicts(in_ctx, new_ctx, overwrite=True) 157 | 158 | for key in list(out_ctx.keys()): 159 | if key.startswith("__"): 160 | out_ctx.pop(key) 161 | 162 | return out_ctx, new_ctx, errors 163 | 164 | 165 | class TaskMappingSpec(mistral_spec_base.MappingSpec): 166 | _schema = {"type": "object", "minProperties": 1, "patternProperties": {r"^\w+$": TaskSpec}} 167 | 168 | def get_task(self, task_name): 169 | return self[task_name] 170 | 171 | def get_next_tasks(self, task_name, *args, **kwargs): 172 | task_spec = self.get_task(task_name) 173 | conditions = kwargs.get("conditions") 174 | 175 | if not conditions: 176 | conditions = ["on-complete", "on-error", "on-success"] 177 | 178 | next_tasks = [] 179 | 180 | for condition in conditions: 181 | for task in getattr(task_spec, condition) or []: 182 | next_tasks.append( 183 | list(task.items())[0] + (condition,) 184 | # The task attribute is either a name or 185 | # it's a dict that contains name and expr. 186 | if isinstance(task, dict) 187 | else (task, None, condition) 188 | ) 189 | 190 | return sorted(next_tasks, key=lambda x: x[0]) 191 | 192 | def get_prev_tasks(self, task_name, *args, **kwargs): 193 | prev_tasks = [] 194 | conditions = kwargs.get("conditions") 195 | 196 | for name, task_spec in six.iteritems(self): 197 | for next_task in self.get_next_tasks(name, conditions=conditions): 198 | if task_name == next_task[0]: 199 | prev_tasks.append((name, next_task[1], next_task[2])) 200 | 201 | return sorted(prev_tasks, key=lambda x: x[0]) 202 | 203 | def get_start_tasks(self): 204 | start_tasks = [ 205 | (task_name, None, None) 206 | for task_name in self.keys() 207 | if not self.get_prev_tasks(task_name) 208 | ] 209 | 210 | return sorted(start_tasks, key=lambda x: x[0]) 211 | 212 | def is_join_task(self, task_name): 213 | task_spec = self.get_task(task_name) 214 | 215 | return task_spec.join is not None 216 | 217 | def is_split_task(self, task_name): 218 | return not self.is_join_task(task_name) and len(self.get_prev_tasks(task_name)) > 1 219 | 220 | def in_cycle(self, task_name): 221 | traversed = [] 222 | q = queue.Queue() 223 | 224 | for task in self.get_next_tasks(task_name): 225 | q.put(task[0]) 226 | 227 | while not q.empty(): 228 | next_task_name = q.get() 229 | 230 | # If the next task matches the original task, then it's in a loop. 231 | if next_task_name == task_name: 232 | return True 233 | 234 | # If the next task has already been traversed but didn't match the 235 | # original task, then there's a loop but the original task is not 236 | # in the loop. 237 | if next_task_name in traversed: 238 | return False 239 | 240 | for task in self.get_next_tasks(next_task_name): 241 | q.put(task[0]) 242 | 243 | traversed.append(next_task_name) 244 | 245 | return False 246 | 247 | def has_cycles(self): 248 | for task_name, task_spec in six.iteritems(self): 249 | if self.in_cycle(task_name): 250 | return True 251 | 252 | return False 253 | 254 | def inspect_context(self, parent=None): 255 | ctxs = {} 256 | errors = [] 257 | parent_ctx = parent.get("ctx", []) if parent else [] 258 | rolling_ctx = list(set(parent_ctx)) 259 | q = queue.Queue() 260 | 261 | for task in self.get_start_tasks(): 262 | q.put((task[0], json_util.deepcopy(rolling_ctx))) 263 | 264 | while not q.empty(): 265 | task_name, task_ctx = q.get() 266 | 267 | if not task_ctx: 268 | task_ctx = ctxs.get(task_name, []) 269 | 270 | task_spec = self.get_task(task_name) 271 | 272 | spec_path = parent.get("spec_path") + "." + task_name 273 | schema_path = parent.get("schema_path") + "." + "properties." + task_name 274 | 275 | task_parent = {"ctx": task_ctx, "spec_path": spec_path, "schema_path": schema_path} 276 | 277 | result = task_spec.inspect_context(parent=task_parent) 278 | errors.extend(result[0]) 279 | task_ctx = list(set(task_ctx + result[1])) 280 | rolling_ctx = list(set(rolling_ctx + task_ctx)) 281 | 282 | for task in self.get_next_tasks(task_name): 283 | next_task_spec = self.get_task(task[0]) 284 | 285 | if not next_task_spec.has_join(): 286 | q.put((task[0], task_ctx)) 287 | else: 288 | ctxs[task[0]] = list(set(ctxs.get(task[0], []) + task_ctx)) 289 | q.put((task[0], None)) 290 | 291 | return (errors, rolling_ctx) 292 | --------------------------------------------------------------------------------