├── .flake8 ├── .github ├── ISSUE_TEMPLATE │ ├── bug_report.md │ └── feature_request.md └── PULL_REQUEST_TEMPLATE.md ├── .gitignore ├── .kokoro ├── kokoro_build.sh ├── presubmit.cfg └── run_tests.sh ├── .pre-commit-config.yaml ├── .travis.yml ├── .travis └── o2a-build-artifacts-sa.json.enc ├── CODE_OF_CONDUCT.md ├── CONTRIBUTING.md ├── LICENSE ├── MANIFEST.in ├── README.md ├── bin ├── o2a ├── o2a-confirm ├── o2a-generate-dependency-graph ├── o2a-generate-index ├── o2a-package-upload ├── o2a-package-upload-test ├── o2a-run-all-configurations ├── o2a-run-all-conversions ├── o2a-run-all-unit-tests ├── o2a-run-sys-test ├── o2a-run-sys-test-complete ├── o2a-validate-all-workflows ├── o2a-validate-workflows ├── o2a_lib-package-upload └── o2a_lib-package-upload-test ├── codecov.yaml ├── dataproc ├── example-map-reduce-job.sh └── oozie-5.2.sh ├── examples ├── .gitignore ├── advancedflow │ ├── configuration.template.properties │ ├── hdfs │ │ └── workflow.xml │ └── job.properties ├── childwf │ ├── configuration.template.properties │ ├── hdfs │ │ ├── input │ │ │ └── Word_Count_input.txt │ │ └── workflow.xml │ └── job.properties ├── decision │ ├── configuration.template.properties │ ├── hdfs │ │ └── workflow.xml │ └── job.properties ├── demo │ ├── configuration.template.properties │ ├── hdfs │ │ ├── pig │ │ │ ├── id.pig │ │ │ └── input-data │ │ │ │ └── test-data.txt │ │ └── workflow.xml │ └── job.properties ├── distcp │ ├── configuration.template.properties │ ├── hdfs │ │ ├── input-data │ │ │ └── test-data.txt │ │ └── workflow.xml │ └── job.properties ├── el │ ├── hdfs │ │ └── workflow.xml │ └── job.properties ├── email │ ├── configuration.template.properties │ ├── hdfs │ │ └── workflow.xml │ └── job.properties ├── fs │ ├── configuration.template.properties │ ├── hdfs │ │ └── workflow.xml │ └── job.properties ├── git │ ├── configuration.template.properties │ ├── hdfs │ │ └── workflow.xml │ └── job.properties ├── hive │ ├── configuration.template.properties │ ├── hdfs │ │ ├── input-data │ │ │ └── table │ │ ├── script.q │ │ └── workflow.xml │ └── job.properties ├── java │ ├── configuration.template.properties │ ├── hdfs │ │ └── workflow.xml │ └── job.properties ├── mapreduce │ ├── configuration.template.properties │ ├── hdfs │ │ ├── input │ │ │ └── Word_Count_input.txt │ │ ├── job-queue-conf.xml │ │ └── workflow.xml │ └── job.properties ├── pig │ ├── configuration.template.properties │ ├── hdfs │ │ ├── id.pig │ │ ├── input-data │ │ │ └── test-data.txt │ │ ├── test_dir │ │ │ ├── test.txt │ │ │ ├── test2.zip │ │ │ └── test3.zip │ │ └── workflow.xml │ └── job.properties ├── shell │ ├── configuration.template.properties │ ├── hdfs │ │ └── workflow.xml │ └── job.properties ├── spark │ ├── configuration.template.properties │ ├── hdfs │ │ └── workflow.xml │ └── job.properties ├── ssh │ ├── hdfs │ │ └── workflow.xml │ └── job.properties └── subwf │ ├── configuration.template.properties │ ├── hdfs │ └── workflow.xml │ └── job.properties ├── images ├── childwf_with_notifications.png ├── childwf_without_notifications.png ├── o2a-dependencies.png └── o2a-dependency-cycles.png ├── mypy.ini ├── o2a ├── __init__.py ├── converter │ ├── __init__.py │ ├── constants.py │ ├── exceptions.py │ ├── mappers.py │ ├── oozie_converter.py │ ├── oozie_node.py │ ├── property_parser.py │ ├── relation.py │ ├── renderers.py │ ├── task.py │ ├── task_group.py │ ├── workflow.py │ └── workflow_xml_parser.py ├── definitions.py ├── mappers │ ├── __init__.py │ ├── action_mapper.py │ ├── base_mapper.py │ ├── decision_mapper.py │ ├── distcp_mapper.py │ ├── dummy_mapper.py │ ├── email_mapper.py │ ├── end_mapper.py │ ├── extensions │ │ └── prepare_mapper_extension.py │ ├── fork_mapper.py │ ├── fs_mapper.py │ ├── git_mapper.py │ ├── hive_mapper.py │ ├── java_mapper.py │ ├── join_mapper.py │ ├── kill_mapper.py │ ├── mapreduce_mapper.py │ ├── pig_mapper.py │ ├── shell_mapper.py │ ├── spark_mapper.py │ ├── ssh_mapper.py │ ├── start_mapper.py │ └── subworkflow_mapper.py ├── o2a.py ├── o2a_libs │ ├── LICENSE │ ├── README.md │ ├── __init__.py │ ├── pyproject.toml │ └── src │ │ ├── __init__.py │ │ └── o2a_lib │ │ ├── __init__.py │ │ ├── el_fs_functions.py │ │ ├── el_parser.py │ │ ├── el_wf_functions.py │ │ ├── functions.py │ │ └── property_utils.py ├── schema │ ├── all-schemas-1.0.xsd │ ├── distcp-action-1.0.xsd │ ├── email-action-0.2.xsd │ ├── git-action-1.0.xsd │ ├── hive-action-1.0.xsd │ ├── hive2-action-1.0.xsd │ ├── oozie-common-1.0.xsd │ ├── oozie-workflow-1.0.xsd │ ├── shell-action-1.0.xsd │ ├── spark-action-1.0.xsd │ └── ssh-action-0.2.xsd ├── scripts │ ├── git.sh │ └── prepare.sh ├── templates │ ├── dag_body.tpl │ ├── decision.tpl │ ├── distcp.tpl │ ├── dummy.tpl │ ├── email.tpl │ ├── fs_op.tpl │ ├── git.tpl │ ├── git_command.tpl │ ├── hadoop_command.tpl │ ├── hive.tpl │ ├── http.tpl │ ├── http_command.tpl │ ├── java.tpl │ ├── kill.tpl │ ├── macros │ │ └── props.tpl │ ├── mapreduce.tpl │ ├── pig.tpl │ ├── pig_command.tpl │ ├── prepare.tpl │ ├── prepare_command.tpl │ ├── props.tpl │ ├── shell.tpl │ ├── spark.tpl │ ├── ssh.tpl │ ├── subwf.tpl │ ├── subworkflow.tpl │ ├── workflow.tpl │ ├── workflow_dot.tpl │ └── xml_escaped_props.tpl ├── transformers │ ├── __init__.py │ ├── add_node_notificaton_transformer.py │ ├── add_workflow_notificaton_transformer.py │ ├── base_transformer.py │ ├── remove_end_transformer.py │ ├── remove_fork_transformer.py │ ├── remove_inaccessible_node_transformer.py │ ├── remove_join_transformer.py │ ├── remove_kill_transformer.py │ └── remove_start_transformer.py └── utils │ ├── __init__.py │ ├── config_extractors.py │ ├── constants.py │ ├── el_utils.py │ ├── file_archive_extractors.py │ ├── file_utils.py │ ├── param_extractor.py │ ├── python_serializer.py │ ├── relation_utils.py │ ├── template_utils.py │ ├── variable_name_utils.py │ └── xml_utils.py ├── pylintrc ├── pyproject.toml ├── requirements.txt ├── setup.cfg ├── setup.py ├── tests ├── __init__.py ├── converter │ ├── __init__.py │ ├── test_oozie_converter.py │ ├── test_renderers.py │ ├── test_task_groups.py │ └── test_workflow_xml_parser.py ├── mappers │ ├── __init__.py │ ├── extensions │ │ ├── __init__.py │ │ └── test_prepare_mapper_extension.py │ ├── test_action_mapper.py │ ├── test_archive_extractor.py │ ├── test_base_mapper.py │ ├── test_decision_mapper.py │ ├── test_distcp_mapper.py │ ├── test_dummy_mapper.py │ ├── test_email_mapper.py │ ├── test_file_extractor.py │ ├── test_fs_mapper.py │ ├── test_git_mapper.py │ ├── test_hive_mapper.py │ ├── test_java_mapper.py │ ├── test_mapreduce_mapper.py │ ├── test_pig_mapper.py │ ├── test_shell_mapper.py │ ├── test_spark_mapper.py │ ├── test_ssh_mapper.py │ └── test_subworkflow_mapper.py ├── o2a_libs │ ├── __init__.py │ ├── test_el_fs_functions.py │ ├── test_el_parser.py │ ├── test_functions.py │ └── test_property_utils.py ├── script_tests │ ├── __init.__.py │ ├── mock │ ├── test_git.py │ ├── test_prepare.py │ └── utils.py ├── test_templates.py ├── transformers │ ├── __init__.py │ ├── test_add_node_notification_transformer.py │ ├── test_add_workflow_notification_transformer.py │ ├── test_inaccessible_node_transformer.py │ ├── test_remove_end_transformer.py │ ├── test_remove_fork_transformer.py │ ├── test_remove_join_transformer.py │ ├── test_remove_kill_transformer.py │ └── test_remove_start_transformer.py └── utils │ ├── __init__.py │ ├── test_config_extractors.py │ ├── test_el_utils.py │ ├── test_param_extractor.py │ ├── test_python_serializer.py │ ├── test_relation_utils.py │ ├── test_variable_name_utils.py │ └── test_xml_utils.py ├── utils └── LICENSE.txt └── yamllint-config.yml /.flake8: -------------------------------------------------------------------------------- 1 | [flake8] 2 | #ignore = 3 | max-line-length = 110 4 | max-complexity = 18 5 | #select = B,C,E,F,W,T4,B9 6 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/bug_report.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Bug report 3 | about: Create a report to help us improve 4 | title: '' 5 | labels: '' 6 | assignees: '' 7 | 8 | --- 9 | **Describe the bug** 10 | A clear and concise description of what the bug is. 11 | 12 | **To Reproduce** 13 | 14 | Steps to reproduce the behavior: 15 | 1. Run '...' 16 | 2. See error 17 | 18 | **Expected behavior** 19 | A clear and concise description of what you expected to happen. 20 | 21 | **Screenshots** 22 | If applicable, add screenshots to help explain your problem. 23 | 24 | **Desktop (please complete the following information):** 25 | - OS: [e.g. MacOS/Debian] 26 | - Version [e.g. 22] 27 | 28 | **Additional context** 29 | Add any other context about the problem here. 30 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/feature_request.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Feature request 3 | about: Suggest an idea for this project 4 | title: '' 5 | labels: '' 6 | assignees: '' 7 | 8 | --- 9 | **Is your feature request related to a problem? Please describe.** 10 | A clear and concise description of what the problem is. Ex. 11 | I'm always frustrated when [...] 12 | 13 | **Describe the solution you'd like** 14 | A clear and concise description of what you want to happen. 15 | 16 | **Describe alternatives you've considered** 17 | A clear and concise description of any alternative solutions or 18 | features you've considered. 19 | 20 | **Additional context** 21 | Add any other context or screenshots about the feature request here. 22 | -------------------------------------------------------------------------------- /.github/PULL_REQUEST_TEMPLATE.md: -------------------------------------------------------------------------------- 1 | Make sure you have checked _all_ steps below and that above description is correct. 2 | 3 | ### Github issue 4 | - [ ] My PR addresses the following [Issue](https://github.com/GoogleCloudPlatform/oozie-to-airflow/issues/XXX) 5 | issue and references it in commit message. 6 | 7 | ### Tests 8 | - [ ] My PR has unit tests testing the functionality (or I explained why it needs no tests) 9 | 10 | ### Commits 11 | - [ ] My commits reference the issue in the message using: Fixes #XXX /Closes #XXX or similar. 12 | 13 | ### Documentation 14 | - [ ] My PR adds documentation that describes how to use it (or I explained why it needs no docs) 15 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # IntelliJ IDEA / PyCharm 2 | .idea 3 | *.iml 4 | 5 | # Python 6 | __pycache__ 7 | *.pyc 8 | 9 | .coverage 10 | 11 | .mypy_cache 12 | .o2a-run-sys-test-cache-dir 13 | output/ 14 | output-artifacts/ 15 | /build/ 16 | /dist/ 17 | *.egg-info/ 18 | /.eggs/ 19 | /pip-wheel-metadata 20 | /o2a/o2a_libs/dist 21 | -------------------------------------------------------------------------------- /.kokoro/kokoro_build.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Copyright 2022 Google LLC 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | # Fail on any error. 18 | 19 | # Display commands being run. 20 | # WARNING: please only enable 'set -x' if necessary for debugging, and be very 21 | # careful if you handle credentials (e.g. from Keystore) with 'set -x': 22 | # statements like "export VAR=$(cat /tmp/keystore/credentials)" will result in 23 | # the credentials being printed in build logs. 24 | # Additionally, recursive invocation with credentials as command-line 25 | # parameters, will print the full command, with credentials, in the build logs. 26 | # set -x 27 | 28 | # Code under repo is checked out to ${KOKORO_ARTIFACTS_DIR}/git. 29 | # The final directory name in this path is determined by the scm name specified 30 | # in the job configuration. 31 | 32 | cd "${KOKORO_ARTIFACTS_DIR}/git/oozie-to-airflow" 33 | .kokoro/run_tests.sh 34 | -------------------------------------------------------------------------------- /.kokoro/presubmit.cfg: -------------------------------------------------------------------------------- 1 | # -*- protobuffer -*- 2 | # proto-file: google3/devtools/kokoro/config/proto/build.proto 3 | # proto-message: BuildConfig 4 | 5 | # Location of the bash script. Should have value /. 6 | # github_scm.name is specified in the job configuration (next section). 7 | build_file: "oozie-to-airflow/.kokoro/kokoro_build.sh" 8 | -------------------------------------------------------------------------------- /.kokoro/run_tests.sh: -------------------------------------------------------------------------------- 1 | # Copyright 2022 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | set -xe 16 | MY_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" 17 | O2A_DIR="$( cd "${MY_DIR}/.." && pwd )" 18 | 19 | # add user's pip binary path to PATH 20 | export PATH="${HOME}/.local/bin:${PATH}" 21 | 22 | if [[ ! -z "${KOKORO_BUILD_ID}" ]]; then # export vars only for Kokoro job 23 | # add USER env variable for the unit tests 24 | export USER="kokoro" 25 | 26 | # add TERM env variable for the conversion tests 27 | export TERM="xterm-256color" 28 | 29 | # Setup service account credentials 30 | export GOOGLE_APPLICATION_CREDENTIALS=${KOKORO_GFILE_DIR}/kokoro/service-account-key.json 31 | 32 | # Setup project id 33 | export PROJECT_ID=$(cat "${KOKORO_GFILE_DIR}/kokoro/project-id.txt") 34 | export COMPOSER_TESTS_PROJECT_ID=PROJECT_ID 35 | fi 36 | 37 | # install xmllint 38 | sudo apt-get update 39 | sudo apt-get install -y libxml2-utils 40 | 41 | # prepare python environment 42 | pyenv install --skip-existing 3.8.10 43 | pyenv install --skip-existing 3.9.5 44 | pyenv global 3.8.10 3.9.5 45 | python -m pip install --user -r ${O2A_DIR}/requirements.txt 46 | 47 | echo -e "******************** Running unit tests... ********************\n" 48 | "${O2A_DIR}/bin/o2a-run-all-unit-tests" 49 | echo -e "******************** Unit tests complete. ********************\n" 50 | echo -e "******************** Running Conversion tests... ********************\n" 51 | "${O2A_DIR}/bin/o2a-run-all-conversions" py 52 | rm -rf output 53 | "${O2A_DIR}/bin/o2a-run-all-conversions" dot 54 | echo -e "******************** Conversion tests complete. ********************\n" 55 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | # Copyright 2019 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | --- 15 | language: python 16 | env: 17 | - SLUGIFY_USES_TEXT_UNIDECODE=yes WITH_COVERAGE=true 18 | python: 19 | - "3.8.10" 20 | cache: pip 21 | addons: 22 | apt: 23 | packages: 24 | - graphviz 25 | - coreutils 26 | - libxml2-utils 27 | install: 28 | - pip uninstall -y mock 29 | - pip install -r requirements.txt 30 | - sudo apt-get install 31 | script: 32 | - ./bin/o2a-run-all-configurations 33 | - pre-commit run --all-files --hook-stage=push 34 | - ./bin/o2a-generate-dependency-graph 35 | - ./bin/o2a-generate-index images 36 | - '[[ ! -z "${GCP_SERVICE_ACCOUNT}" ]] && [[ ! -z "${GCP_BUCKET_NAME}" ]] && 37 | gsutil -m cp -R images gs://${GCP_BUCKET_NAME}/commit-${TRAVIS_COMMIT}/images || true' 38 | - ./bin/o2a-run-all-conversions py 39 | - ./bin/o2a-generate-index output 40 | - '[[ ! -z "${GCP_SERVICE_ACCOUNT}" ]] && [[ ! -z "${GCP_BUCKET_NAME}" ]] && 41 | gsutil -m cp -R output gs://${GCP_BUCKET_NAME}/commit-${TRAVIS_COMMIT}/output-py || true' 42 | - rm -rf output 43 | - ./bin/o2a-run-all-conversions dot 44 | - ./bin/o2a-generate-index output 45 | - '[[ ! -z "${GCP_SERVICE_ACCOUNT}" ]] && [[ ! -z "${GCP_BUCKET_NAME}" ]] && 46 | gsutil -m cp -R output gs://${GCP_BUCKET_NAME}/commit-${TRAVIS_COMMIT}/output-dot || true' 47 | - '[[ ! -z "${GCP_SERVICE_ACCOUNT}" ]] && [[ ! -z "${GCP_BUCKET_NAME}" ]] && 48 | export BUCKET_URL="https://${GCP_BUCKET_NAME}.storage.googleapis.com/commit-${TRAVIS_COMMIT}" && 49 | echo ================================================================================ && 50 | echo Build artifacts URls && 51 | echo ================================================================================ && 52 | echo "Images: ${BUCKET_URL}/images/index.html" && 53 | echo "Dot files: ${BUCKET_URL}/output-dot/index.html" && 54 | echo "Python files: ${BUCKET_URL}/output-py/index.html" || true' 55 | 56 | after_success: 57 | - bash <(curl -s https://codecov.io/bash) 58 | before_install: 59 | - '[[ ! -z "${GCP_SERVICE_ACCOUNT}" ]] && [[ ! -z "${GCP_BUCKET_NAME}" ]] && 60 | gcloud auth activate-service-account --key-file=<(echo "${GCP_SERVICE_ACCOUNT}") || true' 61 | -------------------------------------------------------------------------------- /.travis/o2a-build-artifacts-sa.json.enc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GoogleCloudPlatform/oozie-to-airflow/6bfa193dc61e056d9e54a334790dd5411db1c2b6/.travis/o2a-build-artifacts-sa.json.enc -------------------------------------------------------------------------------- /CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- 1 | 16 | 17 | This project follows 18 | [Google's Open Source Community Guidelines](https://opensource.google.com/conduct/). 19 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include bin/o2a 2 | include bin/o2a-validate-workflows 3 | include o2a/schema/*.xsd 4 | include o2a/scripts/*.sh 5 | recursive-include o2a *.py *.tpl 6 | -------------------------------------------------------------------------------- /bin/o2a: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # Copyright 2019 Google LLC 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | """Entry script for the o2a main function""" 17 | from os import path 18 | 19 | import sys 20 | 21 | sys.path.insert(0, path.abspath(path.join(path.dirname(__file__), path.pardir))) 22 | 23 | if sys.version_info.major < 3 or (sys.version_info.major == 3 and sys.version_info.minor < 6): 24 | print("") 25 | print( 26 | "ERROR! You need to run this script in python version >= 3.8 (and you have {}.{})".format( 27 | sys.version_info.major, sys.version_info.minor 28 | ) 29 | ) 30 | print("") 31 | sys.exit(1) 32 | 33 | # pylint: disable=C0413 34 | import o2a.o2a # noqa: E402 35 | 36 | if __name__ == "__main__": 37 | print("") 38 | o2a.o2a.main() 39 | print("") 40 | -------------------------------------------------------------------------------- /bin/o2a-confirm: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # Copyright 2019 Google LLC 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | read -r -p "${1}. Are you sure? [y/N] " response 18 | case "$response" in 19 | [yY][eE][sS]|[yY]) 20 | exit 0 21 | ;; 22 | *) 23 | exit 1 24 | ;; 25 | esac 26 | -------------------------------------------------------------------------------- /bin/o2a-generate-dependency-graph: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # Copyright 2019 Google LLC 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | set -euo pipefail 16 | MY_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" 17 | O2A_DIR="$( cd "${MY_DIR}/.." && pwd )" 18 | pushd "${O2A_DIR}" 19 | 20 | EXCLUDED_MODULES=(black airflow jinja2 pytz _json parameterized o2a.tests o2a.script-tests) 21 | 22 | pydeps --noshow --max-bacon=0 --noise-level=5 o2a -T png -o "${MY_DIR}/../images/o2a-dependencies.png" -x "${EXCLUDED_MODULES[@]}" 23 | pydeps --noshow --max-bacon=0 --noise-level=5 --show-cycles o2a -T png -o "${MY_DIR}/../images/o2a-dependency-cycles.png" -x "${EXCLUDED_MODULES[@]}" 24 | 25 | popd 26 | -------------------------------------------------------------------------------- /bin/o2a-generate-index: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # Copyright 2019 Google LLC 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | set -euo pipefail 18 | 19 | function usage() { 20 | echo "$0 [TARGET]" 21 | echo "" 22 | echo "TARGET - Directory in which the index will be generated. " 23 | echo "Generated index contains all files in directories and subdirectories." 24 | echo 25 | echo "This script depends on environmental variable: TRAVIS_COMMIT" 26 | echo "See: https://docs.travis-ci.com/user/environment-variables/" 27 | } 28 | 29 | if [[ -z "${TRAVIS_COMMIT-}" ]]; then 30 | echo "You must set TRAVIS_COMMIT environment variable" 31 | exit 1 32 | fi 33 | 34 | if [[ -z "${1-}" ]]; then 35 | echo "You must target directory" 36 | echo "" 37 | usage 38 | exit 1 39 | fi 40 | 41 | TARGET_DIR="${1}" 42 | 43 | cat > "${TARGET_DIR}/index.html" < 45 | 46 | 47 | 48 | 49 | 50 | Artifacts of ${TRAVIS_COMMIT} build 51 | 52 | 53 | 54 | 55 | 56 |
57 |

Artifacts of ${TRAVIS_COMMIT} build

58 |
59 | EOF 60 | 61 | for FILE in $(cd "${TARGET_DIR}" && find . -type f | grep -v "__pycache__" | cut -d "/" -f 2- | sort); do 62 | cat >> "${TARGET_DIR}/index.html" <${FILE} 64 | EOF 65 | 66 | done 67 | 68 | cat >> "${TARGET_DIR}/index.html" < 70 |
71 | 72 | 73 | EOF 74 | -------------------------------------------------------------------------------- /bin/o2a-package-upload: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # Copyright 2019 Google LLC 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | set -euo pipefail 16 | MY_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" 17 | O2A_DIR="$( cd "${MY_DIR}/.." && pwd )" 18 | 19 | pushd "${O2A_DIR}" 20 | echo 21 | "${MY_DIR}/o2a-confirm" "Preparing production package" 22 | echo 23 | rm -rvf dist/* 24 | rm -rvf o2a.egg-info 25 | rm -rvf .eggs 26 | python3 setup.py sdist bdist_wheel 27 | echo 28 | "${MY_DIR}/o2a-confirm" "Uploading to production PyPi" 29 | echo 30 | python3 -m twine upload --repository pypi dist/* 31 | popd 32 | -------------------------------------------------------------------------------- /bin/o2a-package-upload-test: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # Copyright 2019 Google LLC 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | set -euo pipefail 16 | MY_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" 17 | O2A_DIR="$( cd "${MY_DIR}/.." && pwd )" 18 | 19 | pushd "${O2A_DIR}" 20 | echo 21 | "${MY_DIR}/o2a-confirm" "Preparing test package" 22 | echo 23 | rm -rvf dist/* 24 | rm -rvf o2a.egg-info 25 | rm -rvf .eggs 26 | python3 setup.py sdist bdist_wheel 27 | echo 28 | "${MY_DIR}/o2a-confirm" "Uploading to test PyPi" 29 | echo 30 | python3 -m twine upload --repository testpypi dist/* 31 | popd 32 | -------------------------------------------------------------------------------- /bin/o2a-run-all-configurations: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # Copyright 2019 Google LLC 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | set -euo pipefail 16 | MY_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" 17 | O2A_DIR="$( cd "${MY_DIR}/.." && pwd )" 18 | 19 | pushd "${O2A_DIR}" 20 | for EXAMPLE in $(cd examples; ls) 21 | do 22 | echo "Preparing configuration for ${EXAMPLE}" 23 | "${MY_DIR}/o2a-run-sys-test" --phase prepare-configuration -a "${EXAMPLE}" >/dev/null 24 | done 25 | 26 | popd 27 | -------------------------------------------------------------------------------- /bin/o2a-run-all-conversions: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # Copyright 2019 Google LLC 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | set -euo pipefail 16 | MY_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" 17 | O2A_DIR="$( cd "${MY_DIR}/.." && pwd )" 18 | 19 | pushd "${O2A_DIR}" 20 | 21 | OUTPUT_FORMAT=${1-py} 22 | 23 | for EXAMPLE in $(find examples -name workflow.xml | cut -d "/" -f 2) 24 | do 25 | "${MY_DIR}/o2a-run-sys-test" --phase prepare-configuration -a "${EXAMPLE}" 26 | done 27 | for EXAMPLE in $(find examples -name workflow.xml | cut -d "/" -f 2) 28 | do 29 | if [[ "${OUTPUT_FORMAT}" == "py" ]]; then 30 | "${MY_DIR}/o2a-run-sys-test" --phase convert -a "${EXAMPLE}" 31 | elif [[ "${OUTPUT_FORMAT}" == "dot" ]]; then 32 | "${MY_DIR}/o2a-run-sys-test" --phase convert -a "${EXAMPLE}" --dot 33 | else 34 | echo "Unknown format. Supported format: dot, py." 35 | exit 1 36 | fi 37 | done 38 | -------------------------------------------------------------------------------- /bin/o2a-run-all-unit-tests: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # Copyright 2019 Google LLC 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | set -euo pipefail 16 | MY_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" 17 | O2A_DIR="$( cd "${MY_DIR}/.." && pwd )" 18 | 19 | pushd "${O2A_DIR}" 20 | pytest --cov="${O2A_DIR}" tests 21 | popd 22 | -------------------------------------------------------------------------------- /bin/o2a-validate-all-workflows: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # Copyright 2019 Google LLC 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | set -euo pipefail 16 | MY_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" 17 | O2A_DIR="$( cd "${MY_DIR}/.." && pwd )" 18 | 19 | 20 | find "${O2A_DIR}" -name 'workflow.xml' -print0 | xargs -0 "${MY_DIR}/o2a-validate-workflows" 21 | -------------------------------------------------------------------------------- /bin/o2a-validate-workflows: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # Copyright 2019 Google LLC 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | set -euo pipefail 16 | MY_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" 17 | O2A_DIR="$( cd "${MY_DIR}/.." && pwd )" 18 | 19 | if [[ -d "${O2A_DIR}/o2a/schema/" ]]; then 20 | SCHEMA_DIR="${O2A_DIR}/o2a/schema" 21 | else 22 | SCHEMA_DIR="$(python3 -c 'import os; import o2a; print(os.path.dirname(o2a.__file__))')/schema" 23 | fi 24 | 25 | FAILED_WORKFLOWS="" 26 | VERBOSE=${VERBOSE:=""} 27 | XMLLINT_EXTRA_PARAMS="" 28 | 29 | if [[ "${VERBOSE}" != "" ]]; then 30 | XMLLINT_EXTRA_PARAMS="--load-trace" 31 | fi 32 | 33 | for WORKFLOW in "$@"; do 34 | echo 35 | echo "Validating ${WORKFLOW}" 36 | echo 37 | set +e 38 | # Note - if you add new actions add the schema for this action to "all-schemas-1.0.xsd" 39 | xmllint --noout ${XMLLINT_EXTRA_PARAMS} --schema "${SCHEMA_DIR}/all-schemas-1.0.xsd" "${WORKFLOW}" 40 | ERR=$? 41 | set -e 42 | if [[ ${ERR} != "0" ]]; then 43 | FAILED_WORKFLOWS="${FAILED_WORKFLOWS} ${WORKFLOW}" 44 | fi 45 | done 46 | 47 | 48 | if [[ "${FAILED_WORKFLOWS}" != "" ]]; then 49 | echo 50 | echo 51 | echo "Some workflows failed validation:" 52 | echo 53 | for WORKFLOW in ${FAILED_WORKFLOWS} 54 | do 55 | echo "${WORKFLOW}" 56 | done 57 | echo 58 | echo 59 | exit 1 60 | else 61 | echo 62 | echo "Workflows validated properly" 63 | echo 64 | fi 65 | -------------------------------------------------------------------------------- /bin/o2a_lib-package-upload: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # Copyright 2019 Google LLC 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | set -euo pipefail 16 | MY_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" 17 | O2A_DIR="$( cd "${MY_DIR}/.." && pwd )" 18 | 19 | pushd "${O2A_DIR}" 20 | echo 21 | "${MY_DIR}/o2a-confirm" "Preparing production package" 22 | echo 23 | rm -rvf o2a/o2a_libs/dist/* 24 | rm -rvf o2a/o2a_libs/src/o2a_lib.egg-info 25 | rm -rvf .eggs 26 | python3 -m pip install --upgrade build 27 | python3 -m build o2a/o2a_libs 28 | echo 29 | "${MY_DIR}/o2a-confirm" "Uploading to production PyPi" 30 | echo 31 | python3 -m twine upload --repository pypi o2a/o2a_libs/dist/* 32 | popd 33 | -------------------------------------------------------------------------------- /bin/o2a_lib-package-upload-test: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # Copyright 2019 Google LLC 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | set -euo pipefail 16 | MY_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" 17 | O2A_DIR="$( cd "${MY_DIR}/.." && pwd )" 18 | 19 | pushd "${O2A_DIR}" 20 | echo 21 | "${MY_DIR}/o2a-confirm" "Preparing test package" 22 | echo 23 | rm -rvf o2a/o2a_libs/dist/* 24 | rm -rvf o2a/o2a_libs/src/o2a_lib.egg-info 25 | rm -rvf .eggs 26 | python3 -m pip install --upgrade build 27 | python3 -m build o2a/o2a_libs 28 | echo 29 | "${MY_DIR}/o2a-confirm" "Uploading to test PyPi" 30 | echo 31 | python3 -m twine upload --repository testpypi o2a/o2a_libs/dist/* 32 | popd 33 | -------------------------------------------------------------------------------- /codecov.yaml: -------------------------------------------------------------------------------- 1 | # Copyright 2019 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | --- 15 | codecov: 16 | notify: 17 | require_ci_to_pass: true 18 | coverage: 19 | precision: 2 20 | round: down 21 | range: “70…100” 22 | status: 23 | project: true 24 | patch: true 25 | changes: false 26 | comment: 27 | layout: “reach, diff, flags, files, footer” 28 | behavior: default 29 | require_changes: false 30 | -------------------------------------------------------------------------------- /dataproc/example-map-reduce-job.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # Copyright 2019 Google LLC 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | set -euxo pipefail 17 | 18 | # ensure hdfs path exists 19 | hdfs dfs -mkdir -p "/user/$(whoami)/" 20 | 21 | # copy examples to home dir 22 | cp /usr/local/lib/oozie/oozie-examples.tar.gz . 23 | 24 | #unpack 25 | tar -zxvf oozie-examples.tar.gz 26 | 27 | # replace `localhost` with actual hostname 28 | sed -i "s/localhost/$(hostname)/g" examples/apps/map-reduce/job.properties 29 | 30 | # create directory on HDFS for current user 31 | hdfs dfs -put examples "/user/$(whoami)/" 32 | 33 | # run the MR task 34 | oozie job -config examples/apps/map-reduce/job.properties -run 35 | -------------------------------------------------------------------------------- /examples/.gitignore: -------------------------------------------------------------------------------- 1 | # Copyright 2019 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | /*/configuration.properties 16 | -------------------------------------------------------------------------------- /examples/advancedflow/configuration.template.properties: -------------------------------------------------------------------------------- 1 | # Copyright 2019 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | # 16 | # Licensed to the Apache Software Foundation (ASF) under one 17 | # or more contributor license agreements. See the NOTICE file 18 | # distributed with this work for additional information 19 | # regarding copyright ownership. The ASF licenses this file 20 | # to you under the Apache License, Version 2.0 (the 21 | # "License"); you may not use this file except in compliance 22 | # with the License. You may obtain a copy of the License at 23 | # 24 | # http://www.apache.org/licenses/LICENSE-2.0 25 | # 26 | # Unless required by applicable law or agreed to in writing, software 27 | # distributed under the License is distributed on an "AS IS" BASIS, 28 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 29 | # See the License for the specific language governing permissions and 30 | # limitations under the License. 31 | # 32 | 33 | dataproc_cluster={{DATAPROC_CLUSTER_NAME}} 34 | gcp_conn_id=google_cloud_default 35 | gcp_region={{GCP_REGION}} 36 | gcp_uri_prefix=gs://{{COMPOSER_DAG_BUCKET}}/dags 37 | -------------------------------------------------------------------------------- /examples/advancedflow/job.properties: -------------------------------------------------------------------------------- 1 | # Copyright 2019 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | nameNode=hdfs://localhost:8020 16 | resourceManager=localhost:8032 17 | queueName=default 18 | examplesRoot=examples 19 | oozie.wf.application.path=/user/${user.name}/${examplesRoot}/apps/advancedflow 20 | oozie.wf.workflow.notification.url=http://example.com/workflow?job-id=$jobId&status=$status&parent-id=$parentId 21 | -------------------------------------------------------------------------------- /examples/childwf/configuration.template.properties: -------------------------------------------------------------------------------- 1 | # Copyright 2019 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | # 16 | # Licensed to the Apache Software Foundation (ASF) under one 17 | # or more contributor license agreements. See the NOTICE file 18 | # distributed with this work for additional information 19 | # regarding copyright ownership. The ASF licenses this file 20 | # to you under the Apache License, Version 2.0 (the 21 | # "License"); you may not use this file except in compliance 22 | # with the License. You may obtain a copy of the License at 23 | # 24 | # http://www.apache.org/licenses/LICENSE-2.0 25 | # 26 | # Unless required by applicable law or agreed to in writing, software 27 | # distributed under the License is distributed on an "AS IS" BASIS, 28 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 29 | # See the License for the specific language governing permissions and 30 | # limitations under the License. 31 | # 32 | 33 | dataproc_cluster={{DATAPROC_CLUSTER_NAME}} 34 | gcp_conn_id=google_cloud_default 35 | gcp_region={{GCP_REGION}} 36 | # A single item or a comma-separated list 37 | hadoop_jars=hdfs:///user/{{DATAPROC_USER}}/examples/apps/{{LOCAL_APP_NAME}}/lib/wordcount.jar 38 | hadoop_main_class=WordCount 39 | -------------------------------------------------------------------------------- /examples/childwf/job.properties: -------------------------------------------------------------------------------- 1 | # Copyright 2019 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | # 16 | # Licensed to the Apache Software Foundation (ASF) under one 17 | # or more contributor license agreements. See the NOTICE file 18 | # distributed with this work for additional information 19 | # regarding copyright ownership. The ASF licenses this file 20 | # to you under the Apache License, Version 2.0 (the 21 | # "License"); you may not use this file except in compliance 22 | # with the License. You may obtain a copy of the License at 23 | # 24 | # http://www.apache.org/licenses/LICENSE-2.0 25 | # 26 | # Unless required by applicable law or agreed to in writing, software 27 | # distributed under the License is distributed on an "AS IS" BASIS, 28 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 29 | # See the License for the specific language governing permissions and 30 | # limitations under the License. 31 | # 32 | 33 | nameNode=hdfs:// 34 | resourceManager=localhost:8032 35 | queueName=default 36 | examplesRoot=examples 37 | oozie.use.system.libpath=true 38 | oozie.wf.application.path=${nameNode}/user/${user.name}/${examplesRoot}/apps/mapreduce 39 | outputDir=output-data 40 | -------------------------------------------------------------------------------- /examples/decision/configuration.template.properties: -------------------------------------------------------------------------------- 1 | # Copyright 2019 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | dataproc_cluster={{DATAPROC_CLUSTER_NAME}} 16 | gcp_region={{GCP_REGION}} 17 | -------------------------------------------------------------------------------- /examples/decision/hdfs/workflow.xml: -------------------------------------------------------------------------------- 1 | 16 | 17 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | ${firstNotNull("first", "second") == "second"} 41 | 42 | 43 | ${firstNotNull("first", "second") == "first"} 44 | 45 | 46 | 47 | 48 | 49 | 50 | 51 | 52 | 53 | 54 | 55 | 56 | Fake end reached 57 | 58 | 59 | 60 | -------------------------------------------------------------------------------- /examples/decision/job.properties: -------------------------------------------------------------------------------- 1 | # Copyright 2019 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | # 16 | # Licensed to the Apache Software Foundation (ASF) under one 17 | # or more contributor license agreements. See the NOTICE file 18 | # distributed with this work for additional information 19 | # regarding copyright ownership. The ASF licenses this file 20 | # to you under the Apache License, Version 2.0 (the 21 | # "License"); you may not use this file except in compliance 22 | # with the License. You may obtain a copy of the License at 23 | # 24 | # http://www.apache.org/licenses/LICENSE-2.0 25 | # 26 | # Unless required by applicable law or agreed to in writing, software 27 | # distributed under the License is distributed on an "AS IS" BASIS, 28 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 29 | # See the License for the specific language governing permissions and 30 | # limitations under the License. 31 | # 32 | 33 | nameNode=hdfs://localhost:8020 34 | resourceManager=localhost:8032 35 | queueName=default 36 | examplesRoot=examples 37 | oozie.wf.application.path=/user/${user.name}/${examplesRoot}/apps/decision 38 | -------------------------------------------------------------------------------- /examples/demo/configuration.template.properties: -------------------------------------------------------------------------------- 1 | # Copyright 2019 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | dataproc_cluster={{DATAPROC_CLUSTER_NAME}} 16 | gcp_conn_id=google_cloud_default 17 | gcp_region={{GCP_REGION}} 18 | gcp_uri_prefix=gs://{{COMPOSER_DAG_BUCKET}}/dags 19 | -------------------------------------------------------------------------------- /examples/demo/hdfs/pig/id.pig: -------------------------------------------------------------------------------- 1 | -- Copyright 2019 Google LLC 2 | -- 3 | -- Licensed under the Apache License, Version 2.0 (the "License"); 4 | -- you may not use this file except in compliance with the License. 5 | -- You may obtain a copy of the License at 6 | -- 7 | -- http://www.apache.org/licenses/LICENSE-2.0 8 | -- 9 | -- Unless required by applicable law or agreed to in writing, software 10 | -- distributed under the License is distributed on an "AS IS" BASIS, 11 | -- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | -- See the License for the specific language governing permissions and 13 | -- limitations under the License. 14 | 15 | -- 16 | -- Licensed to the Apache Software Foundation (ASF) under one 17 | -- or more contributor license agreements. See the NOTICE file 18 | -- distributed with this work for additional information 19 | -- regarding copyright ownership. The ASF licenses this file 20 | -- to you under the Apache License, Version 2.0 (the 21 | -- "License"); you may not use this file except in compliance 22 | -- with the License. You may obtain a copy of the License at 23 | -- 24 | -- http://www.apache.org/licenses/LICENSE-2.0 25 | -- 26 | -- Unless required by applicable law or agreed to in writing, software 27 | -- distributed under the License is distributed on an "AS IS" BASIS, 28 | -- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 29 | -- See the License for the specific language governing permissions and 30 | -- limitations under the License. 31 | -- 32 | A = load '$INPUT' using PigStorage(':'); 33 | B = foreach A generate $0 as id; 34 | store B into '$OUTPUT' USING PigStorage(); 35 | -------------------------------------------------------------------------------- /examples/demo/hdfs/pig/input-data/test-data.txt: -------------------------------------------------------------------------------- 1 | 1850,0,1,1483789 2 | 2000,90,2,1064581 3 | -------------------------------------------------------------------------------- /examples/demo/job.properties: -------------------------------------------------------------------------------- 1 | # Copyright 2019 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | # 16 | # Licensed to the Apache Software Foundation (ASF) under one 17 | # or more contributor license agreements. See the NOTICE file 18 | # distributed with this work for additional information 19 | # regarding copyright ownership. The ASF licenses this file 20 | # to you under the Apache License, Version 2.0 (the 21 | # "License"); you may not use this file except in compliance 22 | # with the License. You may obtain a copy of the License at 23 | # 24 | # http://www.apache.org/licenses/LICENSE-2.0 25 | # 26 | # Unless required by applicable law or agreed to in writing, software 27 | # distributed under the License is distributed on an "AS IS" BASIS, 28 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 29 | # See the License for the specific language governing permissions and 30 | # limitations under the License. 31 | # 32 | 33 | nameNode=hdfs:// 34 | resourceManager=localhost:8032 35 | queueName=default 36 | examplesRoot=examples 37 | oozie.use.system.libpath=true 38 | oozie.wf.application.path=/user/${user.name}/${examplesRoot}/apps/demo 39 | oozie.wf.workflow.notification.url=http://example.com:8080/workflow?job-id=$jobId&status=$status 40 | oozie.wf.action.notification.url=http://example.com:8080/action?job-id=$jobId&node-name=$nodeName&status=$status 41 | -------------------------------------------------------------------------------- /examples/distcp/configuration.template.properties: -------------------------------------------------------------------------------- 1 | # Copyright 2019 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | dataproc_cluster={{DATAPROC_CLUSTER_NAME}} 16 | gcp_conn_id=google_cloud_default 17 | gcp_region={{GCP_REGION}} 18 | gcp_uri_prefix=gs://{{COMPOSER_DAG_BUCKET}}/dags 19 | -------------------------------------------------------------------------------- /examples/distcp/hdfs/input-data/test-data.txt: -------------------------------------------------------------------------------- 1 | 1850,0,1,1483789 2 | 2000,90,2,1064581 3 | -------------------------------------------------------------------------------- /examples/distcp/job.properties: -------------------------------------------------------------------------------- 1 | # Copyright 2019 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | # 16 | # Licensed to the Apache Software Foundation (ASF) under one 17 | # or more contributor license agreements. See the NOTICE file 18 | # distributed with this work for additional information 19 | # regarding copyright ownership. The ASF licenses this file 20 | # to you under the Apache License, Version 2.0 (the 21 | # "License"); you may not use this file except in compliance 22 | # with the License. You may obtain a copy of the License at 23 | # 24 | # http://www.apache.org/licenses/LICENSE-2.0 25 | # 26 | # Unless required by applicable law or agreed to in writing, software 27 | # distributed under the License is distributed on an "AS IS" BASIS, 28 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 29 | # See the License for the specific language governing permissions and 30 | # limitations under the License. 31 | # 32 | 33 | nameNode=hdfs://localhost:8020 34 | nameNode1=hdfs:// 35 | nameNode2=hdfs://oozie-o2a-2cpu-m 36 | resourceManager=localhost:8032 37 | queueName=default 38 | examplesRoot=examples 39 | 40 | oozie.wf.application.path=/user/${user.name}/${examplesRoot}/apps/distcp 41 | oozie.use.system.libpath=true 42 | -------------------------------------------------------------------------------- /examples/el/hdfs/workflow.xml: -------------------------------------------------------------------------------- 1 | 16 | 17 | 34 | 35 | 36 | 37 | 38 | 39 | ${hostname} 40 | ${concat("ls ", "-l")} 41 | 42 | 43 | 44 | 45 | 46 | 47 | SSH action failed, error message[${wf:errorMessage(wf:lastErrorNode())}] 48 | 49 | 50 | 51 | 52 | -------------------------------------------------------------------------------- /examples/el/job.properties: -------------------------------------------------------------------------------- 1 | # Copyright 2019 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | # 16 | # Licensed to the Apache Software Foundation (ASF) under one 17 | # or more contributor license agreements. See the NOTICE file 18 | # distributed with this work for additional information 19 | # regarding copyright ownership. The ASF licenses this file 20 | # to you under the Apache License, Version 2.0 (the 21 | # "License"); you may not use this file except in compliance 22 | # with the License. You may obtain a copy of the License at 23 | # 24 | # http://www.apache.org/licenses/LICENSE-2.0 25 | # 26 | # Unless required by applicable law or agreed to in writing, software 27 | # distributed under the License is distributed on an "AS IS" BASIS, 28 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 29 | # See the License for the specific language governing permissions and 30 | # limitations under the License. 31 | # 32 | 33 | nameNode=hdfs:// 34 | resourceManager=localhost:8032 35 | queueName=default 36 | examplesRoot=examples 37 | hostname=user@apache.org 38 | oozie.wf.application.path=${nameNode}/user/${user.name}/${examplesRoot}/apps/el 39 | -------------------------------------------------------------------------------- /examples/email/configuration.template.properties: -------------------------------------------------------------------------------- 1 | # Copyright 2019 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | dataproc_cluster={{DATAPROC_CLUSTER_NAME}} 16 | gcp_region={{GCP_REGION}} 17 | -------------------------------------------------------------------------------- /examples/email/hdfs/workflow.xml: -------------------------------------------------------------------------------- 1 | 16 | 17 | 18 | 19 | 20 | 21 | to_1_test@gmail.com,to_2_test@gmail.com 22 | cc_1_test@example.com,cc_2_test@example.com 23 | bcc_1_test@gmail.com,bcc_2_test@gmail.com 24 | Email notifications for ${wf:id()} 25 | Hi ${wf:user()}, the wf ${wf:id()} successfully completed (${examplesRoot}). Bye ${wf:user()} 26 | text/plain 27 | 28 | 29 | 30 | 31 | 32 | Email action failed, error message[${wf:errorMessage(wf:lastErrorNode())}] 33 | 34 | 35 | 36 | -------------------------------------------------------------------------------- /examples/email/job.properties: -------------------------------------------------------------------------------- 1 | # Copyright 2019 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | # 16 | # Licensed to the Apache Software Foundation (ASF) under one 17 | # or more contributor license agreements. See the NOTICE file 18 | # distributed with this work for additional information 19 | # regarding copyright ownership. The ASF licenses this file 20 | # to you under the Apache License, Version 2.0 (the 21 | # "License"); you may not use this file except in compliance 22 | # with the License. You may obtain a copy of the License at 23 | # 24 | # http://www.apache.org/licenses/LICENSE-2.0 25 | # 26 | # Unless required by applicable law or agreed to in writing, software 27 | # distributed under the License is distributed on an "AS IS" BASIS, 28 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 29 | # See the License for the specific language governing permissions and 30 | # limitations under the License. 31 | # 32 | 33 | nameNode=hdfs://localhost:8020 34 | resourceManager=localhost:8032 35 | queueName=default 36 | examplesRoot=examples 37 | oozie.wf.application.path=${nameNode}/user/${user.name}/${examplesRoot}/apps/email 38 | -------------------------------------------------------------------------------- /examples/fs/configuration.template.properties: -------------------------------------------------------------------------------- 1 | # Copyright 2019 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | # 16 | # Licensed to the Apache Software Foundation (ASF) under one 17 | # or more contributor license agreements. See the NOTICE file 18 | # distributed with this work for additional information 19 | # regarding copyright ownership. The ASF licenses this file 20 | # to you under the Apache License, Version 2.0 (the 21 | # "License"); you may not use this file except in compliance 22 | # with the License. You may obtain a copy of the License at 23 | # 24 | # http://www.apache.org/licenses/LICENSE-2.0 25 | # 26 | # Unless required by applicable law or agreed to in writing, software 27 | # distributed under the License is distributed on an "AS IS" BASIS, 28 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 29 | # See the License for the specific language governing permissions and 30 | # limitations under the License. 31 | # 32 | 33 | dataproc_cluster={{DATAPROC_CLUSTER_NAME}} 34 | gcp_conn_id=google_cloud_default 35 | gcp_region={{GCP_REGION}} 36 | gcp_uri_prefix=gs://{{COMPOSER_DAG_BUCKET}}/dags 37 | -------------------------------------------------------------------------------- /examples/fs/job.properties: -------------------------------------------------------------------------------- 1 | # Copyright 2019 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | nameNode=hdfs://localhost:8020 16 | resourceManager=localhost:8032 17 | queueName=default 18 | examplesRoot=examples 19 | oozie.wf.application.path=/user/${user.name}/${examplesRoot}/apps/fs 20 | oozie.wf.workflow.notification.url=http://example.com/workflow?job-id=$jobId&status=$status&parent-id=$parentId 21 | -------------------------------------------------------------------------------- /examples/git/configuration.template.properties: -------------------------------------------------------------------------------- 1 | # Copyright 2019 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | # 16 | # Licensed to the Apache Software Foundation (ASF) under one 17 | # or more contributor license agreements. See the NOTICE file 18 | # distributed with this work for additional information 19 | # regarding copyright ownership. The ASF licenses this file 20 | # to you under the Apache License, Version 2.0 (the 21 | # "License"); you may not use this file except in compliance 22 | # with the License. You may obtain a copy of the License at 23 | # 24 | # http://www.apache.org/licenses/LICENSE-2.0 25 | # 26 | # Unless required by applicable law or agreed to in writing, software 27 | # distributed under the License is distributed on an "AS IS" BASIS, 28 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 29 | # See the License for the specific language governing permissions and 30 | # limitations under the License. 31 | # 32 | 33 | dataproc_cluster={{DATAPROC_CLUSTER_NAME}} 34 | gcp_conn_id=google_cloud_default 35 | gcp_region={{GCP_REGION}} 36 | gcp_uri_prefix=gs://{{COMPOSER_DAG_BUCKET}}/dags 37 | -------------------------------------------------------------------------------- /examples/git/hdfs/workflow.xml: -------------------------------------------------------------------------------- 1 | 16 | 17 | 34 | 35 | 36 | 37 | 38 | 39 | ${resourceManager} 40 | ${nameNode} 41 | 42 | 43 | 44 | https://github.com/GoogleCloudPlatform/oozie-to-airflow.git 45 | master 46 | ${nameNode}/user/${wf:user()}/${examplesRoot}/apps/git/repo 47 | 48 | 49 | test.property.node 50 | ${nameNode} 51 | 52 | 53 | 54 | 55 | 56 | 57 | 58 | 59 | Workflow failed, error 60 | message[${wf:errorMessage(wf:lastErrorNode())}] 61 | 62 | 63 | 64 | 65 | -------------------------------------------------------------------------------- /examples/git/job.properties: -------------------------------------------------------------------------------- 1 | # Copyright 2019 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | nameNode=hdfs://localhost:8020 16 | resourceManager=localhost:8032 17 | queueName=default 18 | examplesRoot=examples 19 | oozie.wf.application.path=/user/${user.name}/${examplesRoot}/apps/git 20 | oozie.use.system.libpath=true 21 | oozie.wf.workflow.notification.url=http://example.com/workflow?job-id=$jobId&status=$status&parent-id=$parentId 22 | -------------------------------------------------------------------------------- /examples/hive/configuration.template.properties: -------------------------------------------------------------------------------- 1 | # Copyright 2019 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | # 16 | # Licensed to the Apache Software Foundation (ASF) under one 17 | # or more contributor license agreements. See the NOTICE file 18 | # distributed with this work for additional information 19 | # regarding copyright ownership. The ASF licenses this file 20 | # to you under the Apache License, Version 2.0 (the 21 | # "License"); you may not use this file except in compliance 22 | # with the License. You may obtain a copy of the License at 23 | # 24 | # http://www.apache.org/licenses/LICENSE-2.0 25 | # 26 | # Unless required by applicable law or agreed to in writing, software 27 | # distributed under the License is distributed on an "AS IS" BASIS, 28 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 29 | # See the License for the specific language governing permissions and 30 | # limitations under the License. 31 | # 32 | 33 | dataproc_cluster={{DATAPROC_CLUSTER_NAME}} 34 | gcp_conn_id=google_cloud_default 35 | gcp_region={{GCP_REGION}} 36 | gcp_uri_prefix=gs://{{COMPOSER_DAG_BUCKET}}/dags 37 | -------------------------------------------------------------------------------- /examples/hive/hdfs/input-data/table: -------------------------------------------------------------------------------- 1 | 1 2 | 2 3 | 3 4 | 42 5 | 999 6 | -42 7 | -------------------------------------------------------------------------------- /examples/hive/hdfs/script.q: -------------------------------------------------------------------------------- 1 | -- 2 | -- Licensed to the Apache Software Foundation (ASF) under one 3 | -- or more contributor license agreements. See the NOTICE file 4 | -- distributed with this work for additional information 5 | -- regarding copyright ownership. The ASF licenses this file 6 | -- to you under the Apache License, Version 2.0 (the 7 | -- "License"); you may not use this file except in compliance 8 | -- with the License. You may obtain a copy of the License at 9 | -- 10 | -- http://www.apache.org/licenses/LICENSE-2.0 11 | -- 12 | -- Unless required by applicable law or agreed to in writing, software 13 | -- distributed under the License is distributed on an "AS IS" BASIS, 14 | -- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | -- See the License for the specific language governing permissions and 16 | -- limitations under the License. 17 | -- 18 | DROP TABLE IF EXISTS test_script; 19 | CREATE EXTERNAL TABLE test_script (a INT) STORED AS TEXTFILE LOCATION '${INPUT}'; 20 | INSERT OVERWRITE DIRECTORY '${OUTPUT}' SELECT * FROM test_script; 21 | -------------------------------------------------------------------------------- /examples/hive/job.properties: -------------------------------------------------------------------------------- 1 | # Copyright 2019 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | # 16 | # Licensed to the Apache Software Foundation (ASF) under one 17 | # or more contributor license agreements. See the NOTICE file 18 | # distributed with this work for additional information 19 | # regarding copyright ownership. The ASF licenses this file 20 | # to you under the Apache License, Version 2.0 (the 21 | # "License"); you may not use this file except in compliance 22 | # with the License. You may obtain a copy of the License at 23 | # 24 | # http://www.apache.org/licenses/LICENSE-2.0 25 | # 26 | # Unless required by applicable law or agreed to in writing, software 27 | # distributed under the License is distributed on an "AS IS" BASIS, 28 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 29 | # See the License for the specific language governing permissions and 30 | # limitations under the License. 31 | # 32 | 33 | nameNode=hdfs:// 34 | resourceManager=localhost:8032 35 | queueName=default 36 | examplesRoot=examples 37 | oozie.use.system.libpath=true 38 | oozie.wf.application.path=${nameNode}/user/${user.name}/${examplesRoot}/apps/hive 39 | -------------------------------------------------------------------------------- /examples/java/configuration.template.properties: -------------------------------------------------------------------------------- 1 | # Copyright 2019 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | # 16 | # Licensed to the Apache Software Foundation (ASF) under one 17 | # or more contributor license agreements. See the NOTICE file 18 | # distributed with this work for additional information 19 | # regarding copyright ownership. The ASF licenses this file 20 | # to you under the Apache License, Version 2.0 (the 21 | # "License"); you may not use this file except in compliance 22 | # with the License. You may obtain a copy of the License at 23 | # 24 | # http://www.apache.org/licenses/LICENSE-2.0 25 | # 26 | # Unless required by applicable law or agreed to in writing, software 27 | # distributed under the License is distributed on an "AS IS" BASIS, 28 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 29 | # See the License for the specific language governing permissions and 30 | # limitations under the License. 31 | # 32 | 33 | dataproc_cluster={{DATAPROC_CLUSTER_NAME}} 34 | gcp_conn_id=google_cloud_default 35 | gcp_region={{GCP_REGION}} 36 | gcp_uri_prefix=gs://{{COMPOSER_DAG_BUCKET}}/dags 37 | -------------------------------------------------------------------------------- /examples/java/hdfs/workflow.xml: -------------------------------------------------------------------------------- 1 | 16 | 17 | 34 | 35 | 36 | 37 | 38 | ${resourceManager} 39 | ${nameNode} 40 | 41 | 42 | mapred.job.queue.name 43 | ${queueName} 44 | 45 | 46 | org.apache.oozie.example.DemoJavaMain 47 | -Dtest1=val1 48 | -Dtest2=val2 49 | Hello 50 | Oozie! 51 | 52 | 53 | 54 | 55 | 56 | Java failed, error message[${wf:errorMessage(wf:lastErrorNode())}] 57 | 58 | 59 | 60 | -------------------------------------------------------------------------------- /examples/java/job.properties: -------------------------------------------------------------------------------- 1 | # Copyright 2019 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | # 16 | # Licensed to the Apache Software Foundation (ASF) under one 17 | # or more contributor license agreements. See the NOTICE file 18 | # distributed with this work for additional information 19 | # regarding copyright ownership. The ASF licenses this file 20 | # to you under the Apache License, Version 2.0 (the 21 | # "License"); you may not use this file except in compliance 22 | # with the License. You may obtain a copy of the License at 23 | # 24 | # http://www.apache.org/licenses/LICENSE-2.0 25 | # 26 | # Unless required by applicable law or agreed to in writing, software 27 | # distributed under the License is distributed on an "AS IS" BASIS, 28 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 29 | # See the License for the specific language governing permissions and 30 | # limitations under the License. 31 | # 32 | 33 | nameNode=hdfs:// 34 | resourceManager=localhost:8032 35 | queueName=default 36 | examplesRoot=examples 37 | oozie.use.system.libpath=true 38 | oozie.wf.application.path=${nameNode}/user/${user.name}/${examplesRoot}/apps/java 39 | -------------------------------------------------------------------------------- /examples/mapreduce/configuration.template.properties: -------------------------------------------------------------------------------- 1 | # Copyright 2019 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | # 16 | # Licensed to the Apache Software Foundation (ASF) under one 17 | # or more contributor license agreements. See the NOTICE file 18 | # distributed with this work for additional information 19 | # regarding copyright ownership. The ASF licenses this file 20 | # to you under the Apache License, Version 2.0 (the 21 | # "License"); you may not use this file except in compliance 22 | # with the License. You may obtain a copy of the License at 23 | # 24 | # http://www.apache.org/licenses/LICENSE-2.0 25 | # 26 | # Unless required by applicable law or agreed to in writing, software 27 | # distributed under the License is distributed on an "AS IS" BASIS, 28 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 29 | # See the License for the specific language governing permissions and 30 | # limitations under the License. 31 | # 32 | 33 | dataproc_cluster={{DATAPROC_CLUSTER_NAME}} 34 | gcp_conn_id=google_cloud_default 35 | gcp_region={{GCP_REGION}} 36 | # A single item or a comma-separated list 37 | hadoop_jars=hdfs:///user/{{DATAPROC_USER}}/examples/apps/{{LOCAL_APP_NAME}}/lib/wordcount.jar 38 | hadoop_main_class=WordCount 39 | -------------------------------------------------------------------------------- /examples/mapreduce/hdfs/job-queue-conf.xml: -------------------------------------------------------------------------------- 1 | 16 | 17 | 18 | 19 | mapred.job.queue.name 20 | ${queueName} 21 | 22 | 23 | -------------------------------------------------------------------------------- /examples/mapreduce/job.properties: -------------------------------------------------------------------------------- 1 | # Copyright 2019 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | # 16 | # Licensed to the Apache Software Foundation (ASF) under one 17 | # or more contributor license agreements. See the NOTICE file 18 | # distributed with this work for additional information 19 | # regarding copyright ownership. The ASF licenses this file 20 | # to you under the Apache License, Version 2.0 (the 21 | # "License"); you may not use this file except in compliance 22 | # with the License. You may obtain a copy of the License at 23 | # 24 | # http://www.apache.org/licenses/LICENSE-2.0 25 | # 26 | # Unless required by applicable law or agreed to in writing, software 27 | # distributed under the License is distributed on an "AS IS" BASIS, 28 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 29 | # See the License for the specific language governing permissions and 30 | # limitations under the License. 31 | # 32 | 33 | nameNode=hdfs:// 34 | resourceManager=localhost:8032 35 | queueName=default 36 | examplesRoot=examples 37 | oozie.use.system.libpath=true 38 | oozie.wf.application.path=${nameNode}/user/${user.name}/${examplesRoot}/apps/mapreduce 39 | outputDir=output 40 | oozie.wf.workflow.notification.url=http://example.com:8080/workflow?job-id=$jobId&status=$status 41 | oozie.wf.action.notification.url=http://example.com:8080/action?job-id=$jobId&node-name=$nodeName&status=$status 42 | -------------------------------------------------------------------------------- /examples/pig/configuration.template.properties: -------------------------------------------------------------------------------- 1 | # Copyright 2019 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | # 16 | # Licensed to the Apache Software Foundation (ASF) under one 17 | # or more contributor license agreements. See the NOTICE file 18 | # distributed with this work for additional information 19 | # regarding copyright ownership. The ASF licenses this file 20 | # to you under the Apache License, Version 2.0 (the 21 | # "License"); you may not use this file except in compliance 22 | # with the License. You may obtain a copy of the License at 23 | # 24 | # http://www.apache.org/licenses/LICENSE-2.0 25 | # 26 | # Unless required by applicable law or agreed to in writing, software 27 | # distributed under the License is distributed on an "AS IS" BASIS, 28 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 29 | # See the License for the specific language governing permissions and 30 | # limitations under the License. 31 | # 32 | 33 | dataproc_cluster={{DATAPROC_CLUSTER_NAME}} 34 | gcp_conn_id=google_cloud_default 35 | gcp_region={{GCP_REGION}} 36 | gcp_uri_prefix=gs://{{COMPOSER_DAG_BUCKET}}/dags 37 | -------------------------------------------------------------------------------- /examples/pig/hdfs/id.pig: -------------------------------------------------------------------------------- 1 | -- Copyright 2019 Google LLC 2 | -- 3 | -- Licensed under the Apache License, Version 2.0 (the "License"); 4 | -- you may not use this file except in compliance with the License. 5 | -- You may obtain a copy of the License at 6 | -- 7 | -- http://www.apache.org/licenses/LICENSE-2.0 8 | -- 9 | -- Unless required by applicable law or agreed to in writing, software 10 | -- distributed under the License is distributed on an "AS IS" BASIS, 11 | -- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | -- See the License for the specific language governing permissions and 13 | -- limitations under the License. 14 | 15 | A = load '$INPUT' using PigStorage(':'); 16 | B = foreach A generate $0 as id; 17 | store B into '$OUTPUT' USING PigStorage(); 18 | dump B; 19 | -------------------------------------------------------------------------------- /examples/pig/hdfs/input-data/test-data.txt: -------------------------------------------------------------------------------- 1 | 1850,0,1,1483789 2 | 2000,90,2,1064581 3 | -------------------------------------------------------------------------------- /examples/pig/hdfs/test_dir/test.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GoogleCloudPlatform/oozie-to-airflow/6bfa193dc61e056d9e54a334790dd5411db1c2b6/examples/pig/hdfs/test_dir/test.txt -------------------------------------------------------------------------------- /examples/pig/hdfs/test_dir/test2.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GoogleCloudPlatform/oozie-to-airflow/6bfa193dc61e056d9e54a334790dd5411db1c2b6/examples/pig/hdfs/test_dir/test2.zip -------------------------------------------------------------------------------- /examples/pig/hdfs/test_dir/test3.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GoogleCloudPlatform/oozie-to-airflow/6bfa193dc61e056d9e54a334790dd5411db1c2b6/examples/pig/hdfs/test_dir/test3.zip -------------------------------------------------------------------------------- /examples/pig/job.properties: -------------------------------------------------------------------------------- 1 | # Copyright 2019 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | # 16 | # Licensed to the Apache Software Foundation (ASF) under one 17 | # or more contributor license agreements. See the NOTICE file 18 | # distributed with this work for additional information 19 | # regarding copyright ownership. The ASF licenses this file 20 | # to you under the Apache License, Version 2.0 (the 21 | # "License"); you may not use this file except in compliance 22 | # with the License. You may obtain a copy of the License at 23 | # 24 | # http://www.apache.org/licenses/LICENSE-2.0 25 | # 26 | # Unless required by applicable law or agreed to in writing, software 27 | # distributed under the License is distributed on an "AS IS" BASIS, 28 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 29 | # See the License for the specific language governing permissions and 30 | # limitations under the License. 31 | # 32 | 33 | nameNode=hdfs:// 34 | resourceManager=localhost:8032 35 | queueName=default 36 | examplesRoot=examples 37 | oozie.use.system.libpath=true 38 | oozie.wf.application.path=${nameNode}/user/${user.name}/${examplesRoot}/apps/pig 39 | -------------------------------------------------------------------------------- /examples/shell/configuration.template.properties: -------------------------------------------------------------------------------- 1 | # Copyright 2019 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | dataproc_cluster={{DATAPROC_CLUSTER_NAME}} 16 | gcp_region={{GCP_REGION}} 17 | -------------------------------------------------------------------------------- /examples/shell/job.properties: -------------------------------------------------------------------------------- 1 | # Copyright 2019 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | # 16 | # Licensed to the Apache Software Foundation (ASF) under one 17 | # or more contributor license agreements. See the NOTICE file 18 | # distributed with this work for additional information 19 | # regarding copyright ownership. The ASF licenses this file 20 | # to you under the Apache License, Version 2.0 (the 21 | # "License"); you may not use this file except in compliance 22 | # with the License. You may obtain a copy of the License at 23 | # 24 | # http://www.apache.org/licenses/LICENSE-2.0 25 | # 26 | # Unless required by applicable law or agreed to in writing, software 27 | # distributed under the License is distributed on an "AS IS" BASIS, 28 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 29 | # See the License for the specific language governing permissions and 30 | # limitations under the License. 31 | # 32 | 33 | nameNode=hdfs://localhost:8020 34 | resourceManager=localhost:8032 35 | queueName=default 36 | examplesRoot=examples 37 | oozie.wf.application.path=${nameNode}/user/${user.name}/${examplesRoot}/apps/shell 38 | -------------------------------------------------------------------------------- /examples/spark/configuration.template.properties: -------------------------------------------------------------------------------- 1 | # Copyright 2019 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | # 16 | # Licensed to the Apache Software Foundation (ASF) under one 17 | # or more contributor license agreements. See the NOTICE file 18 | # distributed with this work for additional information 19 | # regarding copyright ownership. The ASF licenses this file 20 | # to you under the Apache License, Version 2.0 (the 21 | # "License"); you may not use this file except in compliance 22 | # with the License. You may obtain a copy of the License at 23 | # 24 | # http://www.apache.org/licenses/LICENSE-2.0 25 | # 26 | # Unless required by applicable law or agreed to in writing, software 27 | # distributed under the License is distributed on an "AS IS" BASIS, 28 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 29 | # See the License for the specific language governing permissions and 30 | # limitations under the License. 31 | # 32 | 33 | dataproc_cluster={{DATAPROC_CLUSTER_NAME}} 34 | gcp_conn_id=google_cloud_default 35 | gcp_region={{GCP_REGION}} 36 | gcp_uri_prefix=gs://{{COMPOSER_DAG_BUCKET}}/dags 37 | -------------------------------------------------------------------------------- /examples/spark/job.properties: -------------------------------------------------------------------------------- 1 | # Copyright 2019 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | # 16 | # Licensed to the Apache Software Foundation (ASF) under one 17 | # or more contributor license agreements. See the NOTICE file 18 | # distributed with this work for additional information 19 | # regarding copyright ownership. The ASF licenses this file 20 | # to you under the Apache License, Version 2.0 (the 21 | # "License"); you may not use this file except in compliance 22 | # with the License. You may obtain a copy of the License at 23 | # 24 | # http://www.apache.org/licenses/LICENSE-2.0 25 | # 26 | # Unless required by applicable law or agreed to in writing, software 27 | # distributed under the License is distributed on an "AS IS" BASIS, 28 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 29 | # See the License for the specific language governing permissions and 30 | # limitations under the License. 31 | # 32 | 33 | nameNode=hdfs:// 34 | resourceManager=localhost:8032 35 | master=local[*] 36 | queueName=default 37 | examplesRoot=examples 38 | oozie.use.system.libpath=true 39 | oozie.wf.application.path=${nameNode}/user/${user.name}/${examplesRoot}/apps/spark 40 | -------------------------------------------------------------------------------- /examples/ssh/hdfs/workflow.xml: -------------------------------------------------------------------------------- 1 | 16 | 17 | 34 | 35 | 36 | 37 | 38 | 39 | user@apache.org 40 | echo 41 | "Hello Oozie!" 42 | 43 | 44 | 45 | 46 | 47 | 48 | SSH action failed, error message[${wf:errorMessage(wf:lastErrorNode())}] 49 | 50 | 51 | 52 | 53 | -------------------------------------------------------------------------------- /examples/ssh/job.properties: -------------------------------------------------------------------------------- 1 | # Copyright 2019 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | # 16 | # Licensed to the Apache Software Foundation (ASF) under one 17 | # or more contributor license agreements. See the NOTICE file 18 | # distributed with this work for additional information 19 | # regarding copyright ownership. The ASF licenses this file 20 | # to you under the Apache License, Version 2.0 (the 21 | # "License"); you may not use this file except in compliance 22 | # with the License. You may obtain a copy of the License at 23 | # 24 | # http://www.apache.org/licenses/LICENSE-2.0 25 | # 26 | # Unless required by applicable law or agreed to in writing, software 27 | # distributed under the License is distributed on an "AS IS" BASIS, 28 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 29 | # See the License for the specific language governing permissions and 30 | # limitations under the License. 31 | # 32 | 33 | nameNode=hdfs:// 34 | resourceManager=localhost:8032 35 | queueName=default 36 | examplesRoot=examples 37 | oozie.wf.application.path=${nameNode}/user/${user.name}/${examplesRoot}/apps/ssh 38 | -------------------------------------------------------------------------------- /examples/subwf/configuration.template.properties: -------------------------------------------------------------------------------- 1 | # Copyright 2019 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | # 16 | # Licensed to the Apache Software Foundation (ASF) under one 17 | # or more contributor license agreements. See the NOTICE file 18 | # distributed with this work for additional information 19 | # regarding copyright ownership. The ASF licenses this file 20 | # to you under the Apache License, Version 2.0 (the 21 | # "License"); you may not use this file except in compliance 22 | # with the License. You may obtain a copy of the License at 23 | # 24 | # http://www.apache.org/licenses/LICENSE-2.0 25 | # 26 | # Unless required by applicable law or agreed to in writing, software 27 | # distributed under the License is distributed on an "AS IS" BASIS, 28 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 29 | # See the License for the specific language governing permissions and 30 | # limitations under the License. 31 | # 32 | 33 | dataproc_cluster={{DATAPROC_CLUSTER_NAME}} 34 | gcp_conn_id=google_cloud_default 35 | gcp_region={{GCP_REGION}} 36 | gcp_uri_prefix=gs://{{COMPOSER_DAG_BUCKET}}/dags 37 | -------------------------------------------------------------------------------- /examples/subwf/job.properties: -------------------------------------------------------------------------------- 1 | # Copyright 2019 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | # 16 | # Licensed to the Apache Software Foundation (ASF) under one 17 | # or more contributor license agreements. See the NOTICE file 18 | # distributed with this work for additional information 19 | # regarding copyright ownership. The ASF licenses this file 20 | # to you under the Apache License, Version 2.0 (the 21 | # "License"); you may not use this file except in compliance 22 | # with the License. You may obtain a copy of the License at 23 | # 24 | # http://www.apache.org/licenses/LICENSE-2.0 25 | # 26 | # Unless required by applicable law or agreed to in writing, software 27 | # distributed under the License is distributed on an "AS IS" BASIS, 28 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 29 | # See the License for the specific language governing permissions and 30 | # limitations under the License. 31 | # 32 | 33 | nameNode=hdfs:// 34 | resourceManager=localhost:8032 35 | queueName=default 36 | examplesRoot=examples 37 | oozie.use.system.libpath = true 38 | oozie.wf.application.path=${nameNode}/user/${user.name}/${examplesRoot}/apps/subwf 39 | -------------------------------------------------------------------------------- /images/childwf_with_notifications.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GoogleCloudPlatform/oozie-to-airflow/6bfa193dc61e056d9e54a334790dd5411db1c2b6/images/childwf_with_notifications.png -------------------------------------------------------------------------------- /images/childwf_without_notifications.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GoogleCloudPlatform/oozie-to-airflow/6bfa193dc61e056d9e54a334790dd5411db1c2b6/images/childwf_without_notifications.png -------------------------------------------------------------------------------- /images/o2a-dependencies.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GoogleCloudPlatform/oozie-to-airflow/6bfa193dc61e056d9e54a334790dd5411db1c2b6/images/o2a-dependencies.png -------------------------------------------------------------------------------- /images/o2a-dependency-cycles.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GoogleCloudPlatform/oozie-to-airflow/6bfa193dc61e056d9e54a334790dd5411db1c2b6/images/o2a-dependency-cycles.png -------------------------------------------------------------------------------- /mypy.ini: -------------------------------------------------------------------------------- 1 | # Global options: 2 | 3 | [mypy] 4 | python_version = 3.8 5 | warn_return_any = True 6 | warn_unused_configs = True 7 | warn_unused_ignores = False 8 | mypy_path=. 9 | ignore_missing_imports = True 10 | -------------------------------------------------------------------------------- /o2a/__init__.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Copyright 2019 Google LLC 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | """Oozie-to-Airflow converter""" 16 | NAME = "o2a" 17 | -------------------------------------------------------------------------------- /o2a/converter/__init__.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Copyright 2019 Google LLC 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | """Convert-related functions""" 16 | -------------------------------------------------------------------------------- /o2a/converter/constants.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Copyright 2019 Google LLC 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | """Custom constants""" 16 | 17 | # Workflow.xml and all HDFS files should be located in this subfolder 18 | HDFS_FOLDER = "hdfs" 19 | # Lib files are added by default to the path of the mapper specified 20 | LIB_FOLDER = "lib" 21 | -------------------------------------------------------------------------------- /o2a/converter/exceptions.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Copyright 2019 Google LLC 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | """Custom exceptions""" 16 | 17 | 18 | class O2AException(Exception): 19 | """Base class for all exceptions raised by Oozie-to-Airflow.""" 20 | 21 | 22 | class ParseException(O2AException): 23 | """Raised when an error occurs in the parsing phase.""" 24 | -------------------------------------------------------------------------------- /o2a/converter/mappers.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Copyright 2019 Google LLC 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | """Mappers defined for the converter. 16 | 17 | This module contains mappings between Oozie actions and corresponding mappers that handle 18 | particular actions. 19 | 20 | """ 21 | 22 | from typing import Type, Dict 23 | 24 | from o2a.mappers.distcp_mapper import DistCpMapper 25 | from o2a.mappers.action_mapper import ActionMapper 26 | from o2a.mappers.java_mapper import JavaMapper 27 | from o2a.mappers.email_mapper import EmailMapper 28 | from o2a.mappers.fs_mapper import FsMapper 29 | from o2a.mappers.git_mapper import GitMapper 30 | from o2a.mappers.hive_mapper import HiveMapper 31 | from o2a.mappers.mapreduce_mapper import MapReduceMapper 32 | from o2a.mappers.pig_mapper import PigMapper 33 | from o2a.mappers.shell_mapper import ShellMapper 34 | from o2a.mappers.spark_mapper import SparkMapper 35 | from o2a.mappers.ssh_mapper import SSHMapper 36 | from o2a.mappers.subworkflow_mapper import SubworkflowMapper 37 | 38 | ACTION_MAP: Dict[str, Type[ActionMapper]] = { 39 | "ssh": SSHMapper, 40 | "spark": SparkMapper, 41 | "pig": PigMapper, 42 | "fs": FsMapper, 43 | "java": JavaMapper, 44 | "sub-workflow": SubworkflowMapper, 45 | "shell": ShellMapper, 46 | "map-reduce": MapReduceMapper, 47 | "git": GitMapper, 48 | "hive": HiveMapper, 49 | "hive2": HiveMapper, 50 | "distcp": DistCpMapper, 51 | "email": EmailMapper, 52 | } 53 | -------------------------------------------------------------------------------- /o2a/converter/property_parser.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Copyright 2019 Google LLC 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | """PropertyParser""" 16 | import os 17 | 18 | from o2a.converter.workflow import Workflow 19 | from o2a.o2a_libs.src.o2a_lib.property_utils import PropertySet 20 | from o2a.utils import el_utils 21 | from o2a.utils.constants import CONFIG, JOB_PROPS 22 | 23 | 24 | class PropertyParser: 25 | """ 26 | Parse configuration.properties and job.properties to PropertySet 27 | """ 28 | 29 | def __init__(self, workflow: Workflow, props: PropertySet): 30 | self.config_file = os.path.join(workflow.input_directory_path, CONFIG) 31 | self.job_properties_file = os.path.join(workflow.input_directory_path, JOB_PROPS) 32 | self.props = props 33 | 34 | def parse_property(self): 35 | self.read_and_update_job_properties_replace_el() 36 | self.read_config_replace_el() 37 | 38 | def read_config_replace_el(self): 39 | """ 40 | Reads configuration properties to config dictionary. 41 | Replaces EL properties within. 42 | 43 | :return: None 44 | """ 45 | self.props.config = el_utils.extract_evaluate_properties( 46 | properties_file=self.config_file, props=self.props 47 | ) 48 | 49 | def read_and_update_job_properties_replace_el(self): 50 | """ 51 | Reads job properties and updates job_properties dictionary with the read values 52 | Replaces EL job_properties within. 53 | 54 | :return: None 55 | """ 56 | self.props.job_properties.update( 57 | el_utils.extract_evaluate_properties(properties_file=self.job_properties_file, props=self.props) 58 | ) 59 | -------------------------------------------------------------------------------- /o2a/converter/relation.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Copyright 2019 Google LLC 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | """Relation between tasks""" 16 | from typing import NamedTuple 17 | 18 | 19 | class Relation(NamedTuple): 20 | """Class for Airflow relation""" 21 | 22 | from_task_id: str 23 | to_task_id: str 24 | is_error: bool = False 25 | -------------------------------------------------------------------------------- /o2a/converter/task.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Copyright 2019 Google LLC 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | """Representation of Airflow tasks""" 16 | from typing import Dict, Any 17 | 18 | from airflow.utils.trigger_rule import TriggerRule 19 | 20 | from o2a.utils.template_utils import render_template 21 | 22 | 23 | # This is a container for data, so it does not contain public methods intentionally. 24 | class Task: # pylint: disable=too-few-public-methods 25 | """Class for Airflow Task""" 26 | 27 | def __init__( 28 | self, 29 | task_id: str, 30 | template_name: str, 31 | trigger_rule: str = TriggerRule.ONE_SUCCESS, 32 | template_params: Dict[str, Any] = None, 33 | ): 34 | self.task_id = task_id 35 | self.template_name = template_name 36 | self.trigger_rule = trigger_rule 37 | self.template_params: Dict[str, Any] = template_params or {} 38 | 39 | @property 40 | def rendered_template(self): 41 | return render_template( 42 | template_name=self.template_name, 43 | task_id=self.task_id, 44 | trigger_rule=self.trigger_rule, 45 | **self.template_params, 46 | ) 47 | 48 | def __repr__(self) -> str: 49 | return ( 50 | f'Task(task_id="{self.task_id}", ' 51 | f'template_name="{self.template_name}", ' 52 | f'trigger_rule="{self.trigger_rule}", ' 53 | f"template_params={self.template_params})" 54 | ) 55 | 56 | def __eq__(self, other): 57 | if isinstance(other, self.__class__): 58 | return self.__dict__ == other.__dict__ 59 | return False 60 | -------------------------------------------------------------------------------- /o2a/definitions.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Copyright 2019 Google LLC 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | """Common definitions used across the converter""" 16 | import os 17 | 18 | ROOT_DIR = os.path.dirname(os.path.abspath(__file__)) 19 | TPL_PATH = os.path.join(ROOT_DIR, "templates/") 20 | 21 | O2A_PROJECT_PATH = os.path.abspath(os.path.join(os.path.dirname(__file__), os.path.pardir)) 22 | EXAMPLES_PATH = os.path.join(O2A_PROJECT_PATH, "examples") 23 | 24 | # Mapper examples 25 | EXAMPLE_DEMO_PATH = os.path.join(EXAMPLES_PATH, "demo") 26 | EXAMPLE_EL_PATH = os.path.join(EXAMPLES_PATH, "el") 27 | EXAMPLE_PIG_PATH = os.path.join(EXAMPLES_PATH, "pig") 28 | EXAMPLE_SUBWORKFLOW_PATH = os.path.join(EXAMPLES_PATH, "subwf") 29 | -------------------------------------------------------------------------------- /o2a/mappers/__init__.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Copyright 2019 Google LLC 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | """This module contains all mappers used in Oozie to Airflow converter.""" 16 | -------------------------------------------------------------------------------- /o2a/mappers/dummy_mapper.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Copyright 2019 Google LLC 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | """Dummy Mapper that is used as temporary solution while we are implementing the real mappers. 16 | """ 17 | from typing import List, Optional, Set 18 | from xml.etree.ElementTree import Element 19 | 20 | from o2a.converter.relation import Relation 21 | from o2a.converter.task import Task 22 | from o2a.mappers.base_mapper import BaseMapper 23 | from o2a.o2a_libs.src.o2a_lib.property_utils import PropertySet 24 | 25 | 26 | class DummyMapper(BaseMapper): 27 | """Dummy mapper. It always returns one tasks that does nothing. 28 | 29 | It's used in place of not-yet-implemented mappers and as the base class for control nodes. 30 | """ 31 | 32 | def __init__( 33 | self, oozie_node: Element, name: str, dag_name: str, props: Optional[PropertySet] = None, **kwargs 34 | ): 35 | super().__init__( 36 | oozie_node=oozie_node, 37 | name=name, 38 | dag_name=dag_name, 39 | props=props or PropertySet(job_properties={}, config={}), 40 | **kwargs, 41 | ) 42 | 43 | def to_tasks_and_relations(self): 44 | tasks: List[Task] = [Task(task_id=self.name, template_name="dummy.tpl")] 45 | relations: List[Relation] = [] 46 | return tasks, relations 47 | 48 | def required_imports(self) -> Set[str]: 49 | return {"from airflow.operators import empty"} 50 | -------------------------------------------------------------------------------- /o2a/mappers/end_mapper.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Copyright 2019 Google LLC 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | """Maps Oozie end node to Airflow's DAG""" 16 | from o2a.mappers.dummy_mapper import DummyMapper 17 | 18 | 19 | class EndMapper(DummyMapper): 20 | pass 21 | -------------------------------------------------------------------------------- /o2a/mappers/fork_mapper.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Copyright 2019 Google LLC 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | """Maps Oozie fork node to Airflow's task""" 16 | from o2a.mappers.dummy_mapper import DummyMapper 17 | 18 | 19 | class ForkMapper(DummyMapper): 20 | """ 21 | Converts a fork node to an Airflow's task. 22 | """ 23 | -------------------------------------------------------------------------------- /o2a/mappers/join_mapper.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Copyright 2019 Google LLC 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | """Maps Oozie join node to Airflow's task""" 16 | from typing import List 17 | 18 | from airflow.utils.trigger_rule import TriggerRule 19 | 20 | from o2a.converter.relation import Relation 21 | from o2a.converter.task import Task 22 | from o2a.mappers.dummy_mapper import DummyMapper 23 | 24 | 25 | class JoinMapper(DummyMapper): 26 | """ 27 | Converts a Join node to an Airflow's task. 28 | """ 29 | 30 | def to_tasks_and_relations(self): 31 | tasks: List[Task] = [ 32 | Task(task_id=self.name, template_name="dummy.tpl", trigger_rule=TriggerRule.ALL_SUCCESS) 33 | ] 34 | relations: List[Relation] = [] 35 | return tasks, relations 36 | -------------------------------------------------------------------------------- /o2a/mappers/kill_mapper.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Copyright 2019 Google LLC 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | """Kill mapper - maps the workflow end""" 16 | 17 | from o2a.mappers.dummy_mapper import DummyMapper 18 | 19 | 20 | class KillMapper(DummyMapper): 21 | """ 22 | Converts a Kill Oozie node to an Airflow task. 23 | """ 24 | -------------------------------------------------------------------------------- /o2a/mappers/start_mapper.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Copyright 2019 Google LLC 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | """Maps Oozie start node to Airflow's DAG""" 16 | 17 | from o2a.mappers.dummy_mapper import DummyMapper 18 | 19 | 20 | class StartMapper(DummyMapper): 21 | """Maps start node""" 22 | -------------------------------------------------------------------------------- /o2a/o2a_libs/README.md: -------------------------------------------------------------------------------- 1 | # o2a Library 2 | 3 | This is an additional library for Oozie To Airflow migration tool. 4 | Importing this package is required by the Airflow DAGs generated by the Oozie To Airflow migration tool -------------------------------------------------------------------------------- /o2a/o2a_libs/__init__.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Copyright 2019 Google LLC 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | # airflow DAG -required for some versions of airflow to correctly import 17 | -------------------------------------------------------------------------------- /o2a/o2a_libs/pyproject.toml: -------------------------------------------------------------------------------- 1 | [build-system] 2 | requires = ["setuptools>=61.0"] 3 | build-backend = "setuptools.build_meta" 4 | 5 | [project] 6 | name = "o2a_lib" 7 | version = "2.0.1" 8 | authors = [ 9 | { name="Jarek Potiuk", email="jarek.potiuk@polidea.com" }, 10 | { name="Szymon Przedwojski", email="szymon.przedwojski@polidea.com" }, 11 | { name="Kamil Breguła", email="kamil.bregula@polidea.com" }, 12 | { name="Feng Lu", email="fenglu@google.com" }, 13 | { name="Cameron Moberg", email="cjmoberg@google.com" }, 14 | ] 15 | description = "Additional library for Oozie To Airflow migration tool" 16 | readme = "README.md" 17 | requires-python = ">=3.8" 18 | classifiers = [ 19 | "Programming Language :: Python :: 3.8", 20 | "Programming Language :: Python :: 3.9", 21 | "Programming Language :: Python :: 3.10", 22 | "License :: OSI Approved :: Apache Software License", 23 | "Operating System :: OS Independent", 24 | ] 25 | 26 | [project.urls] 27 | "Homepage" = "https://github.com/GoogleCloudPlatform/oozie-to-airflow/tree/master/o2a/o2a_libs" 28 | "Bug Tracker" = "https://github.com/GoogleCloudPlatform/oozie-to-airflow/issues" 29 | -------------------------------------------------------------------------------- /o2a/o2a_libs/src/__init__.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Copyright 2019 Google LLC 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | # airflow DAG -required for some versions of airflow to correctly import 17 | -------------------------------------------------------------------------------- /o2a/o2a_libs/src/o2a_lib/__init__.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Copyright 2019 Google LLC 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | # airflow DAG -required for some versions of airflow to correctly import 17 | -------------------------------------------------------------------------------- /o2a/schema/all-schemas-1.0.xsd: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /o2a/schema/distcp-action-1.0.xsd: -------------------------------------------------------------------------------- 1 | 2 | 19 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | -------------------------------------------------------------------------------- /o2a/schema/email-action-0.2.xsd: -------------------------------------------------------------------------------- 1 | 2 | 19 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | -------------------------------------------------------------------------------- /o2a/schema/git-action-1.0.xsd: -------------------------------------------------------------------------------- 1 | 2 | 17 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | -------------------------------------------------------------------------------- /o2a/schema/hive-action-1.0.xsd: -------------------------------------------------------------------------------- 1 | 2 | 19 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | 49 | 50 | -------------------------------------------------------------------------------- /o2a/schema/shell-action-1.0.xsd: -------------------------------------------------------------------------------- 1 | 2 | 19 | 20 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | 49 | -------------------------------------------------------------------------------- /o2a/schema/ssh-action-0.2.xsd: -------------------------------------------------------------------------------- 1 | 2 | 19 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | -------------------------------------------------------------------------------- /o2a/scripts/prepare.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # Copyright 2019 Google LLC 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | set -x 18 | 19 | DEL_DIRS=() 20 | MK_DIRS=() 21 | 22 | while getopts ":c:r:d:m:" OPT; do 23 | case ${OPT} in 24 | c) CLUSTER="$OPTARG";; 25 | r) REGION="$OPTARG";; 26 | d) DEL_DIRS+=("$OPTARG");; 27 | m) MK_DIRS+=("$OPTARG");; 28 | \?) 29 | echo "Invalid option: -$OPTARG" >&2 30 | exit 1 31 | ;; 32 | :) 33 | echo "Option -$OPTARG requires an argument." >&2 34 | exit 1 35 | ;; 36 | esac 37 | done 38 | shift "$((OPTIND -1))" 39 | 40 | 41 | for DEL_DIR in "${DEL_DIRS[@]}"; do 42 | # Gcloud expects the path not to contain any apostrophes or quotation marks, hence just the ${DEL_DIR}. 43 | gcloud dataproc jobs submit pig --cluster="${CLUSTER}" --region="${REGION}" --execute "fs -rm -f -r ${DEL_DIR}" 44 | done 45 | 46 | for MK_DIR in "${MK_DIRS[@]}"; do 47 | # Gcloud expects the path not to contain any apostrophes or quotation marks, hence just the ${DEL_DIR}. 48 | gcloud dataproc jobs submit pig --cluster="${CLUSTER}" --region="${REGION}" --execute "fs -mkdir -p ${MK_DIR}" 49 | done 50 | -------------------------------------------------------------------------------- /o2a/templates/dag_body.tpl: -------------------------------------------------------------------------------- 1 | {# 2 | Copyright 2019 Google LLC 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | #} 16 | {% for task_group in task_groups %} 17 | {% for task in task_group.all_tasks %} 18 | {{ task.rendered_template }} 19 | {% endfor %} 20 | {% for relation in task_group.relations %} 21 | {{ relation.from_task_id | to_var }}.set_downstream({{ relation.to_task_id | to_var }}) 22 | {% endfor %} 23 | {% endfor %} 24 | 25 | {% for relation in relations %} 26 | {{ relation.from_task_id | to_var }}.set_downstream({{ relation.to_task_id | to_var }}) 27 | {% endfor %} 28 | -------------------------------------------------------------------------------- /o2a/templates/decision.tpl: -------------------------------------------------------------------------------- 1 | {# 2 | Copyright 2019 Google LLC 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | #} 16 | def {{ task_id | to_var }}_decision(*args, **kwargs): 17 | {% filter indent(4, True) %} 18 | task = kwargs.get('task') 19 | decisions = {{ case_dict | to_python }} 20 | for (predicate, decision) in decisions.items(): 21 | {% filter indent(4, True) %} 22 | value = task.render_template(content=predicate, context=TEMPLATE_ENV) 23 | if value in ("true", "True", 1): 24 | return decision 25 | {% endfilter %} 26 | return {{default_case | to_python}} 27 | {% endfilter %} 28 | 29 | 30 | {{ task_id | to_var }} = python.BranchPythonOperator( 31 | task_id={{ task_id | to_python }}, 32 | trigger_rule={{ trigger_rule | to_python }}, 33 | python_callable={{ task_id | to_var }}_decision, 34 | ) 35 | -------------------------------------------------------------------------------- /o2a/templates/distcp.tpl: -------------------------------------------------------------------------------- 1 | {# 2 | Copyright 2019 Google LLC 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | #} 16 | 17 | {% import "macros/props.tpl" as props_macro %} 18 | {{ task_id | to_var }} = bash.BashOperator( 19 | task_id={{ task_id | tojson }}, 20 | trigger_rule={{ trigger_rule | tojson }}, 21 | bash_command={% include "hadoop_command.tpl" %} % (CONFIG['dataproc_cluster'], CONFIG['gcp_region'], 22 | {{ distcp_command | to_python }}), 23 | params={{ props_macro.props(action_node_properties=action_node_properties) }}, 24 | ) 25 | -------------------------------------------------------------------------------- /o2a/templates/dummy.tpl: -------------------------------------------------------------------------------- 1 | {# 2 | Copyright 2019 Google LLC 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | #} 16 | {{ task_id | to_var }} = empty.EmptyOperator( 17 | task_id={{ task_id | to_python }}, 18 | trigger_rule={{ trigger_rule | to_python }} 19 | ) 20 | -------------------------------------------------------------------------------- /o2a/templates/email.tpl: -------------------------------------------------------------------------------- 1 | {# 2 | Copyright 2019 Google LLC 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | #} 16 | {% import "macros/props.tpl" as props_macro %} 17 | {{ task_id | to_var }} = email.EmailOperator( 18 | task_id={{ task_id | to_python }}, 19 | trigger_rule={{ trigger_rule | to_python }}, 20 | to={{ to_addr | to_python }}, 21 | cc={{ cc_addr | to_python }}, 22 | bcc={{ bcc_addr | to_python }}, 23 | subject={{ subject | to_python }}, 24 | html_content={{ body | to_python }}, 25 | params={{ props_macro.props(action_node_properties=action_node_properties) }}, 26 | ) 27 | -------------------------------------------------------------------------------- /o2a/templates/fs_op.tpl: -------------------------------------------------------------------------------- 1 | {# 2 | Copyright 2019 Google LLC 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | #} 16 | {% import "macros/props.tpl" as props_macro %} 17 | {{ task_id | to_var }} = bash.BashOperator( 18 | task_id={{ task_id | to_python }}, 19 | trigger_rule={{ trigger_rule | to_python }}, 20 | bash_command={% include "pig_command.tpl" %} % (CONFIG['dataproc_cluster'], CONFIG['gcp_region'], 21 | shlex.quote({{ pig_command | to_python }})), 22 | params={{ props_macro.props(action_node_properties=action_node_properties) }}, 23 | ) 24 | -------------------------------------------------------------------------------- /o2a/templates/git.tpl: -------------------------------------------------------------------------------- 1 | {# 2 | Copyright 2019 Google LLC 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | #} 16 | {% import "macros/props.tpl" as props_macro %} 17 | {{ task_id | to_var }} = bash.BashOperator( 18 | task_id={{ task_id | to_python }}, 19 | trigger_rule={{ trigger_rule | to_python }}, 20 | bash_command={% include "git_command.tpl" %}, 21 | params={{ props_macro.props(action_node_properties=action_node_properties) }}, 22 | ) 23 | -------------------------------------------------------------------------------- /o2a/templates/git_command.tpl: -------------------------------------------------------------------------------- 1 | {# 2 | Copyright 2019 Google LLC 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | #} 16 | "$DAGS_FOLDER/../data/git.sh " 17 | "--cluster=%s " 18 | "--region=%s " 19 | "--git-uri %s " 20 | "--destination-path %s " 21 | {% if git_branch != '' %}"--branch %s " {% endif %} 22 | {% if key_path != '' %}"--key_path %s " {% endif %} 23 | % (CONFIG['dataproc_cluster'], CONFIG['gcp_region'], 24 | shlex.quote({{ git_uri | to_python }}), 25 | shlex.quote({{ destination_path | to_python }}), 26 | {% if git_branch != '' %}shlex.quote({{ git_branch | to_python }}),{% endif %} 27 | {% if key_path != '' %}shlex.quote({{ key_path | to_python }}),{% endif %} 28 | ) 29 | -------------------------------------------------------------------------------- /o2a/templates/hadoop_command.tpl: -------------------------------------------------------------------------------- 1 | {# 2 | Copyright 2019 Google LLC 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | #} 16 | "gcloud dataproc jobs submit hadoop " 17 | "--cluster=%s " 18 | "--region=%s " 19 | "%s" 20 | -------------------------------------------------------------------------------- /o2a/templates/hive.tpl: -------------------------------------------------------------------------------- 1 | {# 2 | Copyright 2019 Google LLC 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | #} 16 | 17 | {% import "macros/props.tpl" as props_macro %} 18 | {{ task_id | to_var }} = DataprocSubmitJobOperator( 19 | task_id={{ task_id | to_python }}, 20 | trigger_rule={{ trigger_rule | to_python }}, 21 | job=dict( 22 | placement=dict( 23 | cluster_name=CONFIG['dataproc_cluster'], 24 | ), 25 | hive_job=dict( 26 | {% if variables %}script_variables={{ variables | to_python }},{% endif %} 27 | properties={{ props_macro.props(action_node_properties=action_node_properties, xml_escaped=True) }}, 28 | {% if script %}query_file_uri='{}/{}'.format(CONFIG['gcp_uri_prefix'], {{ script | to_python }}),{% endif %} 29 | {% if query_obj %}query_list={{ query_obj | to_python }},{% endif %} 30 | ), 31 | ), 32 | gcp_conn_id=CONFIG['gcp_conn_id'], 33 | region=CONFIG['gcp_region'], 34 | ) 35 | -------------------------------------------------------------------------------- /o2a/templates/http.tpl: -------------------------------------------------------------------------------- 1 | {# 2 | Copyright 2019 Google LLC 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | #} 16 | 17 | {% import "macros/props.tpl" as props_macro %} 18 | {{ task_id | to_var }} = bash.BashOperator( 19 | task_id={{ task_id | to_python }}, 20 | trigger_rule={{ trigger_rule | to_python }}, 21 | bash_command={% include "http_command.tpl" %}, 22 | params={{ props_macro.props(action_node_properties=action_node_properties) }}, 23 | ) 24 | -------------------------------------------------------------------------------- /o2a/templates/http_command.tpl: -------------------------------------------------------------------------------- 1 | {# 2 | Copyright 2019 Google LLC 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | #} 16 | "curl %s; exit {{ error_code }}" 17 | % ( 18 | shlex.quote({{ url | to_python }}), 19 | ) 20 | -------------------------------------------------------------------------------- /o2a/templates/java.tpl: -------------------------------------------------------------------------------- 1 | {# 2 | Copyright 2019 Google LLC 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | #} 16 | {% import "macros/props.tpl" as props_macro %} 17 | {{ task_id | to_var }} = DataprocSubmitJobOperator( 18 | task_id={{ task_id | to_python }}, 19 | trigger_rule={{ trigger_rule | to_python }}, 20 | job=dict( 21 | placement=dict( 22 | cluster_name=CONFIG['dataproc_cluster'], 23 | ), 24 | hadoop_job={{ hadoop_job | to_python }}, 25 | ), 26 | gcp_conn_id=CONFIG['gcp_conn_id'], 27 | region=CONFIG['gcp_region'], 28 | params={{ props_macro.props(action_node_properties=action_node_properties) }}, 29 | ) 30 | -------------------------------------------------------------------------------- /o2a/templates/kill.tpl: -------------------------------------------------------------------------------- 1 | {# 2 | Copyright 2019 Google LLC 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | #} 16 | {{ task_id | to_var }} = bash.BashOperator( 17 | task_id={{ task_id | to_python }}, 18 | trigger_rule={{ trigger_rule | to_python }}, 19 | bash_command='exit 1', 20 | ) 21 | -------------------------------------------------------------------------------- /o2a/templates/macros/props.tpl: -------------------------------------------------------------------------------- 1 | {# 2 | Copyright 2019 Google LLC 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | #} 16 | {% macro props(action_node_properties, xml_escaped=False) -%} 17 | {% if (action_node_properties is defined) and (action_node_properties | length != 0) -%} 18 | PropertySet( 19 | config=CONFIG, 20 | job_properties=JOB_PROPS, 21 | action_node_properties={{ action_node_properties | to_python }}).{% if xml_escaped %}xml_escaped.{% endif %}merged 22 | {% else -%} 23 | PropertySet( 24 | config=CONFIG, 25 | job_properties=JOB_PROPS).{% if xml_escaped %}xml_escaped.{% endif %}merged 26 | {% endif %} 27 | {%- endmacro %} 28 | -------------------------------------------------------------------------------- /o2a/templates/mapreduce.tpl: -------------------------------------------------------------------------------- 1 | {# 2 | Copyright 2019 Google LLC 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | #} 16 | {% import "macros/props.tpl" as props_macro %} 17 | {{ task_id | to_var }} = DataprocSubmitJobOperator( 18 | task_id={{ task_id | to_python }}, 19 | trigger_rule={{ trigger_rule | to_python }}, 20 | job=dict( 21 | placement=dict( 22 | cluster_name=CONFIG['dataproc_cluster'], 23 | ), 24 | hadoop_job=dict( 25 | args=[ 26 | "{{ '{{' }} params['mapreduce.input.fileinputformat.inputdir'] {{ '}}' }}", 27 | "{{ '{{' }} params['mapreduce.output.fileoutputformat.outputdir'] {{ '}}' }}" 28 | ], 29 | jar_file_uris=CONFIG['hadoop_jars'].split(','), 30 | {% if hdfs_files %} 31 | file_uris={{ hdfs_files | to_python }}, 32 | {% endif %} 33 | {% if hdfs_archives %} 34 | archive_uris={{ hdfs_archives | to_python }}, 35 | {% endif %} 36 | main_class=CONFIG['hadoop_main_class'], 37 | ), 38 | ), 39 | gcp_conn_id=CONFIG['gcp_conn_id'], 40 | region=CONFIG['gcp_region'], 41 | params={{ props_macro.props(action_node_properties=action_node_properties) }}, 42 | ) 43 | -------------------------------------------------------------------------------- /o2a/templates/pig.tpl: -------------------------------------------------------------------------------- 1 | {# 2 | Copyright 2019 Google LLC 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | #} 16 | {% import "macros/props.tpl" as props_macro %} 17 | {{ task_id | to_var }} = DataprocSubmitJobOperator( 18 | task_id={{ task_id | to_python }}, 19 | trigger_rule={{ trigger_rule | to_python }}, 20 | job=dict( 21 | placement=dict( 22 | cluster_name=CONFIG['dataproc_cluster'], 23 | ), 24 | pig_job=dict( 25 | script_variables={{ params_dict | to_python }}, 26 | query_file_uri='%s/%s' % (CONFIG['gcp_uri_prefix'], {{ script_file_name | to_python }}), 27 | ), 28 | ), 29 | gcp_conn_id=CONFIG['gcp_conn_id'], 30 | region=CONFIG['gcp_region'], 31 | params={{ props_macro.props(action_node_properties=action_node_properties) }}, 32 | ) 33 | -------------------------------------------------------------------------------- /o2a/templates/pig_command.tpl: -------------------------------------------------------------------------------- 1 | {# 2 | Copyright 2019 Google LLC 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | #} 16 | "gcloud dataproc jobs submit pig " 17 | "--cluster=%s " 18 | "--region=%s " 19 | "--execute %s" 20 | -------------------------------------------------------------------------------- /o2a/templates/prepare.tpl: -------------------------------------------------------------------------------- 1 | {# 2 | Copyright 2019 Google LLC 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | #} 16 | 17 | {% import "macros/props.tpl" as props_macro %} 18 | {{ task_id | to_var }} = bash.BashOperator( 19 | task_id={{ task_id | to_python }}, 20 | trigger_rule={{ trigger_rule | to_python }}, 21 | bash_command={% include "prepare_command.tpl" %}, 22 | params={{ props_macro.props(action_node_properties=action_node_properties) }}, 23 | ) 24 | -------------------------------------------------------------------------------- /o2a/templates/prepare_command.tpl: -------------------------------------------------------------------------------- 1 | {# 2 | Copyright 2019 Google LLC 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | #} 16 | "$DAGS_FOLDER/../data/prepare.sh " 17 | "-c %s -r %s " 18 | {% if delete is not none %}'-d %s '{% endif %} 19 | {% if mkdir is not none %}'-m %s '{% endif %} 20 | % (CONFIG['dataproc_cluster'], CONFIG['gcp_region'], 21 | {% if delete is not none %}shlex.quote({{ delete | to_python }}),{% endif %} 22 | {% if mkdir is not none %}shlex.quote({{ mkdir | to_python }}),{% endif %} 23 | ) 24 | -------------------------------------------------------------------------------- /o2a/templates/props.tpl: -------------------------------------------------------------------------------- 1 | {# 2 | Copyright 2019 Google LLC 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | #} 16 | {% if (action_node_properties is defined) and (action_node_properties | length != 0) -%} 17 | PropertySet( 18 | config=CONFIG, 19 | job_properties=JOB_PROPS, 20 | action_node_properties={{ action_node_properties | to_python }}).merged 21 | {% else -%} 22 | PropertySet( 23 | config=CONFIG, 24 | job_properties=JOB_PROPS).merged 25 | {% endif %} 26 | -------------------------------------------------------------------------------- /o2a/templates/shell.tpl: -------------------------------------------------------------------------------- 1 | {# 2 | Copyright 2019 Google LLC 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | #} 16 | {% import "macros/props.tpl" as props_macro %} 17 | {{ task_id | to_var }} = bash.BashOperator( 18 | task_id={{ task_id | to_python }}, 19 | trigger_rule={{ trigger_rule | to_python }}, 20 | bash_command={% include "pig_command.tpl" %} % (CONFIG['dataproc_cluster'], CONFIG['gcp_region'], 21 | shlex.quote({{ pig_command | to_python }})), 22 | params={{ props_macro.props(action_node_properties=action_node_properties) }}, 23 | ) 24 | -------------------------------------------------------------------------------- /o2a/templates/spark.tpl: -------------------------------------------------------------------------------- 1 | {# 2 | Copyright 2019 Google LLC 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | #} 16 | {% import "macros/props.tpl" as props_macro %} 17 | {{ task_id | to_var }} = DataprocSubmitJobOperator( 18 | task_id={{ task_id | to_python }}, 19 | trigger_rule={{ trigger_rule | to_python }}, 20 | job=dict( 21 | placement=dict( 22 | cluster_name=CONFIG['dataproc_cluster'], 23 | ), 24 | spark_job={{ spark_job | to_python }}, 25 | ), 26 | gcp_conn_id=CONFIG['gcp_conn_id'], 27 | region=CONFIG['gcp_region'], 28 | params={{ props_macro.props(action_node_properties=action_node_properties) }}, 29 | ) -------------------------------------------------------------------------------- /o2a/templates/ssh.tpl: -------------------------------------------------------------------------------- 1 | {# 2 | Copyright 2019 Google LLC 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | #} 16 | {{ task_id | to_var }}_hook = SSHHook( 17 | ssh_conn_id='ssh_default', 18 | username={{ user | to_python }}, 19 | remote_host={{ host | to_python }}, 20 | ) 21 | 22 | {% import "macros/props.tpl" as props_macro %} 23 | {{ task_id | to_var }} = SSHOperator( 24 | task_id={{ task_id | to_python }}, 25 | trigger_rule={{ trigger_rule | to_python }}, 26 | ssh_hook={{ task_id | to_var }}_hook, 27 | command={{ command | to_python }}, 28 | params={{ props_macro.props(action_node_properties=action_node_properties) }}, 29 | ) 30 | -------------------------------------------------------------------------------- /o2a/templates/subwf.tpl: -------------------------------------------------------------------------------- 1 | {# 2 | Copyright 2019 Google LLC 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | #} 16 | 17 | {{ task_id | to_var }} = SubDagOperator( 18 | task_id={{ task_id | to_python }}, 19 | trigger_rule={{ trigger_rule | to_python }}, 20 | subdag=subdag_{{ app_name | to_var }}.sub_dag(dag.dag_id, {{ task_id | to_python }}, dag.start_date, dag.schedule_interval, dag.user_defined_macros), 21 | ) 22 | -------------------------------------------------------------------------------- /o2a/templates/subworkflow.tpl: -------------------------------------------------------------------------------- 1 | {# 2 | Copyright 2019 Google LLC 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | #} 16 | {% for dependency in dependencies %} 17 | {{ dependency }} 18 | {% endfor %} 19 | 20 | CONFIG={{ config | to_python }} 21 | 22 | JOB_PROPS={{ job_properties | to_python }} 23 | 24 | def sub_dag(parent_dag_name, child_dag_name, start_date, schedule_interval, user_defined_macros): 25 | with models.DAG( 26 | '{0}.{1}'.format(parent_dag_name, child_dag_name), 27 | schedule_interval=schedule_interval, # Change to suit your needs 28 | start_date=start_date, # Change to suit your needs 29 | user_defined_macros=user_defined_macros # Change to suit your needs 30 | ) as dag: 31 | 32 | {% filter indent(8, True) %} 33 | {% include "dag_body.tpl" %} 34 | {% endfilter %} 35 | 36 | return dag 37 | -------------------------------------------------------------------------------- /o2a/templates/workflow.tpl: -------------------------------------------------------------------------------- 1 | {# 2 | Copyright 2019 Google LLC 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | #} 16 | 17 | {% for dependency in dependencies %} 18 | {{ dependency }} 19 | {% endfor %} 20 | 21 | 22 | CONFIG={{ config | to_python }} 23 | 24 | JOB_PROPS={{ job_properties | to_python }} 25 | 26 | TASK_MAP={{ task_map | to_python }} 27 | 28 | TEMPLATE_ENV = {**CONFIG, **JOB_PROPS, "functions": functions, "task_map": TASK_MAP } 29 | 30 | with models.DAG( 31 | {{ dag_name | to_python }}, 32 | schedule_interval={% if schedule_interval %}datetime.timedelta(days={{ schedule_interval }}){% else %}None{% endif %}, # Change to suit your needs 33 | start_date=dates.days_ago({{ start_days_ago }}), # Change to suit your needs 34 | user_defined_macros=TEMPLATE_ENV 35 | ) as dag: 36 | 37 | {% filter indent(4, True) %} 38 | {% include "dag_body.tpl" %} 39 | {% endfilter %} 40 | -------------------------------------------------------------------------------- /o2a/templates/xml_escaped_props.tpl: -------------------------------------------------------------------------------- 1 | {# 2 | Copyright 2019 Google LLC 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | #} 16 | {% if (action_node_properties is defined) and (action_node_properties | length != 0) -%} 17 | PropertySet( 18 | config=CONFIG, 19 | job_properties=JOB_PROPS, 20 | action_node_properties={{ action_node_properties | to_python }}).xml_escaped.merged 21 | {% else -%} 22 | PropertySet( 23 | config=CONFIG, 24 | job_properties=JOB_PROPS).xml_escaped.merged 25 | {% endif %} 26 | -------------------------------------------------------------------------------- /o2a/transformers/__init__.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Copyright 2019 Google LLC 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | -------------------------------------------------------------------------------- /o2a/transformers/base_transformer.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Copyright 2019 Google LLC 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | """Base transformer classes""" 16 | 17 | from o2a.converter.workflow import Workflow 18 | 19 | 20 | from o2a.o2a_libs.src.o2a_lib.property_utils import PropertySet 21 | 22 | 23 | class BaseWorkflowTransformer: 24 | """ 25 | Base class for all transformers 26 | """ 27 | 28 | def process_workflow_after_parse_workflow_xml(self, workflow: Workflow): 29 | pass 30 | 31 | def process_workflow_after_convert_nodes(self, workflow: Workflow, props: PropertySet): 32 | pass 33 | -------------------------------------------------------------------------------- /o2a/transformers/remove_end_transformer.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Copyright 2019 Google LLC 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | """ 16 | Remove End Transformer 17 | """ 18 | from o2a.converter.workflow import Workflow 19 | from o2a.mappers.decision_mapper import DecisionMapper 20 | from o2a.mappers.end_mapper import EndMapper 21 | from o2a.transformers.base_transformer import BaseWorkflowTransformer 22 | 23 | 24 | # pylint: disable=too-few-public-methods 25 | class RemoveEndTransformer(BaseWorkflowTransformer): 26 | """ 27 | Remove End nodes with all relations when it's not connected to Decision Node. 28 | """ 29 | 30 | def process_workflow_after_parse_workflow_xml(self, workflow: Workflow): 31 | decision_nodes = workflow.get_nodes_by_type(DecisionMapper) 32 | decision_node_names = {node.name for node in decision_nodes} 33 | end_nodes = workflow.get_nodes_by_type(EndMapper) 34 | 35 | for end_node in end_nodes: 36 | upstream_nodes = workflow.find_upstream_nodes(end_node) 37 | upstream_node_names = {node.name for node in upstream_nodes} 38 | 39 | if not decision_node_names.intersection(upstream_node_names): 40 | workflow.remove_node(end_node) 41 | else: 42 | for upstream_node in upstream_nodes: 43 | if upstream_node.name not in decision_node_names: 44 | upstream_node.downstream_names.remove(end_node.name) 45 | -------------------------------------------------------------------------------- /o2a/transformers/remove_fork_transformer.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Copyright 2019 Google LLC 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | """ 16 | Remove Fork Transformer 17 | """ 18 | 19 | from o2a.converter.workflow import Workflow 20 | from o2a.mappers.fork_mapper import ForkMapper 21 | from o2a.transformers.base_transformer import BaseWorkflowTransformer 22 | 23 | 24 | # pylint: disable=too-few-public-methods 25 | class RemoveForkTransformer(BaseWorkflowTransformer): 26 | """ 27 | Remove fork nodes when there are no upstream nodes 28 | """ 29 | 30 | def process_workflow_after_parse_workflow_xml(self, workflow: Workflow): 31 | fork_nodes = workflow.get_nodes_by_type(ForkMapper) 32 | for fork_node in fork_nodes: 33 | upstream_nodes = workflow.find_upstream_nodes(fork_node) 34 | if not upstream_nodes: 35 | workflow.remove_node(fork_node) 36 | -------------------------------------------------------------------------------- /o2a/transformers/remove_inaccessible_node_transformer.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Copyright 2019 Google LLC 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | """Remove inaccessible transformer""" 16 | 17 | from typing import Dict 18 | 19 | from o2a.converter.oozie_node import OozieNode 20 | from o2a.converter.workflow import Workflow 21 | from o2a.mappers.start_mapper import StartMapper 22 | from o2a.transformers.base_transformer import BaseWorkflowTransformer 23 | 24 | 25 | # pylint: disable=too-few-public-methods 26 | class RemoveInaccessibleNodeTransformer(BaseWorkflowTransformer): 27 | """ 28 | Transformer that remove nodes that are not accessible from Start node. In the case of Airflow, 29 | all nodes are executed. In the case of Oozie, only nodes that have a connection 30 | to Start node are executed. 31 | """ 32 | 33 | def process_workflow_after_parse_workflow_xml(self, workflow: Workflow): 34 | accessible_nodes = self._find_accessible_nodes(workflow) 35 | 36 | for node in workflow.nodes.copy().values(): 37 | if node not in accessible_nodes: 38 | workflow.remove_node(node) 39 | 40 | @staticmethod 41 | def _find_accessible_nodes(workflow: Workflow): 42 | """ 43 | Finds nodes that are reachable from any Start node. 44 | """ 45 | start_nodes = workflow.get_nodes_by_type(StartMapper) 46 | visited_node: Dict[str, OozieNode] = dict() 47 | 48 | def visit_node(node: OozieNode): 49 | if node.name in visited_node: 50 | return 51 | visited_node[node.name] = node 52 | 53 | all_downstream_node_names = [*node.downstream_names] 54 | if node.error_downstream_name: 55 | all_downstream_node_names.append(node.error_downstream_name) 56 | 57 | for node_name in all_downstream_node_names: 58 | visit_node(workflow.nodes[node_name]) 59 | 60 | for start_node in start_nodes: 61 | visit_node(start_node) 62 | 63 | return visited_node.values() 64 | -------------------------------------------------------------------------------- /o2a/transformers/remove_join_transformer.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Copyright 2019 Google LLC 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | """ 16 | Remove Join Transformer 17 | """ 18 | 19 | from o2a.converter.workflow import Workflow 20 | from o2a.mappers.join_mapper import JoinMapper 21 | from o2a.transformers.base_transformer import BaseWorkflowTransformer 22 | 23 | 24 | # pylint: disable=too-few-public-methods 25 | class RemoveJoinTransformer(BaseWorkflowTransformer): 26 | """ 27 | Remove join nodes when there are no downstream nodes 28 | """ 29 | 30 | def process_workflow_after_parse_workflow_xml(self, workflow: Workflow): 31 | join_nodes = workflow.get_nodes_by_type(JoinMapper) 32 | for join_node in join_nodes: 33 | if not join_node.downstream_names: 34 | workflow.remove_node(join_node) 35 | -------------------------------------------------------------------------------- /o2a/transformers/remove_kill_transformer.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Copyright 2019 Google LLC 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | """ 16 | Remove Kill Transformer 17 | """ 18 | from o2a.converter.workflow import Workflow 19 | from o2a.mappers.kill_mapper import KillMapper 20 | from o2a.transformers.base_transformer import BaseWorkflowTransformer 21 | 22 | 23 | # pylint: disable=too-few-public-methods 24 | class RemoveKillTransformer(BaseWorkflowTransformer): 25 | """ 26 | Remove Kill nodes with all relations when it's used in error flow. 27 | """ 28 | 29 | def process_workflow_after_parse_workflow_xml(self, workflow: Workflow): 30 | kill_nodes = workflow.get_nodes_by_type(KillMapper) 31 | for kill_node in kill_nodes: 32 | upstream_nodes = workflow.find_upstream_nodes(kill_node) 33 | if not any(kill_node.name in node.downstream_names for node in upstream_nodes): 34 | workflow.remove_node(kill_node) 35 | -------------------------------------------------------------------------------- /o2a/transformers/remove_start_transformer.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Copyright 2019 Google LLC 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | """ 16 | Remove Start Transformer 17 | """ 18 | 19 | from o2a.converter.workflow import Workflow 20 | from o2a.mappers.start_mapper import StartMapper 21 | from o2a.transformers.base_transformer import BaseWorkflowTransformer 22 | 23 | 24 | # pylint: disable=too-few-public-methods 25 | class RemoveStartTransformer(BaseWorkflowTransformer): 26 | """ 27 | Remove Start nodes with all relations. 28 | """ 29 | 30 | def process_workflow_after_parse_workflow_xml(self, workflow: Workflow): 31 | start_nodes = workflow.get_nodes_by_type(StartMapper) 32 | for start_node in start_nodes: 33 | workflow.remove_node(start_node) 34 | -------------------------------------------------------------------------------- /o2a/utils/__init__.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Copyright 2019 Google LLC 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | """Various utilities used by converter""" 16 | __all__ = ["el_utils", "xml_utils"] 17 | -------------------------------------------------------------------------------- /o2a/utils/constants.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Copyright 2019 Google LLC 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | """ List of project-wide constants """ 16 | 17 | CONFIG = "configuration.properties" 18 | JOB_PROPS = "job.properties" 19 | WORKFLOW_XML = "workflow.xml" 20 | -------------------------------------------------------------------------------- /o2a/utils/file_utils.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Copyright 2019 Google LLC 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | """Utilities to deal with files in workflow folders""" 17 | import os 18 | from typing import List 19 | 20 | 21 | def get_lib_files(library_folder_path: str, extension: str) -> List[str]: 22 | """Returns list of all lib files from the library folder matching the extension provided. 23 | For example if you specify the '.jar' extension it will return names (not paths) of all 24 | the *.jar files. 25 | 26 | It returns an empty list in case there are no files matching the extension and raises Exception 27 | in case the lib folder does not exist or is not a directory. 28 | """ 29 | if os.path.exists(library_folder_path): 30 | if os.path.isdir(library_folder_path): 31 | return [file for file in os.listdir(library_folder_path) if file.endswith(extension)] 32 | raise Exception(f"The {library_folder_path} exists but it is not a directory!") 33 | return [] 34 | -------------------------------------------------------------------------------- /o2a/utils/param_extractor.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Copyright 2019 Google LLC 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | """Extract params from oozie's action node""" 16 | from _elementtree import Element 17 | 18 | from o2a.o2a_libs.src.o2a_lib import el_parser 19 | from o2a.utils import xml_utils 20 | 21 | TAG_PARAM = "param" 22 | 23 | 24 | def extract_param_values_from_action_node(oozie_node: Element): 25 | param_nodes = xml_utils.find_nodes_by_tag(oozie_node, TAG_PARAM) 26 | 27 | new_params = {} 28 | for node in param_nodes: 29 | if not node.text: 30 | continue 31 | param = el_parser.translate(node.text) 32 | key, _, value = param.partition("=") 33 | new_params[key] = value 34 | return new_params 35 | -------------------------------------------------------------------------------- /o2a/utils/relation_utils.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Copyright 2019 Google LLC 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | """Relation utilities""" 16 | 17 | from typing import List, Sequence 18 | 19 | from o2a.converter.task import Task 20 | from o2a.converter.relation import Relation 21 | 22 | 23 | def chain(ops: Sequence[Task]) -> List[Relation]: 24 | """ 25 | Given a number of tasks, builds a relation chain. 26 | 27 | ** Example: ** 28 | 29 | :codeblock: pycon 30 | >>> tasks = [ 31 | ... Task(task_id="task_a", template_name="task_a.tpl"), 32 | ... Task(task_id="task_b", template_name="task_b.tpl"), 33 | ... Task(task_id="task_c", template_name="task_c.tpl") 34 | ... ] 35 | >>> chain(tasks) 36 | [Relation(from_task_id='task_a', to_task_id='task_b'), Relation(from_task_id='task_b', 37 | to_task_id='task_c')] 38 | 39 | :param ops: sequence of tasks 40 | :return: list of relations 41 | """ 42 | return [Relation(from_task_id=a.task_id, to_task_id=b.task_id) for a, b in zip(ops, ops[1::])] 43 | -------------------------------------------------------------------------------- /o2a/utils/template_utils.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Copyright 2019 Google LLC 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | """Template utilities""" 16 | from typing import Dict, Any 17 | 18 | import jinja2 19 | 20 | from o2a.definitions import TPL_PATH 21 | from o2a.utils import python_serializer 22 | 23 | from o2a.utils.variable_name_utils import convert_to_python_variable 24 | 25 | TEMPLATE_LOADER = jinja2.FileSystemLoader(searchpath=TPL_PATH) 26 | TEMPLATE_ENV = jinja2.Environment( 27 | loader=TEMPLATE_LOADER, undefined=jinja2.StrictUndefined, trim_blocks=True, lstrip_blocks=True 28 | ) 29 | TEMPLATE_CACHES: Dict[str, Any] = {} 30 | 31 | TEMPLATE_ENV.filters["to_var"] = convert_to_python_variable 32 | 33 | TEMPLATE_ENV.filters["to_python"] = python_serializer.serialize 34 | 35 | 36 | def render_template(template_name: str, *args, **kwargs) -> str: 37 | """Render Jinja template""" 38 | if template_name not in TEMPLATE_CACHES: 39 | template = TEMPLATE_ENV.get_template(template_name) 40 | TEMPLATE_CACHES[template_name] = template 41 | content: str = TEMPLATE_CACHES[template_name].render(*args, **kwargs) 42 | return content 43 | -------------------------------------------------------------------------------- /o2a/utils/variable_name_utils.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Copyright 2019 Google LLC 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | """Variable name utilities""" 16 | import re 17 | 18 | 19 | def convert_to_python_variable(name: str) -> str: 20 | """ 21 | Creates an identifier that is a valid variable name in Python language. 22 | """ 23 | # Replace all hyphens with underscores 24 | name = name.replace("-", "_") 25 | # Replace invalid characters to underscore 26 | name = re.sub("[^0-9a-zA-Z_]", "_", name) 27 | # Remove leading characters until we find a letter or underscore 28 | name = re.sub("^[^a-zA-Z_]+", "", name) 29 | return name 30 | -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [tool.black] 2 | line-length = 110 3 | target_version = ['py36'] 4 | include = '\.pyi?$' 5 | exclude = ''' 6 | 7 | ( 8 | /( 9 | \.eggs # exclude a few common directories in the 10 | | \.git # root of the project 11 | | \.hg 12 | | \.mypy_cache 13 | | \.tox 14 | | \.venv 15 | | _build 16 | | buck-out 17 | | build 18 | | dist 19 | )/ 20 | ) 21 | ''' 22 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | apache-airflow>=2.0.0 # pyup: ignore 2 | autoflake==1.4 3 | black==23.1a1 4 | flake8==3.8.4 5 | google-api-python-client==1.12.11 6 | isort==5.7.0 7 | j2cli==0.3.10 8 | Jinja2==3.1.2 # pyup: ignore 9 | lark-parser==0.11.1 10 | mypy==1.0.0 11 | parameterized==0.7.5 12 | paramiko==3.1.0 13 | pre-commit==3.3.1 14 | pydeps==1.9.13 15 | pylint==2.6.0 16 | pytest==7.3.1 17 | pytest-cov==4.0.0 18 | safety==1.10.3 19 | sshtunnel==0.4.0 20 | twine==4.0.2 21 | tzlocal==5.0 # pyup: ignore 22 | Werkzeug==2.2.3 # pyup: ignore 23 | yamllint==1.31.0 24 | -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [aliases] 2 | test=pytest 3 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Copyright 2019 Google LLC 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | """Setup script for Oozie to Airflow migration tool""" 16 | from setuptools import setup 17 | 18 | from o2a import NAME 19 | 20 | with open("README.md") as fh: 21 | LONG_DESCRIPTION = fh.read() 22 | 23 | with open("requirements.txt") as f: 24 | REQUIREMENTS = f.read().splitlines() 25 | 26 | setup( 27 | name=NAME, 28 | version="2.0.1", 29 | author="Jarek Potiuk, Szymon Przedwojski, Kamil Breguła, Feng Lu, Cameron Moberg", 30 | author_email="jarek.potiuk@polidea.com, szymon.przedwojski@polidea.com, " 31 | "kamil.bregula@polidea.com, fenglu@google.com, cjmoberg@google.com", 32 | description="Oozie To Airflow migration tool", 33 | long_description=LONG_DESCRIPTION, 34 | long_description_content_type="text/markdown", 35 | url="https://github.com/GoogleCloudPlatform/oozie-to-airflow", 36 | include_package_data=True, 37 | setup_requires=["pytest-runner"], 38 | install_requires=REQUIREMENTS, 39 | tests_require=["pytest"], 40 | scripts=["bin/o2a", "bin/o2a-validate-workflows"], 41 | packages=["o2a"], 42 | classifiers=[ 43 | "Programming Language :: Python :: 3.8", 44 | "Programming Language :: Python :: 3.9", 45 | "Programming Language :: Python :: 3.10", 46 | "License :: OSI Approved :: Apache Software License", 47 | "Operating System :: OS Independent", 48 | ], 49 | ) 50 | -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Copyright 2019 Google LLC 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | -------------------------------------------------------------------------------- /tests/converter/__init__.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Copyright 2019 Google LLC 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | -------------------------------------------------------------------------------- /tests/mappers/__init__.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Copyright 2019 Google LLC 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | -------------------------------------------------------------------------------- /tests/mappers/extensions/__init__.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Copyright 2019 Google LLC 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | -------------------------------------------------------------------------------- /tests/mappers/test_base_mapper.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Copyright 2019 Google LLC 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | # Copyright 2018 Google LLC 17 | # 18 | # Licensed under the Apache License, Version 2.0 (the "License"); 19 | # you may not use this file except in compliance with the License. 20 | # You may obtain a copy of the License at 21 | # 22 | # http://www.apache.org/licenses/LICENSE-2.0 23 | # 24 | # Unless required by applicable law or agreed to in writing, software 25 | # distributed under the License is distributed on an "AS IS" BASIS, 26 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 27 | # See the License for the specific language governing permissions and 28 | # limitations under the License. 29 | """Tests for the base mapper.""" 30 | 31 | import unittest 32 | 33 | from xml.etree import ElementTree as ET 34 | 35 | 36 | from o2a.mappers import base_mapper 37 | from o2a.o2a_libs.src.o2a_lib.property_utils import PropertySet 38 | 39 | 40 | class TestBaseMapper(unittest.TestCase): 41 | def setUp(self): 42 | # language=XML 43 | node_str = """ 44 | 45 | 46 | ${firstNotNull('', '')} 47 | True 48 | 49 | 50 | 51 | """ 52 | self.node = ET.fromstring(node_str) 53 | self.mapper = base_mapper.BaseMapper( 54 | oozie_node=self.node, 55 | name="test_id", 56 | dag_name="DAG_NAME_B", 57 | props=PropertySet(job_properties={}, config={}), 58 | ) 59 | -------------------------------------------------------------------------------- /tests/mappers/test_dummy_mapper.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Copyright 2019 Google LLC 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | """Tests for the Dummy mapper.""" 16 | import ast 17 | import unittest 18 | from xml.etree.ElementTree import Element 19 | 20 | from o2a.converter.task import Task 21 | from o2a.mappers import dummy_mapper 22 | 23 | 24 | class TestDummyMapper(unittest.TestCase): 25 | oozie_node = Element("dummy") 26 | 27 | def test_create_mapper(self): 28 | mapper = self._get_dummy_mapper() 29 | # make sure everything is getting initialized correctly 30 | self.assertEqual("test_id", mapper.name) 31 | 32 | def test_convert_tasks_and_relations(self): 33 | mapper = self._get_dummy_mapper() 34 | 35 | tasks, relations = mapper.to_tasks_and_relations() 36 | 37 | self.assertEqual(tasks, [Task(task_id="test_id", template_name="dummy.tpl")]) 38 | self.assertEqual(relations, []) 39 | 40 | def test_required_imports(self): 41 | mapper = self._get_dummy_mapper() 42 | imps = mapper.required_imports() 43 | imp_str = "\n".join(imps) 44 | ast.parse(imp_str) 45 | 46 | def _get_dummy_mapper(self): 47 | return dummy_mapper.DummyMapper(oozie_node=self.oozie_node, name="test_id", dag_name="DAG_NAME_B") 48 | -------------------------------------------------------------------------------- /tests/o2a_libs/__init__.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Copyright 2019 Google LLC 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | -------------------------------------------------------------------------------- /tests/o2a_libs/test_property_utils.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Copyright 2019 Google LLC 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | """Tests for property utils""" 16 | 17 | from unittest import TestCase 18 | 19 | from o2a.o2a_libs.src.o2a_lib.property_utils import PropertySet 20 | 21 | 22 | class TestPropertySet(TestCase): 23 | def test_xml_escaping(self): 24 | # Given 25 | pset = PropertySet( 26 | job_properties={"url": "http://example.com:8080/workflow?job-id=$jobId&status=$status"}, 27 | action_node_properties={"my.injection.attempt": "1"}, 28 | ) 29 | # When 30 | pset_escaped = pset.xml_escaped.merged 31 | 32 | # Then 33 | self.assertEqual( 34 | "http://example.com:8080/workflow?job-id=$jobId&status=$status", pset_escaped["url"] 35 | ) 36 | self.assertEqual("<value>1</value>", pset_escaped["my.injection.attempt"]) 37 | -------------------------------------------------------------------------------- /tests/script_tests/__init.__.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Copyright 2019 Google LLC 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | -------------------------------------------------------------------------------- /tests/script_tests/mock: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # Saves all calls along with script name and arguments to the log file specified by the 4 | # environment variable "COMMAND_EXECUTION_LOG". 5 | # For further information, see the file: test-git.py 6 | 7 | if [[ -z "$COMMAND_EXECUTION_LOG" ]]; then 8 | echo "You should set \$COMMAND_EXECUTION_LOG variable to enable logging." 9 | exit 1 10 | fi 11 | 12 | function shell_quote_array() { 13 | printf "%q " "$@" 14 | } 15 | 16 | ESCAPED_ARGS="$(shell_quote_array "${@}")" 17 | 18 | echo "$(basename "$0")" "${ESCAPED_ARGS}" >> "$COMMAND_EXECUTION_LOG" 19 | -------------------------------------------------------------------------------- /tests/transformers/__init__.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Copyright 2019 Google LLC 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | -------------------------------------------------------------------------------- /tests/transformers/test_remove_start_transformer.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Copyright 2019 Google LLC 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | """ 16 | Remove Start Transformer tests 17 | """ 18 | import unittest 19 | from unittest import mock 20 | 21 | from o2a.converter.oozie_node import OozieNode 22 | from o2a.converter.task import Task 23 | from o2a.converter.workflow import Workflow 24 | from o2a.mappers.base_mapper import BaseMapper 25 | from o2a.mappers.start_mapper import StartMapper 26 | from o2a.transformers.remove_start_transformer import RemoveStartTransformer 27 | 28 | 29 | class RemoveEndTransformerTest(unittest.TestCase): 30 | def test_should_remove_start_node(self): 31 | transformer = RemoveStartTransformer() 32 | 33 | workflow = Workflow(input_directory_path="", output_directory_path="", dag_name="DAG_NAME_B") 34 | 35 | other_mapper = mock.Mock(spec=BaseMapper) 36 | other_mapper.name = "first_task" 37 | start_mapper = mock.Mock(spec=StartMapper) 38 | start_mapper.name = "start_task" 39 | 40 | workflow.nodes[other_mapper.name] = OozieNode( 41 | mapper=other_mapper, tasks=[self._get_dummy_task(other_mapper.name)] 42 | ) 43 | workflow.nodes[start_mapper.name] = OozieNode( 44 | mapper=start_mapper, tasks=[self._get_dummy_task(start_mapper.name)] 45 | ) 46 | 47 | transformer.process_workflow_after_parse_workflow_xml(workflow) 48 | 49 | self.assertEqual({other_mapper.name}, set(workflow.nodes.keys())) 50 | 51 | @staticmethod 52 | def _get_dummy_task(task_id): 53 | return Task(task_id=task_id, template_name="dummy.tpl") 54 | -------------------------------------------------------------------------------- /tests/utils/__init__.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Copyright 2019 Google LLC 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | -------------------------------------------------------------------------------- /tests/utils/test_relation_utils.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Copyright 2019 Google LLC 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | """Tests Relation utils""" 16 | import unittest 17 | 18 | from o2a.converter.task import Task 19 | from o2a.converter.relation import Relation 20 | from o2a.mappers import fs_mapper 21 | 22 | 23 | # pylint: disable=invalid-name 24 | class chainTestCase(unittest.TestCase): 25 | def test_empty(self): 26 | relations = fs_mapper.chain([]) 27 | self.assertEqual([], relations) 28 | 29 | def test_one(self): 30 | relations = fs_mapper.chain([Task(task_id="A", template_name="")]) 31 | self.assertEqual([], relations) 32 | 33 | def test_multiple(self): 34 | relations = fs_mapper.chain( 35 | [ 36 | Task(task_id="task_1", template_name=""), 37 | Task(task_id="task_2", template_name=""), 38 | Task(task_id="task_3", template_name=""), 39 | Task(task_id="task_4", template_name=""), 40 | ] 41 | ) 42 | self.assertEqual( 43 | [ 44 | Relation(from_task_id="task_1", to_task_id="task_2"), 45 | Relation(from_task_id="task_2", to_task_id="task_3"), 46 | Relation(from_task_id="task_3", to_task_id="task_4"), 47 | ], 48 | relations, 49 | ) 50 | -------------------------------------------------------------------------------- /tests/utils/test_variable_name_utils.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Copyright 2019 Google LLC 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | """Tests Variable Name""" 16 | from unittest import TestCase 17 | 18 | from parameterized import parameterized 19 | 20 | from o2a.utils.variable_name_utils import convert_to_python_variable 21 | 22 | 23 | class ConvertToPythonVariableTestCase(TestCase): 24 | def test_should_keep_original_when_its_not_required(self): 25 | self.assertEqual(convert_to_python_variable("TEST"), "TEST") 26 | 27 | @parameterized.expand( 28 | [ 29 | ("TEST$TEST", "TEST_TEST"), 30 | ("TEST TEST", "TEST_TEST"), 31 | ('TEST"TEST', "TEST_TEST"), 32 | ("TEST'TEST", "TEST_TEST"), 33 | ("TEST'", "TEST_"), 34 | ] 35 | ) 36 | def test_should_replace_invalid_characters_to_underscore(self, input_text, expected_output): 37 | self.assertEqual(convert_to_python_variable(input_text), expected_output) 38 | 39 | def test_should_remove_leading_invalid_characters(self): 40 | self.assertEqual(convert_to_python_variable("123123TEST'TEST"), "TEST_TEST") 41 | -------------------------------------------------------------------------------- /utils/LICENSE.txt: -------------------------------------------------------------------------------- 1 | Copyright 2019 Google LLC 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | -------------------------------------------------------------------------------- /yamllint-config.yml: -------------------------------------------------------------------------------- 1 | # Copyright 2019 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | --- 15 | extends: default 16 | 17 | rules: 18 | line-length: 19 | max: 110 20 | --------------------------------------------------------------------------------