├── .asf.yaml ├── .dlc.json ├── .github ├── PULL_REQUEST_TEMPLATE.md ├── codecov.yml ├── mergeable.yml └── workflows │ └── ci.yaml ├── .gitignore ├── .licenserc.yaml ├── .pre-commit-config.yaml ├── .ruff.toml ├── CONTRIBUTING.md ├── LICENSE ├── MANIFEST.in ├── NOTICE ├── README.md ├── RELEASE.md ├── UPDATING.md ├── docs ├── Makefile ├── make.bat └── source │ ├── _static │ └── .gitkeep │ ├── _templates │ ├── versioning.html │ └── versions.html │ ├── api.rst │ ├── changelog.rst │ ├── cli.rst │ ├── concept.rst │ ├── conf.py │ ├── config.rst │ ├── howto │ ├── index.rst │ ├── multi-resources.rst │ └── remote-submit.rst │ ├── index.rst │ ├── resources_plugin │ ├── develop.rst │ ├── github.rst │ ├── gitlab.rst │ ├── index.rst │ ├── local.rst │ ├── oss.rst │ ├── resource-plugin.rst │ └── s3.rst │ ├── start.rst │ ├── tasks │ ├── condition.rst │ ├── datax.rst │ ├── dependent.rst │ ├── dvc.rst │ ├── flink.rst │ ├── func_wrap.rst │ ├── http.rst │ ├── index.rst │ ├── kubernetes.rst │ ├── map_reduce.rst │ ├── mlflow.rst │ ├── openmldb.rst │ ├── procedure.rst │ ├── python.rst │ ├── pytorch.rst │ ├── sagemaker.rst │ ├── shell.rst │ ├── spark.rst │ ├── sql.rst │ ├── sub_workflow.rst │ └── switch.rst │ └── tutorial.rst ├── examples └── yaml_define │ ├── Condition.yaml │ ├── DataX.yaml │ ├── Dependent.yaml │ ├── Dependent_External.yaml │ ├── Dvc.yaml │ ├── Flink.yaml │ ├── Http.yaml │ ├── Kubernetes.yaml │ ├── MapReduce.yaml │ ├── MoreConfiguration.yaml │ ├── OpenMLDB.yaml │ ├── Procedure.yaml │ ├── Python.yaml │ ├── Pytorch.yaml │ ├── Sagemaker.yaml │ ├── Shell.yaml │ ├── Spark.yaml │ ├── Sql.yaml │ ├── SubWorkflow.yaml │ ├── Switch.yaml │ ├── example_datax.json │ ├── example_sagemaker_params.json │ ├── example_sql.sql │ ├── example_sub_workflow.yaml │ ├── mlflow.yaml │ └── tutorial.yaml ├── setup.cfg ├── setup.py ├── src └── pydolphinscheduler │ ├── __init__.py │ ├── cli │ ├── __init__.py │ └── commands.py │ ├── configuration.py │ ├── constants.py │ ├── core │ ├── __init__.py │ ├── engine.py │ ├── mixin.py │ ├── parameter.py │ ├── process_definition.py │ ├── resource.py │ ├── resource_plugin.py │ ├── task.py │ ├── workflow.py │ └── yaml_workflow.py │ ├── default_config.yaml │ ├── examples │ ├── __init__.py │ ├── bulk_create_example.py │ ├── ext │ │ └── example.sql │ ├── local_parameter_example.py │ ├── multi_resources_example.py │ ├── task_condition_example.py │ ├── task_datax_example.py │ ├── task_dependent_example.py │ ├── task_dvc_example.py │ ├── task_flink_example.py │ ├── task_kubernetes_example.py │ ├── task_map_reduce_example.py │ ├── task_mlflow_example.py │ ├── task_openmldb_example.py │ ├── task_python_example.py │ ├── task_pytorch_example.py │ ├── task_sagemaker_example.py │ ├── task_spark_example.py │ ├── task_sql_example.py │ ├── task_sub_workflow_example.py │ ├── task_switch_example.py │ ├── tutorial.py │ ├── tutorial_decorator.py │ └── tutorial_resource_plugin.py │ ├── exceptions.py │ ├── java_gateway.py │ ├── models │ ├── __init__.py │ ├── base.py │ ├── base_side.py │ ├── connection.py │ ├── datasource.py │ ├── meta.py │ ├── project.py │ ├── queue.py │ ├── tenant.py │ ├── user.py │ └── worker_group.py │ ├── resources_plugin │ ├── __init__.py │ ├── base │ │ ├── __init__.py │ │ ├── bucket.py │ │ └── git.py │ ├── github.py │ ├── gitlab.py │ ├── local.py │ ├── oss.py │ └── s3.py │ ├── tasks │ ├── __init__.py │ ├── condition.py │ ├── datax.py │ ├── dependent.py │ ├── dvc.py │ ├── flink.py │ ├── func_wrap.py │ ├── http.py │ ├── kubernetes.py │ ├── map_reduce.py │ ├── mlflow.py │ ├── openmldb.py │ ├── procedure.py │ ├── python.py │ ├── pytorch.py │ ├── sagemaker.py │ ├── shell.py │ ├── spark.py │ ├── sql.py │ ├── sub_process.py │ ├── sub_workflow.py │ └── switch.py │ ├── utils │ ├── __init__.py │ ├── date.py │ ├── file.py │ ├── string.py │ ├── versions.py │ └── yaml_parser.py │ └── version_ext └── tests ├── __init__.py ├── cli ├── __init__.py ├── test_config.py └── test_version.py ├── core ├── __init__.py ├── test_engine.py ├── test_local_parameter.py ├── test_resource_definition.py ├── test_task.py ├── test_workflow.py └── test_yaml_workflow.py ├── example ├── __init__.py └── test_example.py ├── integration ├── __init__.py ├── conftest.py ├── test_java_gateway.py ├── test_process_definition.py ├── test_project.py ├── test_resources.py ├── test_submit_examples.py ├── test_tenant.py └── test_user.py ├── models ├── __init__.py └── test_database.py ├── resources_plugin ├── __init__.py ├── test_github.py ├── test_gitlab.py ├── test_local.py ├── test_oss.py ├── test_resource_plugin.py └── test_s3.py ├── tasks ├── __init__.py ├── test_condition.py ├── test_datax.py ├── test_dependent.py ├── test_dvc.py ├── test_flink.py ├── test_func_wrap.py ├── test_http.py ├── test_kubernetes.py ├── test_map_reduce.py ├── test_mlflow.py ├── test_openmldb.py ├── test_procedure.py ├── test_python.py ├── test_pytorch.py ├── test_sagemaker.py ├── test_shell.py ├── test_spark.py ├── test_sql.py ├── test_sub_workflow.py └── test_switch.py ├── test_configuration.py ├── test_docs.py ├── testing ├── __init__.py ├── cli.py ├── constants.py ├── decorator.py ├── docker_wrapper.py ├── file.py ├── path.py └── task.py └── utils ├── __init__.py ├── test_date.py ├── test_default_config_yaml.py ├── test_file.py ├── test_string.py ├── test_versions.py └── test_yaml_parser.py /.asf.yaml: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one or more 3 | # contributor license agreements. See the NOTICE file distributed with 4 | # this work for additional information regarding copyright ownership. 5 | # The ASF licenses this file to You under the Apache License, Version 2.0 6 | # (the "License"); you may not use this file except in compliance with 7 | # the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | # 17 | 18 | github: 19 | description: Apache DolphinScheduler Python API, aka PyDolphinscheduler. 20 | homepage: https://dolphinscheduler.apache.org/python/main 21 | labels: 22 | - dolphinscheduler 23 | - workflow 24 | - python 25 | - api 26 | features: 27 | issues: true 28 | enabled_merge_buttons: 29 | squash: true 30 | merge: false 31 | rebase: false 32 | del_branch_on_merge: true 33 | protected_branches: 34 | main: 35 | required_status_checks: 36 | contexts: 37 | - CI 38 | -------------------------------------------------------------------------------- /.dlc.json: -------------------------------------------------------------------------------- 1 | { 2 | "ignorePatterns": [ 3 | { 4 | "pattern": "^http://localhost" 5 | }, 6 | { 7 | "pattern": "^https://img.shields.io/badge" 8 | }, 9 | { 10 | "pattern": "https://twitter.com/dolphinschedule" 11 | } 12 | ], 13 | "httpHeaders": [ 14 | { 15 | "urls": ["https://docs.github.com/"], 16 | "headers": { 17 | "Accept-Encoding": "zstd, br, gzip, deflate" 18 | } 19 | } 20 | ], 21 | "timeout": "10s", 22 | "retryOn429": true, 23 | "retryCount": 10, 24 | "fallbackRetryDelay": "1000s", 25 | "aliveStatusCodes": [ 26 | 200, 27 | 0 28 | ] 29 | } 30 | -------------------------------------------------------------------------------- /.github/PULL_REQUEST_TEMPLATE.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | ## Brief Summary of The Change 4 | 5 | 6 | 7 | ## Pull Request checklist 8 | 9 | I confirm that the following checklist has been completed. 10 | 11 | - [ ] Add/Change **test cases** for the changes. 12 | - [ ] Add/Change the related **documentation**, should also change `docs/source/config.rst` when you change file `default_config.yaml`. 13 | - [ ] (Optional) Add your change to `UPDATING.md` when it is an incompatible change. 14 | -------------------------------------------------------------------------------- /.github/codecov.yml: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | # Configuration file for codecov.io, https://docs.codecov.io/docs/codecovyml-reference 19 | 20 | codecov: 21 | branch: main 22 | bot: "codecov-io" 23 | ci: 24 | - "github.com" 25 | max_report_age: 24 26 | disable_default_path_fixes: no 27 | require_ci_to_pass: yes 28 | notify: 29 | after_n_builds: 1 30 | wait_for_ci: yes 31 | 32 | coverage: 33 | precision: 2 34 | round: down 35 | range: "70...100" 36 | status: 37 | project: 38 | default: 39 | target: auto 40 | threshold: 0% -------------------------------------------------------------------------------- /.github/mergeable.yml: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | --- 18 | version: 2 19 | mergeable: 20 | - when: pull_request.* 21 | name: label-check 22 | validate: 23 | - do: label 24 | must_include: 25 | regex: 'enhancement|bug|improvement|documentation|chore' 26 | message: 'Label must include one of the following: `enhancement`, `bug`, `improvement`, `documentation`, `chore`' 27 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Editor 2 | .idea/ 3 | .vscode/ 4 | 5 | # Cache 6 | __pycache__/ 7 | .DS_Store 8 | .tox/ 9 | .pytest_cache/ 10 | MANIFEST 11 | 12 | # Build 13 | build/ 14 | dist/ 15 | *egg-info/ 16 | 17 | # Test coverage 18 | .coverage 19 | coverage.xml 20 | htmlcov/ 21 | 22 | # the pydolphinscheduler config 23 | config.yaml 24 | 25 | # Release 26 | release/ 27 | -------------------------------------------------------------------------------- /.licenserc.yaml: -------------------------------------------------------------------------------- 1 | # Licensed to Apache Software Foundation (ASF) under one or more contributor 2 | # license agreements. See the NOTICE file distributed with 3 | # this work for additional information regarding copyright 4 | # ownership. Apache Software Foundation (ASF) licenses this file to you under 5 | # the Apache License, Version 2.0 (the "License"); you may 6 | # not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | header: 19 | license: 20 | spdx-id: Apache-2.0 21 | copyright-owner: Apache Software Foundation 22 | 23 | paths-ignore: 24 | - NOTICE 25 | - LICENSE 26 | - 'dist' 27 | - '**/*.md' 28 | - '**/.gitkeep' 29 | 30 | comment: on-failure 31 | -------------------------------------------------------------------------------- /.pre-commit-config.yaml: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | # See https://pre-commit.com for more information 19 | # See https://pre-commit.com/hooks.html for more hooks 20 | 21 | default_stages: [commit, push] 22 | default_language_version: 23 | # force all python hooks to run python3 24 | python: python3 25 | repos: 26 | # Python API Hooks 27 | - repo: https://github.com/psf/black 28 | rev: 22.3.0 29 | hooks: 30 | - id: black 31 | - repo: https://github.com/astral-sh/ruff-pre-commit 32 | rev: v0.1.0 33 | hooks: 34 | - id: ruff 35 | args: [ --fix, --exit-non-zero-on-fix ] 36 | -------------------------------------------------------------------------------- /.ruff.toml: -------------------------------------------------------------------------------- 1 | src = ["src"] 2 | 3 | target-version = "py39" 4 | 5 | # max-line-length = 110 6 | line-length = 110 7 | 8 | lint.extend-select = [ 9 | "I", # isort 10 | "D", # pydocstyle 11 | "UP", # pyupgrade 12 | ] 13 | 14 | lint.ignore = [ 15 | # D107: Missing docstring in __init__ 16 | "D107", 17 | # D105: Missing docstring in magic method 18 | "D105", 19 | # D418: Function/ Method decorated with @overload shouldn’t contain a docstring 20 | "D418", 21 | # D400: First line should end with a period 22 | "D400", 23 | ] 24 | 25 | # Exclude a variety of commonly ignored directories. 26 | exclude = [ 27 | "__pycache__", 28 | ".egg-info", 29 | ".eggs", 30 | ".git", 31 | ".pytest_cache", 32 | ".tox", 33 | "build", 34 | "dist", 35 | "examples", 36 | "venv", 37 | "docs/source/conf.py", 38 | "htmlcov" 39 | ] 40 | 41 | [lint.extend-per-file-ignores] 42 | "*/pydolphinscheduler/side/__init__.py" = ["F401"] 43 | "*/pydolphinscheduler/tasks/__init__.py" = ["F401"] 44 | 45 | [lint.isort] 46 | # Mark sqlfluff, test and it's plugins as known first party 47 | known-first-party = [ 48 | "pydolphinscheduler", 49 | ] 50 | 51 | [lint.pydocstyle] 52 | convention = "google" -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | include LICENSE 19 | include NOTICE 20 | include README.md 21 | include CONTRIBUTING.md 22 | include RELEASE.md 23 | include UPDATING.md 24 | -------------------------------------------------------------------------------- /NOTICE: -------------------------------------------------------------------------------- 1 | Apache DolphinScheduler 2 | Copyright 2017-2022 The Apache Software Foundation 3 | 4 | This product includes software developed at 5 | The Apache Software Foundation (http://www.apache.org/). 6 | -------------------------------------------------------------------------------- /docs/Makefile: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | # Minimal makefile for Sphinx documentation 19 | # 20 | 21 | # You can set these variables from the command line, and also 22 | # from the environment for the first two. 23 | 24 | # Add opts `turn warnings into errors` strict sphinx-build behavior 25 | SPHINXOPTS ?= -W 26 | SPHINXBUILD ?= sphinx-build 27 | SPHINXMULTIVERSION ?= sphinx-multiversion 28 | SOURCEDIR = source 29 | BUILDDIR = build 30 | 31 | # Put it first so that "make" without argument is like "make help". 32 | help: 33 | @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) 34 | 35 | .PHONY: help Makefile 36 | 37 | # Catch-all target: route all unknown targets to Sphinx using the new 38 | # "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). 39 | %: Makefile 40 | @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) 41 | 42 | # Create multiple version of docs 43 | multiversion: 44 | @$(SPHINXMULTIVERSION) "$(SOURCEDIR)" "$(BUILDDIR)/html" 45 | -------------------------------------------------------------------------------- /docs/make.bat: -------------------------------------------------------------------------------- 1 | REM Licensed to the Apache Software Foundation (ASF) under one 2 | REM or more contributor license agreements. See the NOTICE file 3 | REM distributed with this work for additional information 4 | REM regarding copyright ownership. The ASF licenses this file 5 | REM to you under the Apache License, Version 2.0 (the 6 | REM "License"); you may not use this file except in compliance 7 | REM with the License. You may obtain a copy of the License at 8 | REM 9 | REM http://www.apache.org/licenses/LICENSE-2.0 10 | REM 11 | REM Unless required by applicable law or agreed to in writing, 12 | REM software distributed under the License is distributed on an 13 | REM "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | REM KIND, either express or implied. See the License for the 15 | REM specific language governing permissions and limitations 16 | REM under the License. 17 | 18 | @ECHO OFF 19 | 20 | pushd %~dp0 21 | 22 | REM Command file for Sphinx documentation 23 | 24 | if "%SPHINXBUILD%" == "" ( 25 | set SPHINXBUILD=sphinx-build 26 | ) 27 | set SOURCEDIR=source 28 | set BUILDDIR=build 29 | REM Add opts `turn warnings into errors` strict sphinx-build behavior 30 | set SPHINXOPTS=-W 31 | 32 | if "%1" == "" goto help 33 | 34 | %SPHINXBUILD% >NUL 2>NUL 35 | if errorlevel 9009 ( 36 | echo. 37 | echo.The 'sphinx-build' command was not found. Make sure you have Sphinx 38 | echo.installed, then set the SPHINXBUILD environment variable to point 39 | echo.to the full path of the 'sphinx-build' executable. Alternatively you 40 | echo.may add the Sphinx directory to PATH. 41 | echo. 42 | echo.If you don't have Sphinx installed, grab it from 43 | echo.https://www.sphinx-doc.org/ 44 | exit /b 1 45 | ) 46 | 47 | %SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% 48 | goto end 49 | 50 | :help 51 | %SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% 52 | 53 | :end 54 | popd 55 | -------------------------------------------------------------------------------- /docs/source/_static/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/apache/dolphinscheduler-sdk-python/a999364878008c83b33de14f0fc44852e2347cec/docs/source/_static/.gitkeep -------------------------------------------------------------------------------- /docs/source/_templates/versioning.html: -------------------------------------------------------------------------------- 1 | {# 2 | Licensed to the Apache Software Foundation (ASF) under one 3 | or more contributor license agreements. See the NOTICE file 4 | distributed with this work for additional information 5 | regarding copyright ownership. The ASF licenses this file 6 | to you under the Apache License, Version 2.0 (the 7 | "License"); you may not use this file except in compliance 8 | with the License. You may obtain a copy of the License at 9 | 10 | http://www.apache.org/licenses/LICENSE-2.0 11 | 12 | Unless required by applicable law or agreed to in writing, 13 | software distributed under the License is distributed on an 14 | "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | KIND, either express or implied. See the License for the 16 | specific language governing permissions and limitations 17 | under the License. 18 | #} 19 | 20 | {% if versions %} 21 |

{{ _('Versions') }}

22 | 27 | {% endif %} 28 | -------------------------------------------------------------------------------- /docs/source/_templates/versions.html: -------------------------------------------------------------------------------- 1 | {# 2 | Licensed to the Apache Software Foundation (ASF) under one 3 | or more contributor license agreements. See the NOTICE file 4 | distributed with this work for additional information 5 | regarding copyright ownership. The ASF licenses this file 6 | to you under the Apache License, Version 2.0 (the 7 | "License"); you may not use this file except in compliance 8 | with the License. You may obtain a copy of the License at 9 | 10 | http://www.apache.org/licenses/LICENSE-2.0 11 | 12 | Unless required by applicable law or agreed to in writing, 13 | software distributed under the License is distributed on an 14 | "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | KIND, either express or implied. See the License for the 16 | specific language governing permissions and limitations 17 | under the License. 18 | #} 19 | 20 | {%- if current_version %} 21 |
22 | 23 | Other Versions 24 | v: {{ current_version.name }} 25 | 26 | 27 |
28 | {%- if versions.tags %} 29 |
30 |
Tags
31 | {%- for item in versions.tags %} 32 |
{{ item.name }}
33 | {%- endfor %} 34 |
35 | {%- endif %} 36 | {%- if versions.branches %} 37 |
38 |
Branches
39 | {%- for item in versions.branches %} 40 |
{{ item.name }}
41 | {%- endfor %} 42 |
43 | {%- endif %} 44 |
45 |
46 | {%- endif %} 47 | -------------------------------------------------------------------------------- /docs/source/api.rst: -------------------------------------------------------------------------------- 1 | .. Licensed to the Apache Software Foundation (ASF) under one 2 | or more contributor license agreements. See the NOTICE file 3 | distributed with this work for additional information 4 | regarding copyright ownership. The ASF licenses this file 5 | to you under the Apache License, Version 2.0 (the 6 | "License"); you may not use this file except in compliance 7 | with the License. You may obtain a copy of the License at 8 | 9 | .. http://www.apache.org/licenses/LICENSE-2.0 10 | 11 | .. Unless required by applicable law or agreed to in writing, 12 | software distributed under the License is distributed on an 13 | "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | KIND, either express or implied. See the License for the 15 | specific language governing permissions and limitations 16 | under the License. 17 | 18 | API 19 | === 20 | 21 | Core 22 | ---- 23 | 24 | .. automodule:: pydolphinscheduler.core 25 | :inherited-members: 26 | 27 | Models 28 | ------ 29 | 30 | .. automodule:: pydolphinscheduler.models 31 | :inherited-members: 32 | 33 | Tasks 34 | ----- 35 | 36 | .. automodule:: pydolphinscheduler.tasks 37 | :inherited-members: 38 | 39 | Constants 40 | --------- 41 | 42 | .. automodule:: pydolphinscheduler.constants 43 | 44 | Exceptions 45 | ---------- 46 | 47 | .. automodule:: pydolphinscheduler.exceptions 48 | -------------------------------------------------------------------------------- /docs/source/changelog.rst: -------------------------------------------------------------------------------- 1 | Changelog 2 | ========= 3 | 4 | .. changelog:: 5 | :changelog-url: https://dolphinscheduler.apache.org/python/main#changelog 6 | :github: https://github.com/apache/dolphinscheduler-sdk-python/releases/ 7 | :pypi: https://pypi.org/project/apache-dolphinscheduler/ 8 | -------------------------------------------------------------------------------- /docs/source/cli.rst: -------------------------------------------------------------------------------- 1 | .. Licensed to the Apache Software Foundation (ASF) under one 2 | or more contributor license agreements. See the NOTICE file 3 | distributed with this work for additional information 4 | regarding copyright ownership. The ASF licenses this file 5 | to you under the Apache License, Version 2.0 (the 6 | "License"); you may not use this file except in compliance 7 | with the License. You may obtain a copy of the License at 8 | 9 | .. http://www.apache.org/licenses/LICENSE-2.0 10 | 11 | .. Unless required by applicable law or agreed to in writing, 12 | software distributed under the License is distributed on an 13 | "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | KIND, either express or implied. See the License for the 15 | specific language governing permissions and limitations 16 | under the License. 17 | 18 | Command Line Interface 19 | ====================== 20 | 21 | *PyDolphinScheduler* have mechanism call CLI(command line interface) to help user control it in Shell. 22 | 23 | Prepare 24 | ------- 25 | 26 | You have to :ref:`install PyDolphinScheduler ` first before you using 27 | its CLI 28 | 29 | Usage 30 | ----- 31 | 32 | Here is basic usage about the command line of *PyDolphinScheduler* 33 | 34 | .. click:: pydolphinscheduler.cli.commands:cli 35 | :prog: pydolphinscheduler 36 | :nested: full 37 | -------------------------------------------------------------------------------- /docs/source/howto/index.rst: -------------------------------------------------------------------------------- 1 | .. Licensed to the Apache Software Foundation (ASF) under one 2 | or more contributor license agreements. See the NOTICE file 3 | distributed with this work for additional information 4 | regarding copyright ownership. The ASF licenses this file 5 | to you under the Apache License, Version 2.0 (the 6 | "License"); you may not use this file except in compliance 7 | with the License. You may obtain a copy of the License at 8 | 9 | .. http://www.apache.org/licenses/LICENSE-2.0 10 | 11 | .. Unless required by applicable law or agreed to in writing, 12 | software distributed under the License is distributed on an 13 | "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | KIND, either express or implied. See the License for the 15 | specific language governing permissions and limitations 16 | under the License. 17 | 18 | HOWTOs 19 | ====== 20 | 21 | pydolphinscheduler HOWTOs are documents that cover a single, specific topic, and attempt to cover it fairly 22 | completely. This collection is an effort to foster documentation that is more detailed than the :doc:`../concept` 23 | and :doc:`../tutorial`. 24 | 25 | Currently, the HOWTOs are: 26 | 27 | .. toctree:: 28 | :maxdepth: 2 29 | 30 | remote-submit 31 | multi-resources 32 | -------------------------------------------------------------------------------- /docs/source/howto/remote-submit.rst: -------------------------------------------------------------------------------- 1 | .. Licensed to the Apache Software Foundation (ASF) under one 2 | or more contributor license agreements. See the NOTICE file 3 | distributed with this work for additional information 4 | regarding copyright ownership. The ASF licenses this file 5 | to you under the Apache License, Version 2.0 (the 6 | "License"); you may not use this file except in compliance 7 | with the License. You may obtain a copy of the License at 8 | 9 | .. http://www.apache.org/licenses/LICENSE-2.0 10 | 11 | .. Unless required by applicable law or agreed to in writing, 12 | software distributed under the License is distributed on an 13 | "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | KIND, either express or implied. See the License for the 15 | specific language governing permissions and limitations 16 | under the License. 17 | 18 | Submit Your Code from Different machine 19 | ======================================= 20 | 21 | Generally, we use pydolphinscheduler as a client to DolphinScheduler, and consider we may change our workflow 22 | code frequently, the best practice is running :ref:`python gateway service ` 23 | in your server machine and submit the workflow code from your development machine, like a laptop or PC. This behavior 24 | is supported by pydolphinscheduler out of box with one or two single command lines. 25 | 26 | Export Configuration File 27 | ------------------------- 28 | 29 | .. code-block:: bash 30 | 31 | pydolphinscheduler config --init 32 | 33 | your could find more detail in :ref:`configuration exporting ` 34 | 35 | Run API Server in Other Host 36 | ---------------------------- 37 | 38 | .. code-block:: bash 39 | 40 | pydolphinscheduler config --set java_gateway.address 41 | 42 | your could find more detail in :ref:`configuration setting ` 43 | 44 | Run API Server in Other Port 45 | ---------------------------- 46 | 47 | .. code-block:: bash 48 | 49 | pydolphinscheduler config --set java_gateway.port 50 | 51 | your could find more detail in :ref:`configuration setting ` 52 | -------------------------------------------------------------------------------- /docs/source/resources_plugin/develop.rst: -------------------------------------------------------------------------------- 1 | .. Licensed to the Apache Software Foundation (ASF) under one 2 | or more contributor license agreements. See the NOTICE file 3 | distributed with this work for additional information 4 | regarding copyright ownership. The ASF licenses this file 5 | to you under the Apache License, Version 2.0 (the 6 | "License"); you may not use this file except in compliance 7 | with the License. You may obtain a copy of the License at 8 | 9 | .. http://www.apache.org/licenses/LICENSE-2.0 10 | 11 | .. Unless required by applicable law or agreed to in writing, 12 | software distributed under the License is distributed on an 13 | "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | KIND, either express or implied. See the License for the 15 | specific language governing permissions and limitations 16 | under the License. 17 | 18 | How to develop 19 | ============== 20 | 21 | When you want to create a new resource plugin, you need to add a new class in the module `resources_plugin`. 22 | 23 | The resource plugin class needs to inherit the abstract class `ResourcePlugin` and implement its abstract method `read_file` function. 24 | 25 | The parameter of the `__init__` function of `ResourcePlugin` is the prefix of STR type. You can override this function when necessary. 26 | 27 | The `read_file` function parameter of `ResourcePlugin` is the file suffix of STR type, and its return value is the file content, if it exists and is readable. 28 | 29 | 30 | Example 31 | ------- 32 | - Method `__init__`: Initiation method with `param`:`prefix` 33 | 34 | .. literalinclude:: ../../../src/pydolphinscheduler/resources_plugin/local.py 35 | :start-after: [start init_method] 36 | :end-before: [end init_method] 37 | 38 | - Method `read_file`: Get content from the given URI, The function parameter is the suffix of the file path. 39 | 40 | The file prefix has been initialized in init of the resource plugin. 41 | 42 | The prefix plus suffix is the absolute path of the file in this resource. 43 | 44 | .. literalinclude:: ../../../src/pydolphinscheduler/resources_plugin/local.py 45 | :start-after: [start read_file_method] 46 | :end-before: [end read_file_method] 47 | -------------------------------------------------------------------------------- /docs/source/resources_plugin/github.rst: -------------------------------------------------------------------------------- 1 | .. Licensed to the Apache Software Foundation (ASF) under one 2 | or more contributor license agreements. See the NOTICE file 3 | distributed with this work for additional information 4 | regarding copyright ownership. The ASF licenses this file 5 | to you under the Apache License, Version 2.0 (the 6 | "License"); you may not use this file except in compliance 7 | with the License. You may obtain a copy of the License at 8 | 9 | .. http://www.apache.org/licenses/LICENSE-2.0 10 | 11 | .. Unless required by applicable law or agreed to in writing, 12 | software distributed under the License is distributed on an 13 | "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | KIND, either express or implied. See the License for the 15 | specific language governing permissions and limitations 16 | under the License. 17 | 18 | GitHub 19 | ====== 20 | 21 | `GitHub` is a github resource plugin for pydolphinscheduler. 22 | 23 | When using a github resource plugin, you only need to add the `resource_plugin` parameter in the task subclass or workflow definition, 24 | such as `resource_plugin=GitHub(prefix="https://github.com/xxx", access_token="ghpxx")`. 25 | The token parameter is optional. You need to add it when your repository is a private repository. 26 | 27 | You can view this `document `_ 28 | when creating a token. 29 | 30 | For the specific use of resource plugins, you can see `How to use` in :doc:`resource-plugin` 31 | 32 | Dive Into 33 | --------- 34 | 35 | .. automodule:: pydolphinscheduler.resources_plugin.github -------------------------------------------------------------------------------- /docs/source/resources_plugin/gitlab.rst: -------------------------------------------------------------------------------- 1 | .. Licensed to the Apache Software Foundation (ASF) under one 2 | or more contributor license agreements. See the NOTICE file 3 | distributed with this work for additional information 4 | regarding copyright ownership. The ASF licenses this file 5 | to you under the Apache License, Version 2.0 (the 6 | "License"); you may not use this file except in compliance 7 | with the License. You may obtain a copy of the License at 8 | 9 | .. http://www.apache.org/licenses/LICENSE-2.0 10 | 11 | .. Unless required by applicable law or agreed to in writing, 12 | software distributed under the License is distributed on an 13 | "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | KIND, either express or implied. See the License for the 15 | specific language governing permissions and limitations 16 | under the License. 17 | 18 | GitLab 19 | ====== 20 | 21 | `GitLab` is a gitlab resource plugin for pydolphinscheduler. 22 | 23 | When using a gitlab resource plugin, you only need to add the `resource_plugin` parameter in the task subclass or workflow definition, 24 | such as `resource_plugin=GitLab(prefix="xxx")`, if it is a public repository. 25 | 26 | If it is a private or Internal repository, you can use three ways to obtain authentication. 27 | 28 | The first is `Personal Access Tokens`, using `resource_plugin=GitLab(prefix="xxx", private_token="xxx")`. 29 | 30 | The second method is to obtain authentication through `username` and `password`: 31 | 32 | using `resource_plugin=GitLab(prefix="xxx", username="username", password="pwd")`. 33 | 34 | The third method is to obtain authentication through `OAuth Token`: 35 | 36 | using `resource_plugin=GitLab(prefix="xxx", oauth_token="xx")`. 37 | 38 | You can view this `document `_ 39 | when creating a `Personal Access Tokens`. 40 | 41 | For the specific use of resource plugins, you can see `How to use` in :doc:`resource-plugin` 42 | 43 | Dive Into 44 | --------- 45 | 46 | .. automodule:: pydolphinscheduler.resources_plugin.gitlab -------------------------------------------------------------------------------- /docs/source/resources_plugin/index.rst: -------------------------------------------------------------------------------- 1 | .. Licensed to the Apache Software Foundation (ASF) under one 2 | or more contributor license agreements. See the NOTICE file 3 | distributed with this work for additional information 4 | regarding copyright ownership. The ASF licenses this file 5 | to you under the Apache License, Version 2.0 (the 6 | "License"); you may not use this file except in compliance 7 | with the License. You may obtain a copy of the License at 8 | 9 | .. http://www.apache.org/licenses/LICENSE-2.0 10 | 11 | .. Unless required by applicable law or agreed to in writing, 12 | software distributed under the License is distributed on an 13 | "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | KIND, either express or implied. See the License for the 15 | specific language governing permissions and limitations 16 | under the License. 17 | 18 | Resources_plugin 19 | ================ 20 | 21 | In this section 22 | 23 | .. toctree:: 24 | :maxdepth: 1 25 | 26 | develop 27 | resource-plugin 28 | local 29 | github 30 | gitlab 31 | oss 32 | s3 -------------------------------------------------------------------------------- /docs/source/resources_plugin/local.rst: -------------------------------------------------------------------------------- 1 | .. Licensed to the Apache Software Foundation (ASF) under one 2 | or more contributor license agreements. See the NOTICE file 3 | distributed with this work for additional information 4 | regarding copyright ownership. The ASF licenses this file 5 | to you under the Apache License, Version 2.0 (the 6 | "License"); you may not use this file except in compliance 7 | with the License. You may obtain a copy of the License at 8 | 9 | .. http://www.apache.org/licenses/LICENSE-2.0 10 | 11 | .. Unless required by applicable law or agreed to in writing, 12 | software distributed under the License is distributed on an 13 | "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | KIND, either express or implied. See the License for the 15 | specific language governing permissions and limitations 16 | under the License. 17 | 18 | Local 19 | ===== 20 | 21 | `Local` is a local resource plugin for pydolphinscheduler. 22 | 23 | When using a local resource plugin, you only need to add the `resource_plugin` parameter in the task subclass or workflow definition, 24 | such as `resource_plugin=Local("/tmp")`. 25 | 26 | 27 | For the specific use of resource plugins, you can see `How to use` in :doc:`./resource-plugin` 28 | 29 | Dive Into 30 | --------- 31 | 32 | .. automodule:: pydolphinscheduler.resources_plugin.local -------------------------------------------------------------------------------- /docs/source/resources_plugin/oss.rst: -------------------------------------------------------------------------------- 1 | .. Licensed to the Apache Software Foundation (ASF) under one 2 | or more contributor license agreements. See the NOTICE file 3 | distributed with this work for additional information 4 | regarding copyright ownership. The ASF licenses this file 5 | to you under the Apache License, Version 2.0 (the 6 | "License"); you may not use this file except in compliance 7 | with the License. You may obtain a copy of the License at 8 | 9 | .. http://www.apache.org/licenses/LICENSE-2.0 10 | 11 | .. Unless required by applicable law or agreed to in writing, 12 | software distributed under the License is distributed on an 13 | "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | KIND, either express or implied. See the License for the 15 | specific language governing permissions and limitations 16 | under the License. 17 | 18 | OSS 19 | === 20 | 21 | `OSS` is a Aliyun OSS resource plugin for pydolphinscheduler. 22 | 23 | When using a OSS resource plugin, you only need to add the `resource_plugin` parameter in the task subclass or workflow definition, 24 | such as `resource_plugin=OSS(prefix="xxx")`, if the file is publicly readable. 25 | 26 | When the file is private, using `resource_plugin=OSS(prefix="xxx", access_key_id="xxx", access_key_secret="xxx")` 27 | 28 | Notice 29 | The read permission of files in a bucket is inherited from the bucket by default. In other words, if the bucket is private, 30 | the files in it are also private. 31 | 32 | But the read permission of the files in the bucket can be changed, in other words, the files in the private bucket can also be read publicly. 33 | 34 | So whether the `AccessKey` is needed depends on whether the file is private or not. 35 | 36 | You can view this `document `_ 37 | when creating a pair `AccessKey`. 38 | 39 | For the specific use of resource plugins, you can see `How to use` in :doc:`resource-plugin` 40 | 41 | Dive Into 42 | --------- 43 | 44 | .. automodule:: pydolphinscheduler.resources_plugin.OSS 45 | -------------------------------------------------------------------------------- /docs/source/resources_plugin/s3.rst: -------------------------------------------------------------------------------- 1 | .. Licensed to the Apache Software Foundation (ASF) under one 2 | or more contributor license agreements. See the NOTICE file 3 | distributed with this work for additional information 4 | regarding copyright ownership. The ASF licenses this file 5 | to you under the Apache License, Version 2.0 (the 6 | "License"); you may not use this file except in compliance 7 | with the License. You may obtain a copy of the License at 8 | 9 | .. http://www.apache.org/licenses/LICENSE-2.0 10 | 11 | .. Unless required by applicable law or agreed to in writing, 12 | software distributed under the License is distributed on an 13 | "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | KIND, either express or implied. See the License for the 15 | specific language governing permissions and limitations 16 | under the License. 17 | 18 | S3 19 | == 20 | 21 | `S3` is a Amazon S3 resource plugin for pydolphinscheduler. 22 | 23 | When using a Amazon S3 resource plugin, you only need to add the `resource_plugin` parameter in the task subclass or workflow definition, 24 | such as `resource_plugin=S3(prefix="xxx")`, if the file is publicly readable. 25 | 26 | When the file is private, using `resource_plugin=S3(prefix="xxx", access_key_id="xxx", access_key_secret="xxx")` 27 | 28 | You can view this `document `_ 29 | when creating a pair `AccessKey`. 30 | 31 | For the specific use of resource plugins, you can see `How to use` in :doc:`resource-plugin` 32 | 33 | Dive Into 34 | --------- 35 | 36 | .. automodule:: pydolphinscheduler.resources_plugin.S3 37 | -------------------------------------------------------------------------------- /docs/source/tasks/condition.rst: -------------------------------------------------------------------------------- 1 | .. Licensed to the Apache Software Foundation (ASF) under one 2 | or more contributor license agreements. See the NOTICE file 3 | distributed with this work for additional information 4 | regarding copyright ownership. The ASF licenses this file 5 | to you under the Apache License, Version 2.0 (the 6 | "License"); you may not use this file except in compliance 7 | with the License. You may obtain a copy of the License at 8 | 9 | .. http://www.apache.org/licenses/LICENSE-2.0 10 | 11 | .. Unless required by applicable law or agreed to in writing, 12 | software distributed under the License is distributed on an 13 | "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | KIND, either express or implied. See the License for the 15 | specific language governing permissions and limitations 16 | under the License. 17 | 18 | Condition 19 | ========= 20 | 21 | A condition task type's example and dive into information of **PyDolphinScheduler**. 22 | 23 | Example 24 | ------- 25 | 26 | .. literalinclude:: ../../../src/pydolphinscheduler/examples/task_condition_example.py 27 | :start-after: [start workflow_declare] 28 | :end-before: [end workflow_declare] 29 | 30 | Dive Into 31 | --------- 32 | 33 | .. automodule:: pydolphinscheduler.tasks.condition 34 | 35 | YAML file example 36 | ----------------- 37 | 38 | .. literalinclude:: ../../../examples/yaml_define/Condition.yaml 39 | :start-after: # under the License. 40 | :language: yaml 41 | -------------------------------------------------------------------------------- /docs/source/tasks/datax.rst: -------------------------------------------------------------------------------- 1 | .. Licensed to the Apache Software Foundation (ASF) under one 2 | or more contributor license agreements. See the NOTICE file 3 | distributed with this work for additional information 4 | regarding copyright ownership. The ASF licenses this file 5 | to you under the Apache License, Version 2.0 (the 6 | "License"); you may not use this file except in compliance 7 | with the License. You may obtain a copy of the License at 8 | 9 | .. http://www.apache.org/licenses/LICENSE-2.0 10 | 11 | .. Unless required by applicable law or agreed to in writing, 12 | software distributed under the License is distributed on an 13 | "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | KIND, either express or implied. See the License for the 15 | specific language governing permissions and limitations 16 | under the License. 17 | 18 | Datax 19 | ===== 20 | 21 | A DataX task type's example and dive into information of **PyDolphinScheduler**. 22 | 23 | Example 24 | ------- 25 | 26 | .. literalinclude:: ../../../src/pydolphinscheduler/examples/task_datax_example.py 27 | :start-after: [start workflow_declare] 28 | :end-before: [end workflow_declare] 29 | 30 | Resource Limit Example 31 | ---------------------- 32 | 33 | We can add resource limit like CPU quota and max memory by passing parameters when declaring tasks. 34 | 35 | .. literalinclude:: ../../../src/pydolphinscheduler/examples/task_datax_example.py 36 | :start-after: [start resource_limit] 37 | :end-before: [end resource_limit] 38 | 39 | 40 | Dive Into 41 | --------- 42 | 43 | .. automodule:: pydolphinscheduler.tasks.datax 44 | 45 | YAML file example 46 | ----------------- 47 | 48 | .. literalinclude:: ../../../examples/yaml_define/DataX.yaml 49 | :start-after: # under the License. 50 | :language: yaml 51 | 52 | 53 | example_datax.json: 54 | 55 | .. literalinclude:: ../../../examples/yaml_define/example_datax.json 56 | :language: json 57 | -------------------------------------------------------------------------------- /docs/source/tasks/dependent.rst: -------------------------------------------------------------------------------- 1 | .. Licensed to the Apache Software Foundation (ASF) under one 2 | or more contributor license agreements. See the NOTICE file 3 | distributed with this work for additional information 4 | regarding copyright ownership. The ASF licenses this file 5 | to you under the Apache License, Version 2.0 (the 6 | "License"); you may not use this file except in compliance 7 | with the License. You may obtain a copy of the License at 8 | 9 | .. http://www.apache.org/licenses/LICENSE-2.0 10 | 11 | .. Unless required by applicable law or agreed to in writing, 12 | software distributed under the License is distributed on an 13 | "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | KIND, either express or implied. See the License for the 15 | specific language governing permissions and limitations 16 | under the License. 17 | 18 | Dependent 19 | ========= 20 | 21 | A dependent task type's example and dive into information of **PyDolphinScheduler**. 22 | 23 | Example 24 | ------- 25 | 26 | .. literalinclude:: ../../../src/pydolphinscheduler/examples/task_dependent_example.py 27 | :start-after: [start workflow_declare] 28 | :end-before: [end workflow_declare] 29 | 30 | Dive Into 31 | --------- 32 | 33 | .. automodule:: pydolphinscheduler.tasks.dependent 34 | 35 | 36 | YAML file example 37 | ----------------- 38 | 39 | .. literalinclude:: ../../../examples/yaml_define/Dependent.yaml 40 | :start-after: # under the License. 41 | :language: yaml 42 | 43 | Dependent_External.yaml: 44 | 45 | .. literalinclude:: ../../../examples/yaml_define/Dependent_External.yaml 46 | :start-after: # under the License. 47 | :language: yaml 48 | -------------------------------------------------------------------------------- /docs/source/tasks/dvc.rst: -------------------------------------------------------------------------------- 1 | .. Licensed to the Apache Software Foundation (ASF) under one 2 | or more contributor license agreements. See the NOTICE file 3 | distributed with this work for additional information 4 | regarding copyright ownership. The ASF licenses this file 5 | to you under the Apache License, Version 2.0 (the 6 | "License"); you may not use this file except in compliance 7 | with the License. You may obtain a copy of the License at 8 | 9 | .. http://www.apache.org/licenses/LICENSE-2.0 10 | 11 | .. Unless required by applicable law or agreed to in writing, 12 | software distributed under the License is distributed on an 13 | "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | KIND, either express or implied. See the License for the 15 | specific language governing permissions and limitations 16 | under the License. 17 | 18 | DVC 19 | === 20 | 21 | A DVC task type's example and dive into information of **PyDolphinScheduler**. 22 | 23 | Example 24 | ------- 25 | 26 | .. literalinclude:: ../../../src/pydolphinscheduler/examples/task_dvc_example.py 27 | :start-after: [start workflow_declare] 28 | :end-before: [end workflow_declare] 29 | 30 | Dive Into 31 | --------- 32 | 33 | .. automodule:: pydolphinscheduler.tasks.dvc 34 | 35 | 36 | YAML file example 37 | ----------------- 38 | 39 | .. literalinclude:: ../../../examples/yaml_define/Dvc.yaml 40 | :start-after: # under the License. 41 | :language: yaml 42 | -------------------------------------------------------------------------------- /docs/source/tasks/flink.rst: -------------------------------------------------------------------------------- 1 | .. Licensed to the Apache Software Foundation (ASF) under one 2 | or more contributor license agreements. See the NOTICE file 3 | distributed with this work for additional information 4 | regarding copyright ownership. The ASF licenses this file 5 | to you under the Apache License, Version 2.0 (the 6 | "License"); you may not use this file except in compliance 7 | with the License. You may obtain a copy of the License at 8 | 9 | .. http://www.apache.org/licenses/LICENSE-2.0 10 | 11 | .. Unless required by applicable law or agreed to in writing, 12 | software distributed under the License is distributed on an 13 | "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | KIND, either express or implied. See the License for the 15 | specific language governing permissions and limitations 16 | under the License. 17 | 18 | Flink 19 | ===== 20 | 21 | A flink task type's example and dive into information of **PyDolphinScheduler**. 22 | 23 | Example 24 | ------- 25 | 26 | .. literalinclude:: ../../../src/pydolphinscheduler/examples/task_flink_example.py 27 | :start-after: [start workflow_declare] 28 | :end-before: [end workflow_declare] 29 | 30 | Dive Into 31 | --------- 32 | 33 | .. automodule:: pydolphinscheduler.tasks.flink 34 | 35 | YAML file example 36 | ----------------- 37 | 38 | .. literalinclude:: ../../../examples/yaml_define/Flink.yaml 39 | :start-after: # under the License. 40 | :language: yaml 41 | -------------------------------------------------------------------------------- /docs/source/tasks/http.rst: -------------------------------------------------------------------------------- 1 | .. Licensed to the Apache Software Foundation (ASF) under one 2 | or more contributor license agreements. See the NOTICE file 3 | distributed with this work for additional information 4 | regarding copyright ownership. The ASF licenses this file 5 | to you under the Apache License, Version 2.0 (the 6 | "License"); you may not use this file except in compliance 7 | with the License. You may obtain a copy of the License at 8 | 9 | .. http://www.apache.org/licenses/LICENSE-2.0 10 | 11 | .. Unless required by applicable law or agreed to in writing, 12 | software distributed under the License is distributed on an 13 | "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | KIND, either express or implied. See the License for the 15 | specific language governing permissions and limitations 16 | under the License. 17 | 18 | HTTP 19 | ==== 20 | 21 | .. automodule:: pydolphinscheduler.tasks.http 22 | 23 | 24 | YAML file example 25 | ----------------- 26 | 27 | .. literalinclude:: ../../../examples/yaml_define/Http.yaml 28 | :start-after: # under the License. 29 | :language: yaml 30 | -------------------------------------------------------------------------------- /docs/source/tasks/index.rst: -------------------------------------------------------------------------------- 1 | .. Licensed to the Apache Software Foundation (ASF) under one 2 | or more contributor license agreements. See the NOTICE file 3 | distributed with this work for additional information 4 | regarding copyright ownership. The ASF licenses this file 5 | to you under the Apache License, Version 2.0 (the 6 | "License"); you may not use this file except in compliance 7 | with the License. You may obtain a copy of the License at 8 | 9 | .. http://www.apache.org/licenses/LICENSE-2.0 10 | 11 | .. Unless required by applicable law or agreed to in writing, 12 | software distributed under the License is distributed on an 13 | "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | KIND, either express or implied. See the License for the 15 | specific language governing permissions and limitations 16 | under the License. 17 | 18 | Tasks 19 | ===== 20 | 21 | In this section 22 | 23 | .. toctree:: 24 | :maxdepth: 1 25 | 26 | func_wrap 27 | shell 28 | sql 29 | python 30 | http 31 | sub_workflow 32 | 33 | switch 34 | condition 35 | dependent 36 | 37 | spark 38 | flink 39 | map_reduce 40 | procedure 41 | kubernetes 42 | 43 | datax 44 | sagemaker 45 | mlflow 46 | openmldb 47 | pytorch 48 | dvc 49 | -------------------------------------------------------------------------------- /docs/source/tasks/kubernetes.rst: -------------------------------------------------------------------------------- 1 | .. Licensed to the Apache Software Foundation (ASF) under one 2 | or more contributor license agreements. See the NOTICE file 3 | distributed with this work for additional information 4 | regarding copyright ownership. The ASF licenses this file 5 | to you under the Apache License, Version 2.0 (the 6 | "License"); you may not use this file except in compliance 7 | with the License. You may obtain a copy of the License at 8 | 9 | .. http://www.apache.org/licenses/LICENSE-2.0 10 | 11 | .. Unless required by applicable law or agreed to in writing, 12 | software distributed under the License is distributed on an 13 | "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | KIND, either express or implied. See the License for the 15 | specific language governing permissions and limitations 16 | under the License. 17 | 18 | Kubernetes 19 | ========== 20 | 21 | 22 | A Kubernetes task type's example and dive into information of **PyDolphinScheduler**. 23 | 24 | Example 25 | ------- 26 | 27 | .. literalinclude:: ../../../src/pydolphinscheduler/examples/task_kubernetes_example.py 28 | :start-after: [start workflow_declare] 29 | :end-before: [end workflow_declare] 30 | 31 | Dive Into 32 | --------- 33 | 34 | .. automodule:: pydolphinscheduler.tasks.kubernetes 35 | 36 | 37 | YAML file example 38 | ----------------- 39 | 40 | .. literalinclude:: ../../../examples/yaml_define/Kubernetes.yaml 41 | :start-after: # under the License. 42 | :language: yaml 43 | -------------------------------------------------------------------------------- /docs/source/tasks/map_reduce.rst: -------------------------------------------------------------------------------- 1 | .. Licensed to the Apache Software Foundation (ASF) under one 2 | or more contributor license agreements. See the NOTICE file 3 | distributed with this work for additional information 4 | regarding copyright ownership. The ASF licenses this file 5 | to you under the Apache License, Version 2.0 (the 6 | "License"); you may not use this file except in compliance 7 | with the License. You may obtain a copy of the License at 8 | 9 | .. http://www.apache.org/licenses/LICENSE-2.0 10 | 11 | .. Unless required by applicable law or agreed to in writing, 12 | software distributed under the License is distributed on an 13 | "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | KIND, either express or implied. See the License for the 15 | specific language governing permissions and limitations 16 | under the License. 17 | 18 | Map Reduce 19 | ========== 20 | 21 | 22 | A Map Reduce task type's example and dive into information of **PyDolphinScheduler**. 23 | 24 | Example 25 | ------- 26 | 27 | .. literalinclude:: ../../../src/pydolphinscheduler/examples/task_map_reduce_example.py 28 | :start-after: [start workflow_declare] 29 | :end-before: [end workflow_declare] 30 | 31 | Dive Into 32 | --------- 33 | 34 | .. automodule:: pydolphinscheduler.tasks.map_reduce 35 | 36 | 37 | YAML file example 38 | ----------------- 39 | 40 | .. literalinclude:: ../../../examples/yaml_define/MapReduce.yaml 41 | :start-after: # under the License. 42 | :language: yaml 43 | -------------------------------------------------------------------------------- /docs/source/tasks/mlflow.rst: -------------------------------------------------------------------------------- 1 | .. Licensed to the Apache Software Foundation (ASF) under one 2 | or more contributor license agreements. See the NOTICE file 3 | distributed with this work for additional information 4 | regarding copyright ownership. The ASF licenses this file 5 | to you under the Apache License, Version 2.0 (the 6 | "License"); you may not use this file except in compliance 7 | with the License. You may obtain a copy of the License at 8 | 9 | .. http://www.apache.org/licenses/LICENSE-2.0 10 | 11 | .. Unless required by applicable law or agreed to in writing, 12 | software distributed under the License is distributed on an 13 | "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | KIND, either express or implied. See the License for the 15 | specific language governing permissions and limitations 16 | under the License. 17 | 18 | MLflow 19 | ========= 20 | 21 | 22 | A MLflow task type's example and dive into information of **PyDolphinScheduler**. 23 | 24 | Example 25 | ------- 26 | 27 | .. literalinclude:: ../../../src/pydolphinscheduler/examples/task_mlflow_example.py 28 | :start-after: [start workflow_declare] 29 | :end-before: [end workflow_declare] 30 | 31 | Dive Into 32 | --------- 33 | 34 | .. automodule:: pydolphinscheduler.tasks.mlflow 35 | 36 | 37 | YAML file example 38 | ----------------- 39 | 40 | .. literalinclude:: ../../../examples/yaml_define/mlflow.yaml 41 | :start-after: # under the License. 42 | :language: yaml 43 | -------------------------------------------------------------------------------- /docs/source/tasks/openmldb.rst: -------------------------------------------------------------------------------- 1 | .. Licensed to the Apache Software Foundation (ASF) under one 2 | or more contributor license agreements. See the NOTICE file 3 | distributed with this work for additional information 4 | regarding copyright ownership. The ASF licenses this file 5 | to you under the Apache License, Version 2.0 (the 6 | "License"); you may not use this file except in compliance 7 | with the License. You may obtain a copy of the License at 8 | 9 | .. http://www.apache.org/licenses/LICENSE-2.0 10 | 11 | .. Unless required by applicable law or agreed to in writing, 12 | software distributed under the License is distributed on an 13 | "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | KIND, either express or implied. See the License for the 15 | specific language governing permissions and limitations 16 | under the License. 17 | 18 | OpenMLDB 19 | ========= 20 | 21 | 22 | A OpenMLDB task type's example and dive into information of **PyDolphinScheduler**. 23 | 24 | Example 25 | ------- 26 | 27 | .. literalinclude:: ../../../src/pydolphinscheduler/examples/task_openmldb_example.py 28 | :start-after: [start workflow_declare] 29 | :end-before: [end workflow_declare] 30 | 31 | Dive Into 32 | --------- 33 | 34 | .. automodule:: pydolphinscheduler.tasks.openmldb 35 | 36 | 37 | YAML file example 38 | ----------------- 39 | 40 | .. literalinclude:: ../../../examples/yaml_define/OpenMLDB.yaml 41 | :start-after: # under the License. 42 | :language: yaml 43 | -------------------------------------------------------------------------------- /docs/source/tasks/procedure.rst: -------------------------------------------------------------------------------- 1 | .. Licensed to the Apache Software Foundation (ASF) under one 2 | or more contributor license agreements. See the NOTICE file 3 | distributed with this work for additional information 4 | regarding copyright ownership. The ASF licenses this file 5 | to you under the Apache License, Version 2.0 (the 6 | "License"); you may not use this file except in compliance 7 | with the License. You may obtain a copy of the License at 8 | 9 | .. http://www.apache.org/licenses/LICENSE-2.0 10 | 11 | .. Unless required by applicable law or agreed to in writing, 12 | software distributed under the License is distributed on an 13 | "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | KIND, either express or implied. See the License for the 15 | specific language governing permissions and limitations 16 | under the License. 17 | 18 | Procedure 19 | ========= 20 | 21 | .. automodule:: pydolphinscheduler.tasks.procedure 22 | 23 | 24 | YAML file example 25 | ----------------- 26 | 27 | .. literalinclude:: ../../../examples/yaml_define/Procedure.yaml 28 | :start-after: # under the License. 29 | :language: yaml 30 | -------------------------------------------------------------------------------- /docs/source/tasks/python.rst: -------------------------------------------------------------------------------- 1 | .. Licensed to the Apache Software Foundation (ASF) under one 2 | or more contributor license agreements. See the NOTICE file 3 | distributed with this work for additional information 4 | regarding copyright ownership. The ASF licenses this file 5 | to you under the Apache License, Version 2.0 (the 6 | "License"); you may not use this file except in compliance 7 | with the License. You may obtain a copy of the License at 8 | 9 | .. http://www.apache.org/licenses/LICENSE-2.0 10 | 11 | .. Unless required by applicable law or agreed to in writing, 12 | software distributed under the License is distributed on an 13 | "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | KIND, either express or implied. See the License for the 15 | specific language governing permissions and limitations 16 | under the License. 17 | 18 | Python 19 | ====== 20 | 21 | A Python task type's example and dive into information of **PyDolphinScheduler**. 22 | 23 | Example 24 | ------- 25 | 26 | .. literalinclude:: ../../../src/pydolphinscheduler/examples/task_python_example.py 27 | :start-after: [start workflow_declare] 28 | :end-before: [end workflow_declare] 29 | 30 | Resource Limit Example 31 | ---------------------- 32 | 33 | We can add resource limit like CPU quota and max memory by passing parameters when declaring tasks. 34 | 35 | .. literalinclude:: ../../../src/pydolphinscheduler/examples/task_python_example.py 36 | :start-after: [start resource_limit] 37 | :end-before: [end resource_limit] 38 | 39 | Dive Into 40 | --------- 41 | 42 | .. automodule:: pydolphinscheduler.tasks.python 43 | 44 | 45 | YAML file example 46 | ----------------- 47 | 48 | .. literalinclude:: ../../../examples/yaml_define/Python.yaml 49 | :start-after: # under the License. 50 | :language: yaml 51 | -------------------------------------------------------------------------------- /docs/source/tasks/pytorch.rst: -------------------------------------------------------------------------------- 1 | .. Licensed to the Apache Software Foundation (ASF) under one 2 | or more contributor license agreements. See the NOTICE file 3 | distributed with this work for additional information 4 | regarding copyright ownership. The ASF licenses this file 5 | to you under the Apache License, Version 2.0 (the 6 | "License"); you may not use this file except in compliance 7 | with the License. You may obtain a copy of the License at 8 | 9 | .. http://www.apache.org/licenses/LICENSE-2.0 10 | 11 | .. Unless required by applicable law or agreed to in writing, 12 | software distributed under the License is distributed on an 13 | "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | KIND, either express or implied. See the License for the 15 | specific language governing permissions and limitations 16 | under the License. 17 | 18 | Pytorch 19 | ======= 20 | 21 | 22 | A Pytorch task type's example and dive into information of **PyDolphinScheduler**. 23 | 24 | Example 25 | ------- 26 | 27 | .. literalinclude:: ../../../src/pydolphinscheduler/examples/task_pytorch_example.py 28 | :start-after: [start workflow_declare] 29 | :end-before: [end workflow_declare] 30 | 31 | 32 | Resource Limit Example 33 | ---------------------- 34 | 35 | We can add resource limit like CPU quota and max memory by passing parameters when declaring tasks. 36 | 37 | .. literalinclude:: ../../../src/pydolphinscheduler/examples/task_pytorch_example.py 38 | :start-after: [start resource_limit] 39 | :end-before: [end resource_limit] 40 | 41 | 42 | Dive Into 43 | --------- 44 | 45 | .. automodule:: pydolphinscheduler.tasks.pytorch 46 | 47 | 48 | YAML file example 49 | ----------------- 50 | 51 | .. literalinclude:: ../../../examples/yaml_define/Pytorch.yaml 52 | :start-after: # under the License. 53 | :language: yaml 54 | -------------------------------------------------------------------------------- /docs/source/tasks/sagemaker.rst: -------------------------------------------------------------------------------- 1 | .. Licensed to the Apache Software Foundation (ASF) under one 2 | or more contributor license agreements. See the NOTICE file 3 | distributed with this work for additional information 4 | regarding copyright ownership. The ASF licenses this file 5 | to you under the Apache License, Version 2.0 (the 6 | "License"); you may not use this file except in compliance 7 | with the License. You may obtain a copy of the License at 8 | 9 | .. http://www.apache.org/licenses/LICENSE-2.0 10 | 11 | .. Unless required by applicable law or agreed to in writing, 12 | software distributed under the License is distributed on an 13 | "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | KIND, either express or implied. See the License for the 15 | specific language governing permissions and limitations 16 | under the License. 17 | 18 | SageMaker 19 | ========= 20 | 21 | 22 | A SageMaker task type's example and dive into information of **PyDolphinScheduler**. 23 | 24 | Example 25 | ------- 26 | 27 | .. literalinclude:: ../../../src/pydolphinscheduler/examples/task_sagemaker_example.py 28 | :start-after: [start workflow_declare] 29 | :end-before: [end workflow_declare] 30 | 31 | Dive Into 32 | --------- 33 | 34 | .. automodule:: pydolphinscheduler.tasks.sagemaker 35 | 36 | YAML file example 37 | ----------------- 38 | 39 | .. literalinclude:: ../../../examples/yaml_define/Sagemaker.yaml 40 | :start-after: # under the License. 41 | :language: yaml 42 | 43 | example_sagemaker_params.json: 44 | 45 | .. literalinclude:: ../../../examples/yaml_define/example_sagemaker_params.json 46 | :language: json 47 | -------------------------------------------------------------------------------- /docs/source/tasks/shell.rst: -------------------------------------------------------------------------------- 1 | .. Licensed to the Apache Software Foundation (ASF) under one 2 | or more contributor license agreements. See the NOTICE file 3 | distributed with this work for additional information 4 | regarding copyright ownership. The ASF licenses this file 5 | to you under the Apache License, Version 2.0 (the 6 | "License"); you may not use this file except in compliance 7 | with the License. You may obtain a copy of the License at 8 | 9 | .. http://www.apache.org/licenses/LICENSE-2.0 10 | 11 | .. Unless required by applicable law or agreed to in writing, 12 | software distributed under the License is distributed on an 13 | "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | KIND, either express or implied. See the License for the 15 | specific language governing permissions and limitations 16 | under the License. 17 | 18 | Shell 19 | ===== 20 | 21 | A shell task type's example and dive into information of **PyDolphinScheduler**. 22 | 23 | Example 24 | ------- 25 | 26 | .. literalinclude:: ../../../src/pydolphinscheduler/examples/tutorial.py 27 | :start-after: [start workflow_declare] 28 | :end-before: [end task_relation_declare] 29 | 30 | Resource Limit Example 31 | ---------------------- 32 | 33 | We can add resource limit like CPU quota and max memory by passing parameters when declaring tasks. 34 | 35 | .. literalinclude:: ../../../src/pydolphinscheduler/examples/tutorial.py 36 | :start-after: [start resource_limit] 37 | :end-before: [end resource_limit] 38 | 39 | Dive Into 40 | --------- 41 | 42 | .. automodule:: pydolphinscheduler.tasks.shell 43 | 44 | 45 | YAML file example 46 | ----------------- 47 | 48 | .. literalinclude:: ../../../examples/yaml_define/Shell.yaml 49 | :start-after: # under the License. 50 | :language: yaml 51 | -------------------------------------------------------------------------------- /docs/source/tasks/spark.rst: -------------------------------------------------------------------------------- 1 | .. Licensed to the Apache Software Foundation (ASF) under one 2 | or more contributor license agreements. See the NOTICE file 3 | distributed with this work for additional information 4 | regarding copyright ownership. The ASF licenses this file 5 | to you under the Apache License, Version 2.0 (the 6 | "License"); you may not use this file except in compliance 7 | with the License. You may obtain a copy of the License at 8 | 9 | .. http://www.apache.org/licenses/LICENSE-2.0 10 | 11 | .. Unless required by applicable law or agreed to in writing, 12 | software distributed under the License is distributed on an 13 | "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | KIND, either express or implied. See the License for the 15 | specific language governing permissions and limitations 16 | under the License. 17 | 18 | Spark 19 | ===== 20 | 21 | A spark task type's example and dive into information of **PyDolphinScheduler**. 22 | 23 | Example 24 | ------- 25 | 26 | .. literalinclude:: ../../../src/pydolphinscheduler/examples/task_spark_example.py 27 | :start-after: [start workflow_declare] 28 | :end-before: [end workflow_declare] 29 | 30 | Dive Into 31 | --------- 32 | 33 | .. automodule:: pydolphinscheduler.tasks.spark 34 | 35 | 36 | YAML file example 37 | ----------------- 38 | 39 | .. literalinclude:: ../../../examples/yaml_define/Spark.yaml 40 | :start-after: # under the License. 41 | :language: yaml 42 | -------------------------------------------------------------------------------- /docs/source/tasks/switch.rst: -------------------------------------------------------------------------------- 1 | .. Licensed to the Apache Software Foundation (ASF) under one 2 | or more contributor license agreements. See the NOTICE file 3 | distributed with this work for additional information 4 | regarding copyright ownership. The ASF licenses this file 5 | to you under the Apache License, Version 2.0 (the 6 | "License"); you may not use this file except in compliance 7 | with the License. You may obtain a copy of the License at 8 | 9 | .. http://www.apache.org/licenses/LICENSE-2.0 10 | 11 | .. Unless required by applicable law or agreed to in writing, 12 | software distributed under the License is distributed on an 13 | "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | KIND, either express or implied. See the License for the 15 | specific language governing permissions and limitations 16 | under the License. 17 | 18 | Switch 19 | ====== 20 | 21 | A switch task type's example and dive into information of **PyDolphinScheduler**. 22 | 23 | Example 24 | ------- 25 | 26 | .. literalinclude:: ../../../src/pydolphinscheduler/examples/task_switch_example.py 27 | :start-after: [start workflow_declare] 28 | :end-before: [end workflow_declare] 29 | 30 | Dive Into 31 | --------- 32 | 33 | .. automodule:: pydolphinscheduler.tasks.switch 34 | 35 | 36 | YAML file example 37 | ----------------- 38 | 39 | .. literalinclude:: ../../../examples/yaml_define/Switch.yaml 40 | :start-after: # under the License. 41 | :language: yaml 42 | 43 | -------------------------------------------------------------------------------- /examples/yaml_define/Condition.yaml: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | # Define the workflow 19 | workflow: 20 | name: "Condition" 21 | 22 | # Define the tasks within the workflow 23 | tasks: 24 | - { "task_type": "Shell", "name": "pre_task_1", "command": "echo pre_task_1" } 25 | - { "task_type": "Shell", "name": "pre_task_2", "command": "echo pre_task_2" } 26 | - { "task_type": "Shell", "name": "pre_task_3", "command": "echo pre_task_3" } 27 | - { "task_type": "Shell", "name": "success_branch", "command": "echo success_branch" } 28 | - { "task_type": "Shell", "name": "fail_branch", "command": "echo fail_branch" } 29 | 30 | - name: condition 31 | task_type: Condition 32 | success_task: success_branch 33 | failed_task: fail_branch 34 | op: AND 35 | groups: 36 | - op: AND 37 | groups: 38 | - task: pre_task_1 39 | flag: true 40 | - task: pre_task_2 41 | flag: true 42 | - task: pre_task_3 43 | flag: false 44 | -------------------------------------------------------------------------------- /examples/yaml_define/DataX.yaml: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | # Define the workflow 19 | workflow: 20 | name: "DataX" 21 | 22 | # Define the tasks within the workflow 23 | tasks: 24 | - name: task 25 | task_type: DataX 26 | datasource_name: db 27 | datatarget_name: db 28 | sql: show tables; 29 | target_table: table_test 30 | 31 | - name: task_custon_config 32 | task_type: CustomDataX 33 | json: $FILE{"example_datax.json"} 34 | -------------------------------------------------------------------------------- /examples/yaml_define/Dependent_External.yaml: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | # Define the workflow 19 | workflow: 20 | name: "task_dependent_external" 21 | 22 | # Define the tasks within the workflow 23 | tasks: 24 | - { "task_type": "Shell", "name": "task_1", "command": "echo task 1" } 25 | - { "task_type": "Shell", "name": "task_2", "command": "echo task 2" } 26 | - { "task_type": "Shell", "name": "task_3", "command": "echo task 3" } 27 | -------------------------------------------------------------------------------- /examples/yaml_define/Dvc.yaml: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | # Define variable `repository` 19 | repository: &repository "git@github.com:/dvc-data-repository-example.git" 20 | 21 | # Define the workflow 22 | workflow: 23 | name: "DVC" 24 | release_state: "offline" 25 | 26 | # Define the tasks within the workflow 27 | tasks: 28 | - name: init_dvc 29 | task_type: DVCInit 30 | repository: *repository 31 | store_url: ~/dvc_data 32 | 33 | - name: upload_data 34 | task_type: DVCUpload 35 | repository: *repository 36 | data_path_in_dvc_repository: "iris" 37 | data_path_in_worker: ~/source/iris 38 | version: v1 39 | message: upload iris data v1 40 | 41 | - name: download_data 42 | task_type: DVCDownload 43 | repository: *repository 44 | data_path_in_dvc_repository: "iris" 45 | data_path_in_worker: ~/target/iris 46 | version: v1 47 | -------------------------------------------------------------------------------- /examples/yaml_define/Flink.yaml: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | # Define the workflow 19 | workflow: 20 | name: "Flink" 21 | 22 | # Define the tasks within the workflow 23 | tasks: 24 | - name: task 25 | task_type: Flink 26 | main_class: org.apache.flink.streaming.examples.wordcount.WordCount 27 | main_package: test_java.jar 28 | program_type: JAVA 29 | deploy_mode: local 30 | -------------------------------------------------------------------------------- /examples/yaml_define/Http.yaml: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | # Define the workflow 19 | workflow: 20 | name: "Http" 21 | 22 | # Define the tasks within the workflow 23 | tasks: 24 | - name: task 25 | task_type: Http 26 | url: "https://httpbin.org/get" 27 | http_method: "GET" 28 | http_params: 29 | - { "prop": "a", "httpParametersType": "PARAMETER", "value": "1" } 30 | - { "prop": "b", "httpParametersType": "PARAMETER", "value": "2" } 31 | - { 32 | "prop": "Content-Type", 33 | "httpParametersType": "header", 34 | "value": "test", 35 | } 36 | http_check_condition: "STATUS_CODE_CUSTOM" 37 | condition: "404" 38 | -------------------------------------------------------------------------------- /examples/yaml_define/Kubernetes.yaml: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | # Define the workflow 19 | workflow: 20 | name: "kubernetes" 21 | 22 | # Define the tasks within the workflow 23 | tasks: 24 | - name: kubernetes 25 | task_type: K8S 26 | image: ds-dev 27 | namespace: '{ "name": "default","cluster": "lab" }' 28 | minCpuCores: 2.0 29 | minMemorySpace: 10.0 -------------------------------------------------------------------------------- /examples/yaml_define/MapReduce.yaml: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | # Define the workflow 19 | workflow: 20 | name: "MapReduce" 21 | 22 | # Define the tasks within the workflow 23 | tasks: 24 | - name: task 25 | task_type: MR 26 | main_class: wordcount 27 | main_package: test_java.jar 28 | program_type: SCALA 29 | main_args: /dolphinscheduler/tenant_exists/resources/file.txt /output/ds 30 | -------------------------------------------------------------------------------- /examples/yaml_define/MoreConfiguration.yaml: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | # Define the workflow 19 | workflow: 20 | name: "MoreConfiguration" 21 | param: 22 | n: 1 23 | 24 | # Define the tasks within the workflow 25 | tasks: 26 | - name: shell_0 27 | task_type: Shell 28 | description: "yaml define task" 29 | flag: "YES" 30 | command: | 31 | echo "$ENV{HOME}" 32 | echo "${n}" 33 | echo "123" >> text.txt 34 | task_priority: "HIGH" 35 | delay_time: 20 36 | fail_retry_times: 30 37 | fail_retry_interval: 5 38 | timeout: 60 39 | is_cache: true 40 | input_params: 41 | value_VARCHAR: "abc" 42 | value_INTEGER: 123 43 | value_FLOAT: 0.1 44 | value_BOOLEAN: True 45 | output_params: 46 | value_output: "" 47 | value_output2: VARCHAR() 48 | value_data_path: FILE(text.txt) 49 | -------------------------------------------------------------------------------- /examples/yaml_define/OpenMLDB.yaml: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | # Define the workflow 19 | workflow: 20 | name: "OpenMLDB" 21 | 22 | # Define the tasks within the workflow 23 | tasks: 24 | - name: OpenMLDB 25 | task_type: OpenMLDB 26 | zookeeper: "127.0.0.1:2181" 27 | zookeeper_path: "/openmldb" 28 | execute_mode: "online" 29 | sql: | 30 | USE demo_db; 31 | set @@job_timeout=200000; 32 | LOAD DATA INFILE 'file:///tmp/train_sample.csv' 33 | INTO TABLE talkingdata OPTIONS(mode='overwrite'); 34 | -------------------------------------------------------------------------------- /examples/yaml_define/Procedure.yaml: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | # Define the workflow 19 | workflow: 20 | name: "Procedure" 21 | 22 | # Define the tasks within the workflow 23 | tasks: 24 | - name: task 25 | task_type: Procedure 26 | datasource_name: db 27 | method: show tables; 28 | -------------------------------------------------------------------------------- /examples/yaml_define/Python.yaml: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | # Define the workflow 19 | workflow: 20 | name: "Python" 21 | 22 | # Define the tasks within the workflow 23 | tasks: 24 | - name: python 25 | task_type: Python 26 | definition: | 27 | import os 28 | print(os) 29 | print("1") 30 | print("2") 31 | -------------------------------------------------------------------------------- /examples/yaml_define/Pytorch.yaml: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | # Define the workflow 19 | workflow: 20 | name: "Pytorch" 21 | 22 | # Define the tasks within the workflow 23 | tasks: 24 | 25 | # run project with existing environment 26 | - name: task_existing_env 27 | task_type: pytorch 28 | script: main.py 29 | script_params: --dry-run --no-cuda 30 | project_path: https://github.com/pytorch/examples#mnist 31 | python_command: /home/anaconda3/envs/pytorch/bin/python3 32 | 33 | 34 | # run project with creating conda environment 35 | - name: task_conda_env 36 | task_type: pytorch 37 | script: main.py 38 | script_params: --dry-run --no-cuda 39 | project_path: https://github.com/pytorch/examples#mnist 40 | is_create_environment: True 41 | python_env_tool: conda 42 | requirements: requirements.txt 43 | conda_python_version: 3.7 44 | 45 | # run project with creating virtualenv environment 46 | - name: task_virtualenv_env 47 | task_type: pytorch 48 | script: main.py 49 | script_params: --dry-run --no-cuda 50 | project_path: https://github.com/pytorch/examples#mnist 51 | is_create_environment: True 52 | python_env_tool: virtualenv 53 | requirements: requirements.txt 54 | -------------------------------------------------------------------------------- /examples/yaml_define/Sagemaker.yaml: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | # Define the workflow 19 | workflow: 20 | name: "Sagemaker" 21 | release_state: "offline" 22 | 23 | # Define the tasks within the workflow 24 | tasks: 25 | - name: sagemaker 26 | task_type: Sagemaker 27 | sagemaker_request_json: $FILE{"example_sagemaker_params.json"} 28 | 29 | -------------------------------------------------------------------------------- /examples/yaml_define/Shell.yaml: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | # Define the workflow 19 | workflow: 20 | name: "Shell" 21 | release_state: "offline" 22 | run: true 23 | 24 | # Define the tasks within the workflow 25 | tasks: 26 | - name: task_parent 27 | task_type: Shell 28 | command: | 29 | echo hello pydolphinscheduler 30 | echo run task parent 31 | 32 | - name: task_child_one 33 | task_type: Shell 34 | deps: [task_parent] 35 | command: echo "child one" 36 | 37 | - name: task_child_two 38 | task_type: Shell 39 | deps: [task_parent] 40 | command: echo "child two" 41 | -------------------------------------------------------------------------------- /examples/yaml_define/Spark.yaml: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | # Define the workflow 19 | workflow: 20 | name: "Spark" 21 | 22 | # Define the tasks within the workflow 23 | tasks: 24 | - name: task 25 | task_type: Spark 26 | main_class: org.apache.spark.examples.SparkPi 27 | main_package: test_java.jar 28 | program_type: SCALA 29 | deploy_mode: local 30 | -------------------------------------------------------------------------------- /examples/yaml_define/Sql.yaml: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | # Define the workflow 19 | workflow: 20 | name: "Sql" 21 | 22 | # Define the tasks within the workflow 23 | tasks: 24 | - name: task_base 25 | task_type: Sql 26 | datasource_name: "db" 27 | sql: show tables; 28 | 29 | - name: task_multi_line 30 | task_type: Sql 31 | datasource_name: "db" 32 | sql: | 33 | show tables; 34 | select id from version where id=1; 35 | 36 | - name: task_file 37 | task_type: Sql 38 | datasource_name: "db" 39 | sql: $FILE{"example_sql.sql"} 40 | 41 | # Or you can define task "task_union" it with one line 42 | - { "task_type": "Sql", "name": "task_base_one_line", "datasource_name": "db", "sql": "select id from version where id=1;"} 43 | 44 | # Or you can define task "task_union" it with one line 45 | - { "task_type": "Sql", "name": "task_file_one_line", "datasource_name": "db", "sql": '$FILE{"example_sql.sql"}'} 46 | -------------------------------------------------------------------------------- /examples/yaml_define/SubWorkflow.yaml: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | # Define the workflow 19 | workflow: 20 | name: "SubWorkflow" 21 | 22 | tasks: 23 | - name: example_workflow 24 | task_type: SubWorkflow 25 | workflow_name: $WORKFLOW{"example_sub_workflow.yaml"} 26 | 27 | - { "task_type": "Shell", "deps": [example_workflow], "name": "task_3", "command": "echo task 3" } 28 | -------------------------------------------------------------------------------- /examples/yaml_define/Switch.yaml: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | # Define the workflow 19 | workflow: 20 | name: "Switch" 21 | param: 22 | var: 1 23 | 24 | # Define the tasks within the workflow 25 | tasks: 26 | - name: switch_child_1 27 | task_type: Shell 28 | command: echo switch_child_1 29 | 30 | - name: switch_child_2 31 | task_type: Shell 32 | command: echo switch_child_2 33 | 34 | - name: switch 35 | task_type: Switch 36 | condition: 37 | - task: switch_child_1 38 | condition: "${var} > 1" 39 | - task: switch_child_2 40 | -------------------------------------------------------------------------------- /examples/yaml_define/example_datax.json: -------------------------------------------------------------------------------- 1 | { 2 | "job": { 3 | "content": [ 4 | { 5 | "reader": { 6 | "name": "mysqlreader", 7 | "parameter": { 8 | "username": "usr", 9 | "password": "pwd", 10 | "column": [ 11 | "id", 12 | "name", 13 | "code", 14 | "description" 15 | ], 16 | "splitPk": "id", 17 | "connection": [ 18 | { 19 | "table": [ 20 | "source_table" 21 | ], 22 | "jdbcUrl": [ 23 | "jdbc:mysql://127.0.0.1:3306/source_db" 24 | ] 25 | } 26 | ] 27 | } 28 | }, 29 | "writer": { 30 | "name": "mysqlwriter", 31 | "parameter": { 32 | "writeMode": "insert", 33 | "username": "usr", 34 | "password": "pwd", 35 | "column": [ 36 | "id", 37 | "name" 38 | ], 39 | "connection": [ 40 | { 41 | "jdbcUrl": "jdbc:mysql://127.0.0.1:3306/target_db", 42 | "table": [ 43 | "target_table" 44 | ] 45 | } 46 | ] 47 | } 48 | } 49 | } 50 | ], 51 | "setting": { 52 | "errorLimit": { 53 | "percentage": 0, 54 | "record": 0 55 | }, 56 | "speed": { 57 | "channel": 1, 58 | "record": 1000 59 | } 60 | } 61 | } 62 | } 63 | -------------------------------------------------------------------------------- /examples/yaml_define/example_sagemaker_params.json: -------------------------------------------------------------------------------- 1 | { 2 | "ParallelismConfiguration":{ 3 | "MaxParallelExecutionSteps":1 4 | }, 5 | "PipelineExecutionDescription":"run pipeline using ds", 6 | "PipelineExecutionDisplayName":"ds-sagemaker-pipeline", 7 | "PipelineName":"DsSagemakerPipeline", 8 | "PipelineParameters":[ 9 | { 10 | "Name":"InputData", 11 | "Value": "s3://sagemaker/dataset/dataset.csv" 12 | }, 13 | { 14 | "Name":"InferenceData", 15 | "Value": "s3://sagemaker/dataset/inference.csv" 16 | } 17 | ] 18 | } 19 | -------------------------------------------------------------------------------- /examples/yaml_define/example_sql.sql: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | select id from version where id=1; 19 | select id from version where id=2; 20 | select id from version where id=3; 21 | select id from version where id=4; 22 | select id from version where id=5; 23 | -------------------------------------------------------------------------------- /examples/yaml_define/example_sub_workflow.yaml: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | # Define the workflow 19 | workflow: 20 | name: "example_workflow_for_sub_workflow" 21 | 22 | # Define the tasks within the workflow 23 | tasks: 24 | - { "task_type": "Shell", "name": "task_1", "command": "echo task 1" } 25 | - { "task_type": "Shell", "deps": [task_1], "name": "task_2", "command": "echo task 2" } 26 | - { "task_type": "Shell", "deps": [task_2], "name": "task_3", "command": "echo task 3" } 27 | -------------------------------------------------------------------------------- /examples/yaml_define/tutorial.yaml: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | # Define the workflow 19 | workflow: 20 | name: "tutorial" 21 | schedule: "0 0 0 * * ? *" 22 | start_time: "2021-01-01" 23 | release_state: "offline" 24 | run: true 25 | 26 | # Define the tasks within the workflow 27 | tasks: 28 | - name: task_parent 29 | task_type: Shell 30 | command: echo hello pydolphinscheduler 31 | 32 | - name: task_child_one 33 | task_type: Shell 34 | deps: [task_parent] 35 | command: echo "child one" 36 | 37 | - name: task_child_two 38 | task_type: Shell 39 | deps: [task_parent] 40 | command: echo "child two" 41 | 42 | - name: task_union 43 | task_type: Shell 44 | deps: [task_child_one, task_child_two] 45 | command: echo "union" 46 | -------------------------------------------------------------------------------- /src/pydolphinscheduler/__init__.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | """Init root of pydolphinscheduler.""" 19 | 20 | __version__ = "4.1.0-dev" 21 | -------------------------------------------------------------------------------- /src/pydolphinscheduler/cli/__init__.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | """Commands line interface of pydolphinscheduler.""" 19 | -------------------------------------------------------------------------------- /src/pydolphinscheduler/core/__init__.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | """Init pydolphinscheduler.core package.""" 19 | 20 | from pydolphinscheduler.core.engine import Engine 21 | from pydolphinscheduler.core.task import Task 22 | from pydolphinscheduler.core.workflow import Workflow 23 | 24 | __all__ = [ 25 | "Engine", 26 | "Workflow", 27 | "Task", 28 | ] 29 | -------------------------------------------------------------------------------- /src/pydolphinscheduler/core/mixin.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | """WorkerResource Mixin.""" 19 | 20 | 21 | class WorkerResourceMixin: 22 | """Mixin object, declare some attributes for WorkerResource.""" 23 | 24 | def add_attr(self, **kwargs): 25 | """Add attributes to WorkerResource, include cpu_quota and memory_max now.""" 26 | self._cpu_quota = kwargs.get("cpu_quota", -1) 27 | self._memory_max = kwargs.get("memory_max", -1) 28 | if hasattr(self, "_DEFINE_ATTR"): 29 | self._DEFINE_ATTR |= {"cpu_quota", "memory_max"} 30 | 31 | @property 32 | def cpu_quota(self): 33 | """Get cpu_quota.""" 34 | return self._cpu_quota 35 | 36 | @property 37 | def memory_max(self): 38 | """Get memory_max.""" 39 | return self._memory_max 40 | -------------------------------------------------------------------------------- /src/pydolphinscheduler/core/process_definition.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | """This module is deprecated. Please use `pydolphinscheduler.core.workflow.Workflow`.""" 19 | 20 | import warnings 21 | 22 | from pydolphinscheduler.core.workflow import Workflow as ProcessDefinition # noqa 23 | 24 | warnings.warn( 25 | "This module is deprecated and will be remove in 4.1.0. Please use" 26 | "`pydolphinscheduler.core.workflow.Workflow` instead.", 27 | DeprecationWarning, 28 | stacklevel=2, 29 | ) 30 | -------------------------------------------------------------------------------- /src/pydolphinscheduler/core/resource_plugin.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | """DolphinScheduler ResourcePlugin object.""" 19 | 20 | from abc import ABCMeta, abstractmethod 21 | 22 | from pydolphinscheduler.exceptions import PyResPluginException 23 | 24 | 25 | # [start resource_plugin_definition] 26 | class ResourcePlugin(metaclass=ABCMeta): 27 | """ResourcePlugin object, declare resource plugin for task and workflow to dolphinscheduler. 28 | 29 | :param prefix: A string representing the prefix of ResourcePlugin. 30 | 31 | """ 32 | 33 | # [start init_method] 34 | def __init__(self, prefix: str, *args, **kwargs): 35 | self.prefix = prefix 36 | 37 | # [end init_method] 38 | 39 | # [start abstractmethod read_file] 40 | @abstractmethod 41 | def read_file(self, suf: str): 42 | """Get the content of the file. 43 | 44 | The address of the file is the prefix of the resource plugin plus the parameter suf. 45 | """ 46 | 47 | # [end abstractmethod read_file] 48 | 49 | def get_index(self, s: str, x, n): 50 | """Find the subscript of the nth occurrence of the X character in the string s.""" 51 | if n <= s.count(x): 52 | all_index = [key for key, value in enumerate(s) if value == x] 53 | return all_index[n - 1] 54 | else: 55 | raise PyResPluginException("Incomplete path.") 56 | 57 | 58 | # [end resource_plugin_definition] 59 | -------------------------------------------------------------------------------- /src/pydolphinscheduler/examples/__init__.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | """Init examples package which provides users with pydolphinscheduler examples.""" 19 | -------------------------------------------------------------------------------- /src/pydolphinscheduler/examples/bulk_create_example.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | """ 19 | This example show you how to create workflows in batch mode. 20 | 21 | After this example run, we will create 10 workflows named `workflow:`, and with 3 tasks 22 | named `task:-workflow:` in each workflow. Task shape as below 23 | 24 | task:1-workflow:1 -> task:2-workflow:1 -> task:3-workflow:1 25 | 26 | Each workflow is linear since we set `IS_CHAIN=True`, you could change task to parallel by set it to `False`. 27 | """ 28 | 29 | from pydolphinscheduler.core.workflow import Workflow 30 | from pydolphinscheduler.tasks.shell import Shell 31 | 32 | NUM_WORKFLOWS = 10 33 | NUM_TASKS = 5 34 | # Whether task should dependent on pre one or not 35 | # False will create workflow with independent task, while True task will dependent on pre-task and dependence 36 | # link like `pre_task -> current_task -> next_task`, default True 37 | IS_CHAIN = True 38 | 39 | for wf in range(0, NUM_WORKFLOWS): 40 | workflow_name = f"workflow:{wf}" 41 | 42 | with Workflow(name=workflow_name) as workflow: 43 | for t in range(0, NUM_TASKS): 44 | task_name = f"task:{t}-{workflow_name}" 45 | command = f"echo This is task {task_name}" 46 | task = Shell(name=task_name, command=command) 47 | 48 | if IS_CHAIN and t > 0: 49 | pre_task_name = f"task:{t-1}-{workflow_name}" 50 | workflow.get_one_task_by_name(pre_task_name) >> task 51 | 52 | # We just submit workflow and task definition without set schedule time or run it manually 53 | workflow.submit() 54 | -------------------------------------------------------------------------------- /src/pydolphinscheduler/examples/ext/example.sql: -------------------------------------------------------------------------------- 1 | select * 2 | from resource_plugin 3 | where id = 1 -------------------------------------------------------------------------------- /src/pydolphinscheduler/examples/task_condition_example.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | # [start workflow_declare] 19 | r""" 20 | A example workflow for task condition. 21 | 22 | This example will create five task in single workflow, with four shell task and one condition task. Task 23 | condition have one upstream which we declare explicit with syntax `parent >> condition`, and three downstream 24 | automatically set dependence by condition task by passing parameter `condition`. The graph of this workflow 25 | like: 26 | pre_task_1 -> -> success_branch 27 | \ / 28 | pre_task_2 -> -> conditions -> 29 | / \ 30 | pre_task_3 -> -> fail_branch 31 | . 32 | """ 33 | 34 | from pydolphinscheduler.core.workflow import Workflow 35 | from pydolphinscheduler.tasks.condition import FAILURE, SUCCESS, And, Condition 36 | from pydolphinscheduler.tasks.shell import Shell 37 | 38 | with Workflow(name="task_condition_example") as workflow: 39 | pre_task_1 = Shell(name="pre_task_1", command="echo pre_task_1") 40 | pre_task_2 = Shell(name="pre_task_2", command="echo pre_task_2") 41 | pre_task_3 = Shell(name="pre_task_3", command="echo pre_task_3") 42 | cond_operator = And( 43 | And( 44 | SUCCESS(pre_task_1, pre_task_2), 45 | FAILURE(pre_task_3), 46 | ), 47 | ) 48 | 49 | success_branch = Shell(name="success_branch", command="echo success_branch") 50 | fail_branch = Shell(name="fail_branch", command="echo fail_branch") 51 | 52 | condition = Condition( 53 | name="condition", 54 | condition=cond_operator, 55 | success_task=success_branch, 56 | failed_task=fail_branch, 57 | ) 58 | workflow.submit() 59 | # [end workflow_declare] 60 | -------------------------------------------------------------------------------- /src/pydolphinscheduler/examples/task_dvc_example.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | # [start workflow_declare] 19 | """A example workflow for task dvc.""" 20 | 21 | from pydolphinscheduler.core.workflow import Workflow 22 | from pydolphinscheduler.tasks import DVCDownload, DVCInit, DVCUpload 23 | 24 | repository = "git@github.com:/dvc-data-repository-example.git" 25 | 26 | with Workflow( 27 | name="task_dvc_example", 28 | ) as workflow: 29 | init_task = DVCInit(name="init_dvc", repository=repository, store_url="~/dvc_data") 30 | upload_task = DVCUpload( 31 | name="upload_data", 32 | repository=repository, 33 | data_path_in_dvc_repository="iris", 34 | data_path_in_worker="~/source/iris", 35 | version="v1", 36 | message="upload iris data v1", 37 | ) 38 | 39 | download_task = DVCDownload( 40 | name="download_data", 41 | repository=repository, 42 | data_path_in_dvc_repository="iris", 43 | data_path_in_worker="~/target/iris", 44 | version="v1", 45 | ) 46 | 47 | init_task >> upload_task >> download_task 48 | 49 | workflow.run() 50 | 51 | # [end workflow_declare] 52 | -------------------------------------------------------------------------------- /src/pydolphinscheduler/examples/task_flink_example.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | # [start workflow_declare] 19 | """A example workflow for task flink.""" 20 | 21 | from pydolphinscheduler.core.workflow import Workflow 22 | from pydolphinscheduler.tasks.flink import DeployMode, Flink, ProgramType 23 | 24 | with Workflow(name="task_flink_example") as workflow: 25 | task = Flink( 26 | name="task_flink", 27 | main_class="org.apache.flink.streaming.examples.wordcount.WordCount", 28 | main_package="WordCount.jar", 29 | program_type=ProgramType.JAVA, 30 | deploy_mode=DeployMode.LOCAL, 31 | ) 32 | workflow.run() 33 | # [end workflow_declare] 34 | -------------------------------------------------------------------------------- /src/pydolphinscheduler/examples/task_kubernetes_example.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | # [start workflow_declare] 19 | """A example workflow for task kubernetes.""" 20 | 21 | from pydolphinscheduler.core.workflow import Workflow 22 | from pydolphinscheduler.tasks.kubernetes import Kubernetes 23 | 24 | with Workflow( 25 | name="task_kubernetes_example", 26 | ) as workflow: 27 | task_k8s = Kubernetes( 28 | name="task_k8s", 29 | image="ds-dev", 30 | namespace=str({"name": "default", "cluster": "lab"}), 31 | min_cpu_cores=2.0, 32 | min_memory_space=10.0, 33 | ) 34 | workflow.submit() 35 | # [end workflow_declare] 36 | -------------------------------------------------------------------------------- /src/pydolphinscheduler/examples/task_map_reduce_example.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | # [start workflow_declare] 19 | """A example workflow for task mr.""" 20 | 21 | from pydolphinscheduler.core.engine import ProgramType 22 | from pydolphinscheduler.core.workflow import Workflow 23 | from pydolphinscheduler.tasks.map_reduce import MR 24 | 25 | with Workflow(name="task_map_reduce_example") as workflow: 26 | task = MR( 27 | name="task_mr", 28 | main_class="wordcount", 29 | main_package="hadoop-mapreduce-examples-3.3.1.jar", 30 | program_type=ProgramType.JAVA, 31 | main_args="/dolphinscheduler/tenant_exists/resources/file.txt /output/ds", 32 | ) 33 | workflow.run() 34 | # [end workflow_declare] 35 | -------------------------------------------------------------------------------- /src/pydolphinscheduler/examples/task_openmldb_example.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | # [start workflow_declare] 19 | """A example workflow for task openmldb.""" 20 | 21 | from pydolphinscheduler.core.workflow import Workflow 22 | from pydolphinscheduler.tasks.openmldb import OpenMLDB 23 | 24 | sql = """USE demo_db; 25 | set @@job_timeout=200000; 26 | LOAD DATA INFILE 'file:///tmp/train_sample.csv' 27 | INTO TABLE talkingdata OPTIONS(mode='overwrite'); 28 | """ 29 | 30 | with Workflow( 31 | name="task_openmldb_example", 32 | ) as workflow: 33 | task_openmldb = OpenMLDB( 34 | name="task_openmldb", 35 | zookeeper="127.0.0.1:2181", 36 | zookeeper_path="/openmldb", 37 | execute_mode="offline", 38 | sql=sql, 39 | ) 40 | 41 | workflow.run() 42 | # [end workflow_declare] 43 | -------------------------------------------------------------------------------- /src/pydolphinscheduler/examples/task_python_example.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | # [start workflow_declare] 19 | """An example workflow for task python.""" 20 | 21 | from pydolphinscheduler.core.workflow import Workflow 22 | from pydolphinscheduler.tasks.python import Python 23 | 24 | with Workflow( 25 | name="task_python_example", 26 | ) as workflow: 27 | task_python = Python( 28 | name="task", 29 | definition="print('hello world.')", 30 | ) 31 | 32 | # [start resource_limit] 33 | python_resources_limit = Python( 34 | name="python_resources_limit", 35 | definition="print('hello world.')", 36 | cpu_quota=1, 37 | memory_max=100, 38 | ) 39 | # [end resource_limit] 40 | 41 | task_python >> python_resources_limit 42 | workflow.submit() 43 | # [end workflow_declare] 44 | -------------------------------------------------------------------------------- /src/pydolphinscheduler/examples/task_sagemaker_example.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | # [start workflow_declare] 19 | """A example workflow for task sagemaker.""" 20 | import json 21 | 22 | from pydolphinscheduler.core.workflow import Workflow 23 | from pydolphinscheduler.tasks.sagemaker import SageMaker 24 | 25 | sagemaker_request_data = { 26 | "ParallelismConfiguration": {"MaxParallelExecutionSteps": 1}, 27 | "PipelineExecutionDescription": "test Pipeline", 28 | "PipelineExecutionDisplayName": "AbalonePipeline", 29 | "PipelineName": "AbalonePipeline", 30 | "PipelineParameters": [ 31 | {"Name": "ProcessingInstanceType", "Value": "ml.m4.xlarge"}, 32 | {"Name": "ProcessingInstanceCount", "Value": "2"}, 33 | ], 34 | } 35 | 36 | with Workflow( 37 | name="task_sagemaker_example", 38 | ) as workflow: 39 | task_sagemaker = SageMaker( 40 | name="task_sagemaker", 41 | sagemaker_request_json=json.dumps(sagemaker_request_data, indent=2), 42 | ) 43 | 44 | workflow.run() 45 | # [end workflow_declare] 46 | -------------------------------------------------------------------------------- /src/pydolphinscheduler/examples/task_spark_example.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | # [start workflow_declare] 19 | """A example workflow for task spark.""" 20 | 21 | from pydolphinscheduler.core.workflow import Workflow 22 | from pydolphinscheduler.tasks.spark import DeployMode, ProgramType, Spark 23 | 24 | with Workflow(name="task_spark_example") as workflow: 25 | task = Spark( 26 | name="task_spark", 27 | main_class="org.apache.spark.examples.SparkPi", 28 | main_package="spark-examples_2.12-3.2.0.jar", 29 | program_type=ProgramType.JAVA, 30 | deploy_mode=DeployMode.LOCAL, 31 | ) 32 | workflow.run() 33 | # [end workflow_declare] 34 | -------------------------------------------------------------------------------- /src/pydolphinscheduler/examples/task_sql_example.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | # [start workflow_declare] 19 | 20 | """A example workflow for task SQL.""" 21 | from pathlib import Path 22 | 23 | from pydolphinscheduler.core.workflow import Workflow 24 | from pydolphinscheduler.resources_plugin import Local 25 | from pydolphinscheduler.tasks.sql import Sql, SqlType 26 | 27 | with Workflow( 28 | name="task_sql_example", 29 | ) as workflow: 30 | # [start bare_sql_desc] 31 | bare_sql = Sql( 32 | name="bare_sql", 33 | datasource_name="metadata", 34 | sql="select * from t_ds_version", 35 | ) 36 | # [end bare_sql_desc] 37 | # [start sql_file_desc] 38 | sql_file = Sql( 39 | name="sql_file", 40 | datasource_name="metadata", 41 | sql="ext/example.sql", 42 | sql_type=SqlType.SELECT, 43 | resource_plugin=Local(prefix=str(Path(__file__).parent)), 44 | ) 45 | # [end sql_file_desc] 46 | # [start sql_with_pre_post_desc] 47 | sql_with_pre_post = Sql( 48 | name="sql_with_pre_post", 49 | datasource_name="metadata", 50 | sql="select * from t_ds_version", 51 | pre_statements=[ 52 | "update table_one set version = '1.3.6'", 53 | "delete from table_two where version = '1.3.6'", 54 | ], 55 | post_statements="update table_one set version = '3.0.0'", 56 | ) 57 | # [end sql_with_pre_post_desc] 58 | 59 | bare_sql >> [ 60 | sql_file, 61 | sql_with_pre_post, 62 | ] 63 | 64 | workflow.submit() 65 | 66 | # [end workflow_declare] 67 | -------------------------------------------------------------------------------- /src/pydolphinscheduler/examples/task_sub_workflow_example.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | """A example workflow for task sub workflow.""" 19 | 20 | # [start tutorial] 21 | # [start package_import] 22 | # Import Workflow object to define your workflow attributes 23 | from pydolphinscheduler.core.workflow import Workflow 24 | 25 | # Import task Shell object cause we would create some shell tasks later 26 | from pydolphinscheduler.tasks.sub_workflow import SubWorkflow 27 | from pydolphinscheduler.tasks.shell import Shell 28 | 29 | 30 | # [start workflow_declare] 31 | # [start sub_workflow_declare] 32 | with Workflow(name="sub_workflow_downstream") as wf_downstream, Workflow( 33 | name="task_sub_workflow_example" 34 | ) as wf_upstream: 35 | sub_workflow_ds_task = Shell( 36 | name="task_sub_workflow", 37 | command="echo 'call sub workflow success!'", 38 | workflow=wf_downstream, 39 | ) 40 | wf_downstream.submit() 41 | # [end sub_workflow_declare] 42 | 43 | sub_workflow_pre = Shell( 44 | name="pre-task", 45 | command="echo 'prefix task for sub workflow'", 46 | workflow=wf_upstream, 47 | ) 48 | # [start sub_workflow_task_declare] 49 | sw_task = SubWorkflow( 50 | name="sub_workflow", 51 | workflow_name=wf_downstream.name, 52 | workflow=wf_upstream, 53 | ) 54 | # [end sub_workflow_task_declare] 55 | sub_workflow_pre >> sw_task 56 | # Please make sure workflow with name `wf_downstream.name` exists when we submit or run sub workflow task 57 | wf_upstream.run() 58 | # [end workflow_declare] 59 | -------------------------------------------------------------------------------- /src/pydolphinscheduler/examples/task_switch_example.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | # [start workflow_declare] 19 | r""" 20 | A example workflow for task switch. 21 | 22 | This example will create four task in single workflow, with three shell task and one switch task. Task switch 23 | have one upstream which we declare explicit with syntax `parent >> switch`, and two downstream automatically 24 | set dependence by switch task by passing parameter `condition`. The graph of this workflow like: 25 | --> switch_child_1 26 | / 27 | parent -> switch -> 28 | \ 29 | --> switch_child_2 30 | . 31 | """ 32 | 33 | from pydolphinscheduler.core.workflow import Workflow 34 | from pydolphinscheduler.tasks.shell import Shell 35 | from pydolphinscheduler.tasks.switch import Branch, Default, Switch, SwitchCondition 36 | 37 | with Workflow(name="task_switch_example", param={"var": "1"}) as workflow: 38 | parent = Shell(name="parent", command="echo parent") 39 | switch_child_1 = Shell(name="switch_child_1", command="echo switch_child_1") 40 | switch_child_2 = Shell(name="switch_child_2", command="echo switch_child_2") 41 | switch_condition = SwitchCondition( 42 | Branch(condition="${var} > 1", task=switch_child_1), 43 | Default(task=switch_child_2), 44 | ) 45 | 46 | switch = Switch(name="switch", condition=switch_condition) 47 | parent >> switch 48 | workflow.submit() 49 | # [end workflow_declare] 50 | -------------------------------------------------------------------------------- /src/pydolphinscheduler/examples/tutorial_resource_plugin.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | r""" 19 | A tutorial example take you to experience pydolphinscheduler resource plugin. 20 | 21 | Resource plug-ins can be defined in workflows and tasks 22 | 23 | it will instantiate and run all the task it have. 24 | """ 25 | import os 26 | from pathlib import Path 27 | 28 | # [start tutorial_resource_plugin] 29 | # [start package_import] 30 | # Import Workflow object to define your workflow attributes 31 | from pydolphinscheduler.core.workflow import Workflow 32 | 33 | # Import task Shell object cause we would create some shell tasks later 34 | from pydolphinscheduler.resources_plugin.local import Local 35 | from pydolphinscheduler.tasks.shell import Shell 36 | 37 | # [end package_import] 38 | 39 | # [start workflow_declare] 40 | with Workflow( 41 | name="tutorial_resource_plugin", 42 | schedule="0 0 0 * * ? *", 43 | start_time="2021-01-01", 44 | resource_plugin=Local("/tmp"), 45 | ) as workflow: 46 | # [end workflow_declare] 47 | # [start task_declare] 48 | file = "resource.sh" 49 | path = Path("/tmp").joinpath(file) 50 | with open(str(path), "w") as f: 51 | f.write("echo tutorial resource plugin") 52 | task_parent = Shell( 53 | name="local-resource-example", 54 | command=file, 55 | ) 56 | print(task_parent.task_params) 57 | os.remove(path) 58 | # [end task_declare] 59 | 60 | # [start submit_or_run] 61 | workflow.run() 62 | # [end submit_or_run] 63 | # [end tutorial_resource_plugin] 64 | -------------------------------------------------------------------------------- /src/pydolphinscheduler/exceptions.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | """Exceptions for pydolphinscheduler.""" 19 | 20 | 21 | class PyDSBaseException(Exception): 22 | """Base exception for pydolphinscheduler.""" 23 | 24 | 25 | class PyDSParamException(PyDSBaseException): 26 | """Exception for pydolphinscheduler parameter verify error.""" 27 | 28 | 29 | class PyDSTaskNoFoundException(PyDSBaseException): 30 | """Exception for pydolphinscheduler workflow task no found error.""" 31 | 32 | 33 | class PyDSJavaGatewayException(PyDSBaseException): 34 | """Exception for pydolphinscheduler Java gateway error.""" 35 | 36 | 37 | class PyDSWorkflowNotAssignException(PyDSBaseException): 38 | """Exception for pydolphinscheduler workflow not assign error.""" 39 | 40 | 41 | class PyDSConfException(PyDSBaseException): 42 | """Exception for pydolphinscheduler configuration error.""" 43 | 44 | 45 | class PyResPluginException(PyDSBaseException): 46 | """Exception for pydolphinscheduler resource plugin error.""" 47 | -------------------------------------------------------------------------------- /src/pydolphinscheduler/models/__init__.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | """Init Models package, keeping object related to DolphinScheduler covert from Java Gateway Service.""" 19 | 20 | from pydolphinscheduler.models.base import Base 21 | from pydolphinscheduler.models.base_side import BaseSide 22 | from pydolphinscheduler.models.project import Project 23 | from pydolphinscheduler.models.queue import Queue 24 | from pydolphinscheduler.models.tenant import Tenant 25 | from pydolphinscheduler.models.user import User 26 | from pydolphinscheduler.models.worker_group import WorkerGroup 27 | 28 | __all__ = [ 29 | "Base", 30 | "BaseSide", 31 | "Project", 32 | "Tenant", 33 | "User", 34 | "Queue", 35 | "WorkerGroup", 36 | ] 37 | -------------------------------------------------------------------------------- /src/pydolphinscheduler/models/base_side.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | """Module for models object.""" 19 | 20 | from __future__ import annotations 21 | 22 | from pydolphinscheduler import configuration 23 | from pydolphinscheduler.models import Base 24 | 25 | 26 | class BaseSide(Base): 27 | """Base class for models object, it declare base behavior for them.""" 28 | 29 | def __init__(self, name: str, description: str | None = None): 30 | super().__init__(name, description) 31 | 32 | @classmethod 33 | def create_if_not_exists( 34 | cls, 35 | # TODO comment for avoiding cycle import 36 | # user: Optional[User] = ProcessDefinitionDefault.USER 37 | user=configuration.WORKFLOW_USER, 38 | ): 39 | """Create Base if not exists.""" 40 | raise NotImplementedError 41 | 42 | def delete_all(self): 43 | """Delete all method.""" 44 | if not self: 45 | return 46 | list_pro = [key for key in self.__dict__.keys()] 47 | for key in list_pro: 48 | self.__delattr__(key) 49 | -------------------------------------------------------------------------------- /src/pydolphinscheduler/models/connection.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | 19 | """DolphinScheduler Connection object. 20 | 21 | Including the basic information of database connection. 22 | """ 23 | 24 | from typing import NamedTuple 25 | 26 | 27 | class Connection(NamedTuple): 28 | """Basic information of database connection.""" 29 | 30 | host: str 31 | port: int 32 | schema: str 33 | username: str 34 | password: str 35 | -------------------------------------------------------------------------------- /src/pydolphinscheduler/models/queue.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | """DolphinScheduler User object.""" 19 | 20 | from __future__ import annotations 21 | 22 | from pydolphinscheduler import configuration 23 | from pydolphinscheduler.models import BaseSide 24 | 25 | 26 | class Queue(BaseSide): 27 | """DolphinScheduler Queue object.""" 28 | 29 | def __init__( 30 | self, 31 | name: str = configuration.WORKFLOW_QUEUE, 32 | description: str | None = "", 33 | ): 34 | super().__init__(name, description) 35 | -------------------------------------------------------------------------------- /src/pydolphinscheduler/models/worker_group.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | """DolphinScheduler Worker Group object.""" 19 | 20 | from __future__ import annotations 21 | 22 | from pydolphinscheduler.models import BaseSide 23 | 24 | 25 | class WorkerGroup(BaseSide): 26 | """DolphinScheduler Worker Group object.""" 27 | 28 | def __init__(self, name: str, address: str, description: str | None = None): 29 | super().__init__(name, description) 30 | self.address = address 31 | -------------------------------------------------------------------------------- /src/pydolphinscheduler/resources_plugin/__init__.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | """Init resources_plugin package.""" 19 | from pydolphinscheduler.resources_plugin.github import GitHub 20 | from pydolphinscheduler.resources_plugin.gitlab import GitLab 21 | from pydolphinscheduler.resources_plugin.local import Local 22 | from pydolphinscheduler.resources_plugin.oss import OSS 23 | from pydolphinscheduler.resources_plugin.s3 import S3 24 | 25 | __all__ = ["Local", "GitHub", "GitLab", "OSS", "S3"] 26 | -------------------------------------------------------------------------------- /src/pydolphinscheduler/resources_plugin/base/__init__.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | """Init base package.""" 19 | -------------------------------------------------------------------------------- /src/pydolphinscheduler/resources_plugin/local.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | """DolphinScheduler local resource plugin.""" 19 | 20 | import os 21 | from pathlib import Path 22 | 23 | from pydolphinscheduler.core.resource_plugin import ResourcePlugin 24 | from pydolphinscheduler.exceptions import PyResPluginException 25 | 26 | 27 | class Local(ResourcePlugin): 28 | """Local object, declare local resource plugin for task and workflow to dolphinscheduler. 29 | 30 | :param prefix: A string representing the prefix of Local. 31 | """ 32 | 33 | # [start init_method] 34 | def __init__(self, prefix: str, *args, **kwargs): 35 | super().__init__(prefix, *args, **kwargs) 36 | 37 | # [end init_method] 38 | 39 | # [start read_file_method] 40 | def read_file(self, suf: str): 41 | """Get the content of the file. 42 | 43 | The address of the file is the prefix of the resource plugin plus the parameter suf. 44 | """ 45 | path = Path(self.prefix).joinpath(suf) 46 | if not path.exists(): 47 | raise PyResPluginException(f"{str(path)} is not found") 48 | if not os.access(str(path), os.R_OK): 49 | raise PyResPluginException( 50 | f"You don't have permission to access {self.prefix + suf}" 51 | ) 52 | with open(path) as f: 53 | content = f.read() 54 | return content 55 | 56 | # [end read_file_method] 57 | -------------------------------------------------------------------------------- /src/pydolphinscheduler/tasks/func_wrap.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | """Task function wrapper allows using decorator to create a task.""" 19 | 20 | import functools 21 | import inspect 22 | import types 23 | from pathlib import Path 24 | 25 | from stmdency.extractor import Extractor 26 | 27 | from pydolphinscheduler.exceptions import PyDSParamException 28 | from pydolphinscheduler.tasks.python import Python 29 | 30 | 31 | def _exists_other_decorator(func: types.FunctionType) -> None: 32 | """Check if the function has other decorators except @task. 33 | 34 | :param func: The function which wraps by decorator ``@task``. 35 | """ 36 | lines = inspect.getsourcelines(func)[0] 37 | 38 | for line in lines: 39 | strip_line = line.strip() 40 | if strip_line.startswith("@") and not strip_line == "@task": 41 | raise PyDSParamException( 42 | "Do no support other decorators for function ``task`` decorator." 43 | ) 44 | 45 | 46 | def task(func: types.FunctionType): 47 | """Decorate which covert Python functions into pydolphinscheduler task. 48 | 49 | :param func: The function which wraps by decorator ``@task``. 50 | """ 51 | 52 | @functools.wraps(func) 53 | def wrapper(*args, **kwargs): 54 | _exists_other_decorator(func) 55 | loc = func.__code__.co_filename 56 | extractor = Extractor(Path(loc).open("r").read()) 57 | stm = extractor.get_code(func.__name__) 58 | return Python( 59 | name=kwargs.get("name", func.__name__), 60 | definition=f"{stm}{func.__name__}()", 61 | *args, 62 | **kwargs, 63 | ) 64 | 65 | return wrapper 66 | -------------------------------------------------------------------------------- /src/pydolphinscheduler/tasks/kubernetes.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | """Task Kubernetes.""" 19 | from pydolphinscheduler.constants import TaskType 20 | from pydolphinscheduler.core.task import BatchTask 21 | 22 | 23 | class Kubernetes(BatchTask): 24 | """Task Kubernetes object, declare behavior for Kubernetes task to dolphinscheduler. 25 | 26 | :param name: task name 27 | :param image: the registry url for image. 28 | :param namespace: the namespace for running Kubernetes task. 29 | :param min_cpu_cores: min CPU requirement for running Kubernetes task. 30 | :param min_memory_space: min memory requirement for running Kubernetes task. 31 | :param params_map: It is a local user-defined parameter for Kubernetes task. 32 | """ 33 | 34 | _task_custom_attr = { 35 | "image", 36 | "namespace", 37 | "min_cpu_cores", 38 | "min_memory_space", 39 | } 40 | 41 | def __init__( 42 | self, 43 | name: str, 44 | image: str, 45 | namespace: str, 46 | min_cpu_cores: float, 47 | min_memory_space: float, 48 | *args, 49 | **kwargs 50 | ): 51 | super().__init__(name, TaskType.KUBERNETES, *args, **kwargs) 52 | self.image = image 53 | self.namespace = namespace 54 | self.min_cpu_cores = min_cpu_cores 55 | self.min_memory_space = min_memory_space 56 | -------------------------------------------------------------------------------- /src/pydolphinscheduler/tasks/map_reduce.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | """Task MR.""" 19 | 20 | from __future__ import annotations 21 | 22 | from pydolphinscheduler.constants import TaskType 23 | from pydolphinscheduler.core.engine import Engine, ProgramType 24 | 25 | 26 | class MR(Engine): 27 | """Task mr object, declare behavior for mr task to dolphinscheduler.""" 28 | 29 | _task_custom_attr = { 30 | "app_name", 31 | "main_args", 32 | "others", 33 | } 34 | 35 | def __init__( 36 | self, 37 | name: str, 38 | main_class: str, 39 | main_package: str, 40 | program_type: ProgramType | None = ProgramType.SCALA, 41 | app_name: str | None = None, 42 | main_args: str | None = None, 43 | others: str | None = None, 44 | *args, 45 | **kwargs, 46 | ): 47 | super().__init__( 48 | name, TaskType.MR, main_class, main_package, program_type, *args, **kwargs 49 | ) 50 | self.app_name = app_name 51 | self.main_args = main_args 52 | self.others = others 53 | -------------------------------------------------------------------------------- /src/pydolphinscheduler/tasks/openmldb.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | """Task OpenMLDB.""" 19 | 20 | from pydolphinscheduler.constants import TaskType 21 | from pydolphinscheduler.core.task import BatchTask 22 | 23 | 24 | class OpenMLDB(BatchTask): 25 | """Task OpenMLDB object, declare behavior for OpenMLDB task to dolphinscheduler. 26 | 27 | :param name: task name 28 | :param zookeeper: OpenMLDB cluster zookeeper address, e.g. 127.0.0.1:2181. 29 | :param zookeeper_path: OpenMLDB cluster zookeeper path, e.g. /openmldb. 30 | :param execute_mode: Determine the init mode, offline or online. You can switch it in sql statementself. 31 | :param sql: SQL statement. 32 | """ 33 | 34 | _task_custom_attr = { 35 | "zk", 36 | "zk_path", 37 | "execute_mode", 38 | "sql", 39 | } 40 | 41 | def __init__( 42 | self, name, zookeeper, zookeeper_path, execute_mode, sql, *args, **kwargs 43 | ): 44 | super().__init__(name, TaskType.OPENMLDB, *args, **kwargs) 45 | self.zk = zookeeper 46 | self.zk_path = zookeeper_path 47 | self.execute_mode = execute_mode 48 | self.sql = sql 49 | -------------------------------------------------------------------------------- /src/pydolphinscheduler/tasks/sagemaker.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | """Task SageMaker.""" 19 | 20 | from pydolphinscheduler.constants import TaskType 21 | from pydolphinscheduler.core.task import BatchTask 22 | 23 | 24 | class SageMaker(BatchTask): 25 | """Task SageMaker object, declare behavior for SageMaker task to dolphinscheduler. 26 | 27 | :param name: A unique, meaningful string for the SageMaker task. 28 | :param sagemaker_request_json: Request parameters of StartPipelineExecution， 29 | see also `AWS API 30 | `_ 31 | 32 | """ 33 | 34 | _task_custom_attr = { 35 | "sagemaker_request_json", 36 | } 37 | 38 | def __init__(self, name: str, sagemaker_request_json: str, *args, **kwargs): 39 | super().__init__(name, TaskType.SAGEMAKER, *args, **kwargs) 40 | self.sagemaker_request_json = sagemaker_request_json 41 | -------------------------------------------------------------------------------- /src/pydolphinscheduler/tasks/shell.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | """Task shell.""" 19 | 20 | from pydolphinscheduler.constants import TaskType 21 | from pydolphinscheduler.core.mixin import WorkerResourceMixin 22 | from pydolphinscheduler.core.task import BatchTask 23 | 24 | 25 | class Shell(WorkerResourceMixin, BatchTask): 26 | """Task shell object, declare behavior for shell task to dolphinscheduler. 27 | 28 | :param name: A unique, meaningful string for the shell task. 29 | :param command: One or more command want to run in this task. 30 | 31 | It could be simply command:: 32 | 33 | Shell(name=..., command="echo task shell") 34 | 35 | or maybe same commands trying to do complex task:: 36 | 37 | command = '''echo task shell step 1; 38 | echo task shell step 2; 39 | echo task shell step 3 40 | ''' 41 | 42 | Shell(name=..., command=command) 43 | 44 | """ 45 | 46 | # TODO maybe we could use instance name to replace attribute `name` 47 | # which is simplify as `task_shell = Shell(command = "echo 1")` and 48 | # task.name assign to `task_shell` 49 | 50 | _task_custom_attr = { 51 | "raw_script", 52 | } 53 | 54 | ext: set = {".sh", ".zsh"} 55 | ext_attr: str = "_raw_script" 56 | 57 | def __init__(self, name: str, command: str, *args, **kwargs): 58 | self._raw_script = command 59 | super().__init__(name, TaskType.SHELL, *args, **kwargs) 60 | self.add_attr(**kwargs) 61 | -------------------------------------------------------------------------------- /src/pydolphinscheduler/tasks/sub_process.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | """This module is deprecated. Please use `pydolphinscheduler.tasks.sub_workflow.SubWorkflow`.""" 19 | 20 | import warnings 21 | 22 | from pydolphinscheduler.tasks.sub_workflow import SubWorkflow 23 | 24 | warnings.warn( 25 | "This module is deprecated and will be remove in 4.1.0. " 26 | "Please use `pydolphinscheduler.tasks.sub_workflow.SubWorkflow` instead.", 27 | DeprecationWarning, 28 | stacklevel=2, 29 | ) 30 | 31 | 32 | class SubProcess(SubWorkflow): 33 | """Task SubProcess object, declare behavior for SubProcess task to dolphinscheduler. 34 | 35 | This module is deprecated and will be remove in 4.1.0. Please use 36 | `pydolphinscheduler.tasks.sub_workflow.SubWorkflow` instead. 37 | """ 38 | 39 | def __init__(self, name: str, process_definition_name: str, *args, **kwargs): 40 | super().__init__( 41 | name=name, workflow_name=process_definition_name, *args, **kwargs 42 | ) 43 | -------------------------------------------------------------------------------- /src/pydolphinscheduler/tasks/sub_workflow.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | """Task sub workflow.""" 19 | 20 | from __future__ import annotations 21 | 22 | from pydolphinscheduler.constants import TaskType 23 | from pydolphinscheduler.core.task import BatchTask 24 | from pydolphinscheduler.exceptions import PyDSWorkflowNotAssignException 25 | from pydolphinscheduler.java_gateway import gateway 26 | 27 | 28 | class SubWorkflow(BatchTask): 29 | """Task SubWorkflow object, declare behavior for SubWorkflow task to dolphinscheduler.""" 30 | 31 | _task_custom_attr = {"workflow_definition_code"} 32 | 33 | def __init__(self, name: str, workflow_name: str, *args, **kwargs): 34 | super().__init__(name, TaskType.SUB_WORKFLOW, *args, **kwargs) 35 | self.workflow_name = workflow_name 36 | 37 | @property 38 | def workflow_definition_code(self) -> str: 39 | """Get workflow code, a wrapper for :func:`get_workflow_info`. 40 | 41 | We can not change this function name to workflow_code, because it is a keyword used in 42 | dolphinscheduler itself. 43 | """ 44 | return self.get_workflow_info(self.workflow_name).get("code") 45 | 46 | def get_workflow_info(self, workflow_name: str) -> dict: 47 | """Get workflow info from java gateway, contains workflow id, name, code.""" 48 | if not self.workflow: 49 | raise PyDSWorkflowNotAssignException( 50 | "Workflow must be provider for task SubWorkflow." 51 | ) 52 | return gateway.get_workflow_info( 53 | self.workflow.user.name, 54 | self.workflow.project.name, 55 | workflow_name, 56 | ) 57 | -------------------------------------------------------------------------------- /src/pydolphinscheduler/utils/__init__.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | """Init utils package.""" 19 | -------------------------------------------------------------------------------- /src/pydolphinscheduler/utils/file.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | """File util for pydolphinscheduler.""" 19 | 20 | from __future__ import annotations 21 | 22 | from pathlib import Path 23 | 24 | 25 | def write( 26 | content: str, 27 | to_path: str, 28 | create: bool | None = True, 29 | overwrite: bool | None = False, 30 | ) -> None: 31 | """Write configs dict to configuration file. 32 | 33 | :param content: The source string want to write to :param:`to_path`. 34 | :param to_path: The path want to write content. 35 | :param create: Whether create the file parent directory or not if it does not exist. 36 | If set ``True`` will create file with :param:`to_path` if path not exists, otherwise 37 | ``False`` will not create. Default ``True``. 38 | :param overwrite: Whether overwrite the file or not if it exists. If set ``True`` 39 | will overwrite the exists content, otherwise ``False`` will not overwrite it. Default ``True``. 40 | """ 41 | path = Path(to_path) 42 | if not path.parent.exists(): 43 | if create: 44 | path.parent.mkdir(parents=True) 45 | else: 46 | raise ValueError( 47 | "Parent directory do not exists and set param `create` to `False`." 48 | ) 49 | if not path.exists(): 50 | with path.open(mode="w") as f: 51 | f.write(content) 52 | elif overwrite: 53 | with path.open(mode="w") as f: 54 | f.write(content) 55 | else: 56 | raise FileExistsError( 57 | "File %s already exists and you choose not overwrite mode.", to_path 58 | ) 59 | -------------------------------------------------------------------------------- /src/pydolphinscheduler/utils/string.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | """String util function collections.""" 19 | 20 | from pydolphinscheduler.constants import Delimiter 21 | 22 | 23 | def attr2camel(attr: str, include_private=True): 24 | """Covert class attribute name to camel case.""" 25 | if include_private: 26 | attr = attr.lstrip(Delimiter.UNDERSCORE) 27 | return snake2camel(attr) 28 | 29 | 30 | def snake2camel(snake: str): 31 | """Covert snake case to camel case.""" 32 | components = snake.split(Delimiter.UNDERSCORE) 33 | return components[0] + "".join(x.title() for x in components[1:]) 34 | 35 | 36 | def class_name2camel(class_name: str): 37 | """Covert class name string to camel case.""" 38 | class_name = class_name.lstrip(Delimiter.UNDERSCORE) 39 | return class_name[0].lower() + snake2camel(class_name[1:]) 40 | -------------------------------------------------------------------------------- /src/pydolphinscheduler/utils/versions.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | """Util versions.""" 19 | 20 | from pathlib import Path 21 | 22 | from packaging.requirements import Requirement as packaging_Requirement 23 | from packaging.version import InvalidVersion 24 | 25 | from pydolphinscheduler.constants import Version 26 | 27 | 28 | def version_match(name: str, version: str) -> bool: 29 | """Check if the version of external system matches current python sdk version. 30 | 31 | :param name: External system name in file ``Version.FILE_NAME`` 32 | :param version: External system current version 33 | """ 34 | path = Path(__file__).parent.parent.joinpath(Version.FILE_NAME) 35 | with path.open() as match: 36 | for line in match.readlines(): 37 | req = packaging_Requirement(line) 38 | if req.name == name: 39 | try: 40 | return req.specifier.contains(version) 41 | except InvalidVersion: 42 | return False 43 | raise ValueError(f"{name} is not in {Version.FILE_NAME}") 44 | -------------------------------------------------------------------------------- /src/pydolphinscheduler/version_ext: -------------------------------------------------------------------------------- 1 | dolphinscheduler>=3.2.0,<3.3.0 -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | """Init tests package.""" 19 | -------------------------------------------------------------------------------- /tests/cli/__init__.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | """Init command line interface tests.""" 19 | -------------------------------------------------------------------------------- /tests/cli/test_version.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | """Test command line interface subcommand `version`.""" 19 | 20 | from unittest.mock import patch 21 | 22 | import pytest 23 | 24 | import pydolphinscheduler 25 | from pydolphinscheduler.cli.commands import cli 26 | from tests.testing.cli import CliTestWrapper 27 | 28 | 29 | def test_version(): 30 | """Test whether subcommand `version` correct.""" 31 | cli_test = CliTestWrapper(cli, ["version"]) 32 | cli_test.assert_success(output=f"{pydolphinscheduler.__version__}") 33 | 34 | 35 | @pytest.mark.parametrize( 36 | "version, part, idx", 37 | [ 38 | ("1.2.3", "major", 0), 39 | ("0.1.3", "minor", 1), 40 | ("3.1.0", "micro", 2), 41 | ("1.2.3-beta-1", "micro", 2), 42 | ("1.2.3-alpha", "micro", 2), 43 | ("1.2.3a2", "micro", 2), 44 | ("1.2.3b1", "micro", 2), 45 | ], 46 | ) 47 | @patch("pydolphinscheduler.__version__") 48 | def test_version_part(mock_version, version: str, part: str, idx: int): 49 | """Test subcommand `version` option `--part`.""" 50 | mock_version.return_value = version 51 | cli_test = CliTestWrapper(cli, ["version", "--part", part]) 52 | cli_test.assert_success(output=f"{pydolphinscheduler.__version__.split('.')[idx]}") 53 | 54 | 55 | @pytest.mark.parametrize( 56 | "option, output", 57 | [ 58 | # not support option 59 | (["version", "--not-support"], "No such option"), 60 | # not support option value 61 | (["version", "--part", "abc"], "Invalid value for '--part'"), 62 | ], 63 | ) 64 | def test_version_not_support_option(option, output): 65 | """Test subcommand `version` not support option or option value.""" 66 | cli_test = CliTestWrapper(cli, option) 67 | cli_test.assert_fail(ret_code=2, output=output, fuzzy=True) 68 | -------------------------------------------------------------------------------- /tests/core/__init__.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | """Init core package tests.""" 19 | -------------------------------------------------------------------------------- /tests/example/__init__.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | """Init example package tests.""" 19 | -------------------------------------------------------------------------------- /tests/integration/__init__.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | """Test integration between Python API and PythonGatewayService.""" 19 | -------------------------------------------------------------------------------- /tests/integration/test_process_definition.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | """Test workflow in integration.""" 19 | from __future__ import annotations 20 | 21 | import pytest 22 | 23 | from pydolphinscheduler.core.workflow import Workflow 24 | from pydolphinscheduler.tasks.shell import Shell 25 | 26 | WORKFLOW_NAME = "test_change_exists_attr_pd" 27 | TASK_NAME = f"task_{WORKFLOW_NAME}" 28 | 29 | 30 | @pytest.mark.parametrize( 31 | "pre, post", 32 | [ 33 | ( 34 | { 35 | "user": "pre_user", 36 | }, 37 | { 38 | "user": "post_user", 39 | }, 40 | ) 41 | ], 42 | ) 43 | def test_change_workflow_attr(pre: dict, post: dict): 44 | """Test whether workflow success when specific attribute change.""" 45 | assert pre.keys() == post.keys(), "Not equal keys for pre and post attribute." 46 | for attrs in [pre, post]: 47 | with Workflow(name=WORKFLOW_NAME, **attrs) as workflow: 48 | Shell(name=TASK_NAME, command="echo 1") 49 | workflow.submit() 50 | -------------------------------------------------------------------------------- /tests/integration/test_resources.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | """Test pydolphinscheduler resources.""" 19 | 20 | import pytest 21 | from py4j.java_gateway import JavaObject 22 | 23 | from pydolphinscheduler.core.resource import Resource 24 | from pydolphinscheduler.models.user import User 25 | from tests.testing.constants import UNIT_TEST_TENANT, UNIT_TEST_USER_NAME 26 | 27 | name = "unittest_resource.txt" 28 | content = "unittest_resource_content" 29 | 30 | 31 | @pytest.fixture(scope="module") 32 | def tmp_user(): 33 | """Get a temporary user.""" 34 | user = User( 35 | name=UNIT_TEST_USER_NAME, 36 | password="unittest-password", 37 | email="test-email@abc.com", 38 | phone="17366637777", 39 | tenant=UNIT_TEST_TENANT, 40 | queue="test-queue", 41 | status=1, 42 | ) 43 | user.create_if_not_exists() 44 | yield 45 | user.delete() 46 | 47 | 48 | @pytest.mark.skip(reason="Not work for latest code, at Jul 4, 2024") 49 | def test_create_or_update(tmp_user): 50 | """Test create or update resource to java gateway.""" 51 | resource = Resource(name=name, content=content, user_name=UNIT_TEST_USER_NAME) 52 | result = resource.create_or_update_resource() 53 | assert result is not None and isinstance(result, JavaObject) 54 | assert result.getAlias() == name 55 | 56 | 57 | @pytest.mark.skip(reason="Not work for latest code, at Jul 4, 2024") 58 | def test_get_resource_info(tmp_user): 59 | """Test get resource info from java gateway.""" 60 | resource = Resource(name=name, user_name=UNIT_TEST_USER_NAME) 61 | result = resource.get_info_from_database() 62 | assert result is not None and isinstance(result, JavaObject) 63 | assert result.getAlias() == name 64 | -------------------------------------------------------------------------------- /tests/integration/test_submit_examples.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | """Test whether success submit examples DAG to PythonGatewayService.""" 19 | 20 | import subprocess 21 | from pathlib import Path 22 | 23 | import pytest 24 | 25 | from tests.testing.constants import ignore_exec_examples 26 | from tests.testing.path import path_example 27 | 28 | 29 | @pytest.mark.parametrize( 30 | "example_path", 31 | [ 32 | path 33 | for path in path_example.iterdir() 34 | if path.is_file() and path.stem not in ignore_exec_examples 35 | ], 36 | ) 37 | def test_exec_white_list_example(example_path: Path): 38 | """Test execute examples and submit DAG to PythonGatewayService.""" 39 | try: 40 | # Because our task decorator used module ``inspect`` to get the source, and it will 41 | # raise IOError when call it by built-in function ``exec``, so we change to ``subprocess.check_call`` 42 | subprocess.check_call(["python", str(example_path)]) 43 | except subprocess.CalledProcessError: 44 | raise RuntimeError("Run example %s failed.", example_path.stem) 45 | 46 | 47 | def test_exec_multiple_times(): 48 | """Test whether process definition can be executed more than one times.""" 49 | tutorial_path = path_example.joinpath("tutorial.py") 50 | time = 0 51 | while time < 3: 52 | try: 53 | subprocess.check_call(["python", str(tutorial_path)]) 54 | except subprocess.CalledProcessError: 55 | raise RuntimeError("Run example %s failed.", tutorial_path.stem) 56 | time += 1 57 | -------------------------------------------------------------------------------- /tests/models/__init__.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | """Init modules package tests.""" 19 | -------------------------------------------------------------------------------- /tests/models/test_database.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | """Test Database.""" 19 | import json 20 | from unittest.mock import patch 21 | 22 | import pytest 23 | 24 | from pydolphinscheduler.models.connection import Connection 25 | from pydolphinscheduler.models.datasource import Datasource 26 | 27 | TEST_DATABASE_DATASOURCE_NAME = "test_datasource" 28 | TEST_DATABASE_TYPE = "mysql" 29 | 30 | TEST_CONNECTION_PARAMS = { 31 | "user": "root", 32 | "password": "mysql", 33 | "address": "jdbc:mysql://127.0.0.1:3306", 34 | "database": "test", 35 | "jdbcUrl": "jdbc:mysql://127.0.0.1:3306/test", 36 | "driverClassName": "com.mysql.cj.jdbc.Driver", 37 | "validationQuery": "select 1", 38 | } 39 | 40 | TEST_CONNECTION_ARG = { 41 | "host": "127.0.0.1", 42 | "port": 3306, 43 | "schema": "test", 44 | "username": "root", 45 | "password": "mysql", 46 | } 47 | 48 | 49 | datasource = Datasource( 50 | id_=1, 51 | type_=TEST_DATABASE_TYPE, 52 | name=TEST_DATABASE_DATASOURCE_NAME, 53 | connection_params=json.dumps(TEST_CONNECTION_PARAMS), 54 | user_id=1, 55 | ) 56 | 57 | 58 | @pytest.mark.parametrize( 59 | "attr, value", 60 | [ 61 | ("connection", Connection(**TEST_CONNECTION_ARG)), 62 | ("host", "127.0.0.1"), 63 | ("port", 3306), 64 | ("username", "root"), 65 | ("password", "mysql"), 66 | ("schema", "test"), 67 | ], 68 | ) 69 | @patch.object(Datasource, "get", return_value=datasource) 70 | def test_get_datasource_attr(mock_datasource, attr, value): 71 | """Test get datasource attr.""" 72 | datasource_get = Datasource.get(TEST_DATABASE_DATASOURCE_NAME, TEST_DATABASE_TYPE) 73 | assert value == getattr(datasource_get, attr) 74 | -------------------------------------------------------------------------------- /tests/resources_plugin/__init__.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | """Init resources_plugin package tests.""" 19 | -------------------------------------------------------------------------------- /tests/resources_plugin/test_resource_plugin.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | """Test abstract class resource_plugin.""" 19 | 20 | import pytest 21 | 22 | from pydolphinscheduler.exceptions import PyResPluginException 23 | from pydolphinscheduler.resources_plugin import GitHub 24 | 25 | 26 | @pytest.mark.parametrize( 27 | "attr, expected", 28 | [ 29 | ( 30 | { 31 | "s": "https://api.github.com/repos/apache/dolphinscheduler/contents/script/install.sh", 32 | "x": "/", 33 | "n": 2, 34 | }, 35 | 7, 36 | ), 37 | ( 38 | { 39 | "s": "https://api.github.com", 40 | "x": ":", 41 | "n": 1, 42 | }, 43 | 5, 44 | ), 45 | ], 46 | ) 47 | def test_github_get_index(attr, expected): 48 | """Test the get_index function of the abstract class resource_plugin.""" 49 | github = GitHub(prefix="prefix") 50 | assert expected == github.get_index(**attr) 51 | 52 | 53 | @pytest.mark.parametrize( 54 | "attr", 55 | [ 56 | { 57 | "s": "https://api.github.com", 58 | "x": "/", 59 | "n": 3, 60 | }, 61 | { 62 | "s": "https://api.github.com/", 63 | "x": "/", 64 | "n": 4, 65 | }, 66 | ], 67 | ) 68 | def test_github_get_index_exception(attr): 69 | """Test exception to get_index function of abstract class resource_plugin.""" 70 | with pytest.raises( 71 | PyResPluginException, 72 | match="Incomplete path.", 73 | ): 74 | github = GitHub(prefix="prefix") 75 | github.get_index(**attr) 76 | -------------------------------------------------------------------------------- /tests/tasks/__init__.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | """Init tasks package tests.""" 19 | -------------------------------------------------------------------------------- /tests/tasks/test_flink.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | """Test Task Flink.""" 19 | 20 | from unittest.mock import patch 21 | 22 | from pydolphinscheduler.tasks.flink import DeployMode, Flink, FlinkVersion, ProgramType 23 | 24 | 25 | @patch( 26 | "pydolphinscheduler.core.engine.Engine.get_resource_info", 27 | return_value=({"id": 1, "name": "test"}), 28 | ) 29 | def test_flink_get_define(mock_resource): 30 | """Test task flink function get_define.""" 31 | code = 123 32 | version = 1 33 | name = "test_flink_get_define" 34 | main_class = "org.apache.flink.test_main_class" 35 | main_package = "test_main_package" 36 | program_type = ProgramType.JAVA 37 | deploy_mode = DeployMode.LOCAL 38 | 39 | expect_task_params = { 40 | "mainClass": main_class, 41 | "mainJar": { 42 | "id": 1, 43 | }, 44 | "programType": program_type, 45 | "deployMode": deploy_mode, 46 | "flinkVersion": FlinkVersion.LOW_VERSION, 47 | "slot": 1, 48 | "parallelism": 1, 49 | "taskManager": 2, 50 | "jobManagerMemory": "1G", 51 | "taskManagerMemory": "2G", 52 | "appName": None, 53 | "mainArgs": None, 54 | "others": None, 55 | "localParams": [], 56 | "resourceList": [], 57 | "dependence": {}, 58 | "conditionResult": {"successNode": [""], "failedNode": [""]}, 59 | "waitStartTimeout": {}, 60 | } 61 | with patch( 62 | "pydolphinscheduler.core.task.Task.gen_code_and_version", 63 | return_value=(code, version), 64 | ): 65 | task = Flink(name, main_class, main_package, program_type, deploy_mode) 66 | assert task.task_params == expect_task_params 67 | -------------------------------------------------------------------------------- /tests/tasks/test_kubernetes.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | """Test Task Kubernetes.""" 19 | 20 | from unittest.mock import patch 21 | 22 | from pydolphinscheduler.tasks.kubernetes import Kubernetes 23 | 24 | 25 | def test_kubernetes_get_define(): 26 | """Test task kubernetes function get_define.""" 27 | code = 123 28 | version = 1 29 | name = "test_kubernetes_get_define" 30 | image = "ds-dev" 31 | namespace = str({"name": "default", "cluster": "lab"}) 32 | minCpuCores = 2.0 33 | minMemorySpace = 10.0 34 | 35 | expect_task_params = { 36 | "resourceList": [], 37 | "localParams": [], 38 | "image": image, 39 | "namespace": namespace, 40 | "minCpuCores": minCpuCores, 41 | "minMemorySpace": minMemorySpace, 42 | "dependence": {}, 43 | "conditionResult": {"successNode": [""], "failedNode": [""]}, 44 | "waitStartTimeout": {}, 45 | } 46 | with patch( 47 | "pydolphinscheduler.core.task.Task.gen_code_and_version", 48 | return_value=(code, version), 49 | ): 50 | k8s = Kubernetes(name, image, namespace, minCpuCores, minMemorySpace) 51 | assert k8s.task_params == expect_task_params 52 | -------------------------------------------------------------------------------- /tests/tasks/test_map_reduce.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | """Test Task MR.""" 19 | 20 | from unittest.mock import patch 21 | 22 | from pydolphinscheduler.tasks.map_reduce import MR, ProgramType 23 | 24 | 25 | @patch( 26 | "pydolphinscheduler.core.engine.Engine.get_resource_info", 27 | return_value=({"id": 1, "name": "test"}), 28 | ) 29 | def test_mr_get_define(mock_resource): 30 | """Test task mr function get_define.""" 31 | code = 123 32 | version = 1 33 | name = "test_mr_get_define" 34 | main_class = "org.apache.mr.test_main_class" 35 | main_package = "test_main_package" 36 | program_type = ProgramType.JAVA 37 | main_args = "/dolphinscheduler/resources/file.txt /output/ds" 38 | 39 | expect_task_params = { 40 | "mainClass": main_class, 41 | "mainJar": { 42 | "id": 1, 43 | }, 44 | "programType": program_type, 45 | "appName": None, 46 | "mainArgs": main_args, 47 | "others": None, 48 | "localParams": [], 49 | "resourceList": [], 50 | "dependence": {}, 51 | "conditionResult": {"successNode": [""], "failedNode": [""]}, 52 | "waitStartTimeout": {}, 53 | } 54 | with patch( 55 | "pydolphinscheduler.core.task.Task.gen_code_and_version", 56 | return_value=(code, version), 57 | ): 58 | task = MR(name, main_class, main_package, program_type, main_args=main_args) 59 | assert task.task_params == expect_task_params 60 | -------------------------------------------------------------------------------- /tests/tasks/test_openmldb.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | """Test Task OpenMLDB.""" 19 | from unittest.mock import patch 20 | 21 | from pydolphinscheduler.tasks.openmldb import OpenMLDB 22 | 23 | 24 | def test_openmldb_get_define(): 25 | """Test task openmldb function get_define.""" 26 | zookeeper = "127.0.0.1:2181" 27 | zookeeper_path = "/openmldb" 28 | execute_mode = "offline" 29 | 30 | sql = """USE demo_db; 31 | set @@job_timeout=200000; 32 | LOAD DATA INFILE 'file:///tmp/train_sample.csv' 33 | INTO TABLE talkingdata OPTIONS(mode='overwrite'); 34 | """ 35 | 36 | code = 123 37 | version = 1 38 | name = "test_openmldb_get_define" 39 | expect_task_params = { 40 | "resourceList": [], 41 | "localParams": [], 42 | "zk": zookeeper, 43 | "zkPath": zookeeper_path, 44 | "executeMode": execute_mode, 45 | "sql": sql, 46 | "dependence": {}, 47 | "conditionResult": {"successNode": [""], "failedNode": [""]}, 48 | "waitStartTimeout": {}, 49 | } 50 | with patch( 51 | "pydolphinscheduler.core.task.Task.gen_code_and_version", 52 | return_value=(code, version), 53 | ): 54 | openmldb = OpenMLDB(name, zookeeper, zookeeper_path, execute_mode, sql) 55 | assert openmldb.task_params == expect_task_params 56 | -------------------------------------------------------------------------------- /tests/tasks/test_spark.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | """Test Task Spark.""" 19 | 20 | from unittest.mock import patch 21 | 22 | from pydolphinscheduler.tasks.spark import DeployMode, ProgramType, Spark 23 | 24 | 25 | @patch( 26 | "pydolphinscheduler.core.engine.Engine.get_resource_info", 27 | return_value=({"id": 1, "name": "test"}), 28 | ) 29 | def test_spark_get_define(mock_resource): 30 | """Test task spark function get_define.""" 31 | code = 123 32 | version = 1 33 | name = "test_spark_get_define" 34 | main_class = "org.apache.spark.test_main_class" 35 | main_package = "test_main_package" 36 | program_type = ProgramType.JAVA 37 | deploy_mode = DeployMode.LOCAL 38 | 39 | expect_task_params = { 40 | "mainClass": main_class, 41 | "mainJar": { 42 | "id": 1, 43 | }, 44 | "programType": program_type, 45 | "deployMode": deploy_mode, 46 | "driverCores": 1, 47 | "driverMemory": "512M", 48 | "numExecutors": 2, 49 | "executorMemory": "2G", 50 | "executorCores": 2, 51 | "appName": None, 52 | "mainArgs": None, 53 | "others": None, 54 | "localParams": [], 55 | "resourceList": [], 56 | "dependence": {}, 57 | "conditionResult": {"successNode": [""], "failedNode": [""]}, 58 | "waitStartTimeout": {}, 59 | } 60 | with patch( 61 | "pydolphinscheduler.core.task.Task.gen_code_and_version", 62 | return_value=(code, version), 63 | ): 64 | task = Spark(name, main_class, main_package, program_type, deploy_mode) 65 | assert task.task_params == expect_task_params 66 | -------------------------------------------------------------------------------- /tests/testing/__init__.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | """Init testing package, it provider easy way for pydolphinscheduler test.""" 19 | -------------------------------------------------------------------------------- /tests/testing/constants.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | """Constants variables for test module.""" 19 | 20 | import os 21 | 22 | # Record some task without example in directory `example`. Some of them maybe can not write example, 23 | # but most of them just without adding by mistake, and we should add it later. 24 | task_without_example = { 25 | "http", 26 | "procedure", 27 | } 28 | 29 | # The examples ignore test to run it. Those examples could not be run directly cause it need other 30 | # support like resource files, data source and etc. But we should try to run them later for more coverage 31 | ignore_exec_examples = { 32 | "task_datax_example", 33 | "task_flink_example", 34 | "task_map_reduce_example", 35 | "task_spark_example", 36 | # TODO activate it when dolphinscheduler default resource center is local file 37 | "multi_resources_example", 38 | "task_sql_example", 39 | } 40 | 41 | # pydolphinscheduler environment home 42 | ENV_PYDS_HOME = "PYDS_HOME" 43 | 44 | # whether in dev mode, if true we will add or remove some tests. Or make be and more detail infos when 45 | # test failed. 46 | DEV_MODE = str( 47 | os.environ.get("PY_DOLPHINSCHEDULER_DEV_MODE", False) 48 | ).strip().lower() in {"true", "t", "1"} 49 | 50 | # default token 51 | TOKEN = "jwUDzpLsNKEFER4*a8gruBH_GsAurNxU7A@Xc" 52 | 53 | UNIT_TEST_USER_NAME = "unittest_user" 54 | UNIT_TEST_TENANT = "unittest_tenant" 55 | -------------------------------------------------------------------------------- /tests/testing/decorator.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | """Decorator module for testing module.""" 19 | 20 | import types 21 | from functools import wraps 22 | 23 | 24 | def foo(func: types.FunctionType): 25 | """Decorate which do nothing for testing module.""" 26 | 27 | @wraps(func) 28 | def wrapper(): 29 | print("foo decorator called.") 30 | func() 31 | 32 | return wrapper 33 | -------------------------------------------------------------------------------- /tests/testing/file.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | """Testing util about file operating.""" 19 | from __future__ import annotations 20 | 21 | from pathlib import Path 22 | 23 | 24 | def get_file_content(path: str | Path) -> str: 25 | """Get file content in given path.""" 26 | with open(path) as f: 27 | return f.read() 28 | 29 | 30 | def delete_file(path: str | Path) -> None: 31 | """Delete file in given path.""" 32 | path = Path(path).expanduser() if isinstance(path, str) else path.expanduser() 33 | if path.exists(): 34 | path.unlink() 35 | -------------------------------------------------------------------------------- /tests/testing/path.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | """Handle path related issue in test module.""" 19 | from __future__ import annotations 20 | 21 | from collections.abc import Generator 22 | from pathlib import Path 23 | from typing import Any 24 | 25 | project_root = Path(__file__).parent.parent.parent 26 | 27 | path_code_tasks = project_root.joinpath("src", "pydolphinscheduler", "tasks") 28 | path_example = project_root.joinpath("src", "pydolphinscheduler", "examples") 29 | path_yaml_example = project_root.joinpath("examples", "yaml_define") 30 | path_doc_tasks = project_root.joinpath("docs", "source", "tasks") 31 | path_default_config_yaml = project_root.joinpath( 32 | "src", "pydolphinscheduler", "default_config.yaml" 33 | ) 34 | 35 | 36 | def get_all_examples() -> Generator[Path, Any, None]: 37 | """Get all examples files path in examples directory.""" 38 | return (ex for ex in path_example.iterdir() if ex.is_file()) 39 | 40 | 41 | def get_tasks(ignore_name: set = None) -> Generator[Path, Any, None]: 42 | """Get all tasks files path in src/pydolphinscheduler/tasks directory.""" 43 | if not ignore_name: 44 | ignore_name = set() 45 | return ( 46 | ex 47 | for ex in path_code_tasks.iterdir() 48 | if ex.is_file() and ex.name not in ignore_name 49 | ) 50 | 51 | 52 | def get_doc_tasks(ignore_name: set = None) -> Generator[Path, Any, None]: 53 | """Get all tasks document path in docs/source/tasks directory.""" 54 | if not ignore_name: 55 | ignore_name = set() 56 | return ( 57 | ex 58 | for ex in path_doc_tasks.iterdir() 59 | if ex.is_file() and ex.name not in ignore_name 60 | ) 61 | -------------------------------------------------------------------------------- /tests/testing/task.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | """Mock class Task for other test.""" 19 | 20 | import uuid 21 | 22 | from pydolphinscheduler.core.task import Task as SourceTask 23 | 24 | 25 | class Task(SourceTask): 26 | """Mock class :class:`pydolphinscheduler.core.task.Task` for unittest.""" 27 | 28 | DEFAULT_VERSION = 1 29 | 30 | def gen_code_and_version(self): 31 | """Mock java gateway code and version, convenience method for unittest.""" 32 | return uuid.uuid1().time, self.DEFAULT_VERSION 33 | 34 | 35 | class TaskWithCode(SourceTask): 36 | """Mock class :class:`pydolphinscheduler.core.task.Task` and it return some code and version.""" 37 | 38 | def __init__( 39 | self, name: str, task_type: str, code: int, version: int, *args, **kwargs 40 | ): 41 | self._constant_code = code 42 | self._constant_version = version 43 | super().__init__(name, task_type, *args, **kwargs) 44 | 45 | def gen_code_and_version(self): 46 | """Mock java gateway code and version, convenience method for unittest.""" 47 | return self._constant_code, self._constant_version 48 | -------------------------------------------------------------------------------- /tests/utils/__init__.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | """Init tests for utils package.""" 19 | -------------------------------------------------------------------------------- /tests/utils/test_default_config_yaml.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | """Test default config file.""" 19 | 20 | from ruamel.yaml import YAML 21 | from ruamel.yaml.comments import CommentedMap 22 | 23 | from tests.testing.path import path_default_config_yaml 24 | 25 | 26 | def nested_key_check(comment_map: CommentedMap) -> None: 27 | """Test whether default configuration file exists specific character.""" 28 | for key, val in comment_map.items(): 29 | assert "." not in key, f"There is not allowed special character in key `{key}`." 30 | if isinstance(val, CommentedMap): 31 | nested_key_check(val) 32 | 33 | 34 | def test_key_without_dot_delimiter(): 35 | """Test wrapper of whether default configuration file exists specific character.""" 36 | yaml = YAML() 37 | with open(path_default_config_yaml) as f: 38 | comment_map = yaml.load(f.read()) 39 | nested_key_check(comment_map) 40 | --------------------------------------------------------------------------------