├── .github ├── ISSUE_TEMPLATE │ ├── bug_report.md │ ├── config.yml │ └── enhancement_proposal.md ├── pull_request_template ├── semantic.yml └── workflows │ ├── build.yml │ └── docs.yml ├── .gitignore ├── .isort.cfg ├── .pre-commit-config.yaml ├── ADOPTERS.md ├── AUTHORS ├── CODE_OF_CONDUCT.md ├── CONTRIBUTING.md ├── CONTRIBUTORS ├── LICENSE ├── MANIFEST.in ├── README.md ├── RELEASE.md ├── SECURITY.md ├── couler ├── __init__.py ├── _version.py ├── argo.py ├── argo_submitter.py ├── core │ ├── __init__.py │ ├── config.py │ ├── constants.py │ ├── proto_repr.py │ ├── run_templates.py │ ├── states.py │ ├── step_update_utils.py │ ├── syntax │ │ ├── __init__.py │ │ ├── concurrent.py │ │ ├── conditional.py │ │ ├── dag.py │ │ ├── dns.py │ │ ├── exit_handler.py │ │ ├── image_pull_secret.py │ │ ├── loop.py │ │ ├── predicates.py │ │ ├── recursion.py │ │ ├── toleration.py │ │ └── volume.py │ ├── templates │ │ ├── __init__.py │ │ ├── artifact.py │ │ ├── cache.py │ │ ├── container.py │ │ ├── dns.py │ │ ├── image_pull_secret.py │ │ ├── job.py │ │ ├── output.py │ │ ├── script.py │ │ ├── secret.py │ │ ├── step.py │ │ ├── template.py │ │ ├── toleration.py │ │ ├── volume.py │ │ ├── volume_claim.py │ │ └── workflow.py │ ├── utils.py │ └── workflow_validation_utils.py ├── docker_submitter.py ├── proto │ ├── __init__.py │ └── couler_pb2.py ├── steps │ ├── katib.py │ ├── mpi.py │ ├── pod_utils.py │ ├── pytorch.py │ └── tensorflow.py └── tests │ ├── __init__.py │ ├── argo_test.py │ ├── argo_yaml_test.py │ ├── artifact_test.py │ ├── cluster_config_test.py │ ├── cron_workflow_test.py │ ├── daemon_step_test.py │ ├── dag_test.py │ ├── env_test.py │ ├── input_parameter_test.py │ ├── katib_step_test.py │ ├── map_test.py │ ├── mpi_step_test.py │ ├── proto_repr_test.py │ ├── pytorch_step_test.py │ ├── resource_test.py │ ├── run_concurrent_test.py │ ├── secret_test.py │ ├── step_output_test.py │ ├── tensorflow_step_test.py │ ├── test_data │ ├── artifact_passing_golden.yaml │ ├── cron_workflow_golden.yaml │ ├── dag_golden_1.yaml │ ├── dag_golden_2.yaml │ ├── dummy_cluster_config.py │ ├── input_para_golden_1.yaml │ ├── input_para_golden_2.yaml │ ├── input_para_golden_3.yaml │ ├── input_para_golden_4.yaml │ ├── output_golden_1.yaml │ ├── output_golden_2.yaml │ ├── parameter_passing_golden.yaml │ ├── resource_config_golden.yaml │ ├── run_concurrent_golden.yaml │ ├── run_concurrent_golden_2.yaml │ ├── run_concurrent_golden_3.yaml │ ├── run_concurrent_subtasks_golden.yaml │ ├── secret_golden.yaml │ ├── while_golden.yaml │ └── workflow_basic_golden.yaml │ ├── utils_test.py │ ├── while_test.py │ ├── workflow_basic_test.py │ └── workflow_validation_utils_test.py ├── docs ├── NL-to-Unified-Programming-Interface │ ├── Algorithm.png │ ├── Method.md │ ├── Method_overview.pdf │ └── Running_example.pdf ├── README.md ├── TEMPLATE.md ├── Technical-Report-of-Couler │ ├── README.md │ └── Tech-Report-of-Couler-Unified-Machine-Learning-Workflow-Optimization-in-Cloud.pdf ├── adopters.md ├── assets │ ├── logo-white.svg │ ├── logo.svg │ └── stylesheets │ │ └── extra.css ├── contributing.md ├── couler-api-design.md ├── couler-step-zoo.md ├── couler-tekton-design.md ├── examples.md └── getting-started.md ├── examples ├── coin_flip.py ├── dag.py ├── default_submitter.py ├── depends.py ├── hello_world.py └── node_assign.py ├── go.mod ├── go.sum ├── go └── couler │ ├── commands │ └── submit.go │ ├── conversion │ ├── argo_workflow.go │ └── argo_workflow_test.go │ ├── optimization │ ├── optimization.go │ └── optimization_test.go │ ├── proto │ └── couler │ │ └── v1 │ │ ├── couler.pb.go │ │ └── proto.go │ └── submitter │ ├── argo_submitter.go │ └── argo_submitter_test.go ├── integration_tests ├── dag_depends_example.py ├── dag_example.py ├── flip_coin_example.py ├── flip_coin_security_context_example.py ├── memoization_example.py ├── mpi_example.py └── volume_example.py ├── manifests └── mpi-operator.yaml ├── mkdocs.yml ├── proto └── couler.proto ├── requirements-dev.txt ├── requirements.txt ├── scripts ├── integration_tests.sh ├── test_go.sh ├── test_python.sh └── validate_workflow_statuses.sh ├── setup.py └── templates └── LICENSE.txt /.github/ISSUE_TEMPLATE/bug_report.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Reproducible bug report 3 | about: Create a reproducible bug report. Not for support requests. 4 | labels: 'bug' 5 | --- 6 | ## Summary 7 | 8 | What happened/what you expected to happen? 9 | 10 | ## Diagnostics 11 | 12 | What is the version of Couler you are using? 13 | 14 | What is the version of the workflow engine you are using? 15 | 16 | Any logs or other information that could help debugging? 17 | 18 | --- 19 | 20 | **Message from the maintainers**: 21 | 22 | Impacted by this bug? Give it a 👍. We prioritize the issues with the most 👍. 23 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/config.yml: -------------------------------------------------------------------------------- 1 | blank_issues_enabled: false 2 | 3 | contact_links: 4 | - name: Chat on Slack 5 | url: https://join.slack.com/t/couler/shared_invite/zt-i0m7ziol-1gz4_p_aXmWM7KVpxJ4k9Q 6 | about: Chatting with the community can help 7 | - name: Share your use case 8 | url: https://github.com/couler-proj/couler/blob/master/ADOPTERS.md 9 | about: Add your organization to the list of Couler adopters and talk with the maintainers -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/enhancement_proposal.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Enhancement proposal 3 | about: Propose an enhancement for this project 4 | labels: 'enhancement' 5 | --- 6 | # Summary 7 | 8 | What change needs making? 9 | 10 | # Use Cases 11 | 12 | When would you use this? 13 | 14 | --- 15 | 16 | **Message from the maintainers**: 17 | 18 | Impacted by this bug? Give it a 👍. We prioritize the issues with the most 👍. 19 | -------------------------------------------------------------------------------- /.github/pull_request_template: -------------------------------------------------------------------------------- 1 | 10 | 11 | ### What changes were proposed in this pull request? 12 | 16 | 17 | 18 | ### Why are the changes needed? 19 | 24 | 25 | 26 | ### Does this PR introduce _any_ user-facing change? 27 | 32 | 33 | 34 | ### How was this patch tested? 35 | 40 | -------------------------------------------------------------------------------- /.github/semantic.yml: -------------------------------------------------------------------------------- 1 | # Always validate the PR title, and ignore the commits 2 | titleOnly: true 3 | -------------------------------------------------------------------------------- /.github/workflows/build.yml: -------------------------------------------------------------------------------- 1 | name: CI 2 | 3 | on: 4 | push: 5 | branches: [ master, dev-* ] 6 | pull_request: 7 | branches: [ master ] 8 | 9 | jobs: 10 | build: 11 | 12 | runs-on: ubuntu-18.04 13 | env: 14 | ACTIONS_ALLOW_UNSECURE_COMMANDS: 'true' 15 | strategy: 16 | matrix: 17 | python-version: [3.6.7] 18 | 19 | steps: 20 | - uses: actions/checkout@v2 21 | - name: Set up Python ${{ matrix.python-version }} 22 | uses: actions/setup-python@v2 23 | with: 24 | python-version: ${{ matrix.python-version }} 25 | - name: Setup Go 26 | uses: actions/setup-go@v2 27 | with: 28 | go-version: '1.15.0' # The Go version to download (if necessary) and use. 29 | - name: Install dependencies 30 | run: | 31 | python -m pip install --upgrade pip 32 | python -m pip install -r requirements.txt -r requirements-dev.txt 33 | go get github.com/golang/protobuf/protoc-gen-go@v1.3.2 34 | go get golang.org/x/lint/golint 35 | go get github.com/argoproj/argo@v0.0.0-20210125193418-4cb5b7eb8075 36 | - name: Install protoc 37 | uses: arduino/setup-protoc@v1 38 | with: 39 | version: '3.14.0' 40 | ## TODO: This is temporarily commented out to unblock PRs. 41 | # - name: Sanity checks 42 | # run: | 43 | # pre-commit run -a --show-diff-on-failure 44 | - name: Python Unit tests 45 | run: | 46 | set -e 47 | bash ./scripts/test_python.sh 48 | - name: Build docs 49 | run: | 50 | mkdocs build 51 | - name: Go Unit tests 52 | run: | 53 | set -e 54 | bash ./scripts/test_go.sh 55 | - uses: opsgang/ga-setup-minikube@v0.1.1 56 | with: 57 | minikube-version: 1.22.0 58 | k8s-version: 1.18.3 59 | - name: Integration tests 60 | run: | 61 | minikube config set vm-driver docker 62 | minikube config set kubernetes-version 1.18.3 63 | minikube start 64 | 65 | kubectl create ns argo 66 | kubectl create sa default -n argo 67 | kubectl apply -n argo -f https://raw.githubusercontent.com/argoproj/argo-workflows/v2.12.6/manifests/quick-start-minimal.yaml 68 | kubectl wait -n argo --for=condition=Ready pods --all --timeout=300s 69 | 70 | kubectl apply -n argo -f manifests/mpi-operator.yaml 71 | 72 | go build -buildmode=c-shared -o submit.so go/couler/commands/submit.go 73 | scripts/integration_tests.sh 74 | export E2E_TEST=true 75 | go test -timeout 3m ./go/couler/submitter/... -v 76 | -------------------------------------------------------------------------------- /.github/workflows/docs.yml: -------------------------------------------------------------------------------- 1 | name: docs 2 | on: 3 | push: 4 | branches: 5 | - master 6 | jobs: 7 | deploy: 8 | runs-on: ubuntu-18.04 9 | steps: 10 | - uses: actions/checkout@v2 11 | - uses: actions/setup-python@v2 12 | with: 13 | python-version: 3.6.7 14 | - run: python -m pip install -r requirements.txt -r requirements-dev.txt 15 | - run: mkdocs gh-deploy --force 16 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Mac 2 | .DS_Store 3 | 4 | # Byte-compiled / optimized / DLL files 5 | __pycache__ 6 | *.py[cod] 7 | *$py.class 8 | 9 | # C extensions 10 | *.so 11 | *.h 12 | 13 | # Distribution / packaging 14 | .Python 15 | build/ 16 | develop-eggs/ 17 | dist/ 18 | downloads/ 19 | eggs/ 20 | .eggs/ 21 | lib/ 22 | lib64/ 23 | parts/ 24 | sdist/ 25 | var/ 26 | wheels/ 27 | *.egg-info/ 28 | .installed.cfg 29 | *.egg 30 | MANIFEST 31 | node_modules 32 | 33 | # PyInstaller 34 | # Usually these files are written by a python script from a template 35 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 36 | *.manifest 37 | *.spec 38 | 39 | # Installer logs 40 | pip-log.txt 41 | pip-delete-this-directory.txt 42 | 43 | # Unit test / coverage reports 44 | htmlcov/ 45 | .coverage 46 | .coverage.* 47 | .cache 48 | nosetests.xml 49 | coverage.xml 50 | *.cover 51 | .hypothesis/ 52 | .pytest_cache/ 53 | .cache 54 | 55 | # Sphinx documentation 56 | docs/_build/ 57 | 58 | # Jupyter Notebook 59 | .ipynb_checkpoints 60 | 61 | # Environments 62 | env 63 | env3 64 | .env 65 | .venv 66 | env/ 67 | venv/ 68 | ENV/ 69 | env.bak/ 70 | venv.bak/ 71 | .python-version 72 | 73 | # Editor files 74 | .*project 75 | *.swp 76 | *.swo 77 | *.idea 78 | *.vscode 79 | *.iml 80 | 81 | # mkdocs documentation 82 | /site 83 | 84 | # mypy 85 | .mypy_cache/ 86 | 87 | # java targets 88 | target/ 89 | 90 | # R notebooks 91 | .Rproj.user 92 | example/tutorial/R/*.nb.html 93 | 94 | # travis_wait command logs 95 | travis_wait*.log 96 | -------------------------------------------------------------------------------- /.isort.cfg: -------------------------------------------------------------------------------- 1 | [settings] 2 | multi_line_output=3 3 | line_length=79 4 | known_third_party=docker,google,kubernetes,pyaml,setuptools,strgen,stringcase,yaml 5 | include_trailing_comma=True 6 | -------------------------------------------------------------------------------- /.pre-commit-config.yaml: -------------------------------------------------------------------------------- 1 | repos: 2 | - repo: https://github.com/Lucas-C/pre-commit-hooks 3 | rev: v1.1.9 4 | hooks: 5 | - id: insert-license 6 | name: Add license for all python files 7 | exclude: ^(\.github|examples)/.*$ 8 | types: [python] 9 | args: 10 | - --comment-style 11 | - "|#|" 12 | - --license-filepath 13 | - templates/LICENSE.txt 14 | - --fuzzy-match-generates-todo 15 | - repo: https://github.com/asottile/seed-isort-config 16 | rev: v1.9.1 17 | hooks: 18 | - id: seed-isort-config 19 | - repo: https://github.com/pre-commit/mirrors-isort 20 | rev: v4.3.20 21 | hooks: 22 | - id: isort 23 | - repo: https://github.com/ambv/black 24 | rev: 19.3b0 25 | hooks: 26 | - id: black 27 | args: [--line-length=79] 28 | - repo: https://github.com/pre-commit/pre-commit-hooks 29 | rev: v2.2.3 30 | hooks: 31 | - id: flake8 32 | - repo: git://github.com/dnephin/pre-commit-golang 33 | rev: v0.3.5 34 | hooks: 35 | - id: go-fmt 36 | - id: go-lint 37 | - id: no-go-testing 38 | 39 | exclude: 'couler/proto/*' 40 | -------------------------------------------------------------------------------- /ADOPTERS.md: -------------------------------------------------------------------------------- 1 | # Adopters of Couler 2 | 3 | This page contains a list of organizations who are using Couler. If you'd like to be included here, please send a pull request which modifies this file. 4 | 5 | 6 | 1. [Ant Group](https://www.antgroup.com/) 7 | 1. [Bytedance](https://www.bytedance.com/) 8 | 1. [Determined](https://determined.ai/) 9 | 1. [FreeWheel](https://freewheel.com/) 10 | 1. [Konnecto](https://www.konnecto.com/) 11 | 1. [Onepanel](https://docs.onepanel.ai/) 12 | 1. [Pipekit](https://pipekit.io/) 13 | 1. [PITS Globale Datenrettungsdienste](https://www.pitsdatenrettung.de/) 14 | 1. [SQLFlow](https://github.com/sql-machine-learning/sqlflow) 15 | -------------------------------------------------------------------------------- /AUTHORS: -------------------------------------------------------------------------------- 1 | # This is the official list of Couler authors for copyright purposes. 2 | # This file is distinct from the CONTRIBUTORS files. 3 | # See the latter for an explanation. 4 | 5 | # Names should be added to this file as: 6 | # Name or Organization 7 | # The email address is not required for organizations. 8 | # Please keep the list in alphabetical order. 9 | 10 | Ant Group 11 | Chao Huang 12 | Jiang Qian 13 | Bo Sang 14 | Mingjie Tang 15 | Yuan Tang 16 | Yi Wang 17 | Wei Yan -------------------------------------------------------------------------------- /CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- 1 | # Contributor Covenant Code of Conduct 2 | 3 | ## Our Pledge 4 | 5 | In the interest of fostering an open and welcoming environment, we as 6 | contributors and maintainers pledge to making participation in our project and 7 | our community a harassment-free experience for everyone, regardless of age, body 8 | size, disability, ethnicity, sex characteristics, gender identity and expression, 9 | level of experience, education, socio-economic status, nationality, personal 10 | appearance, race, religion, or sexual identity and orientation. 11 | 12 | ## Our Standards 13 | 14 | Examples of behavior that contributes to creating a positive environment 15 | include: 16 | 17 | * Using welcoming and inclusive language 18 | * Being respectful of differing viewpoints and experiences 19 | * Gracefully accepting constructive criticism 20 | * Focusing on what is best for the community 21 | * Showing empathy towards other community members 22 | 23 | Examples of unacceptable behavior by participants include: 24 | 25 | * The use of sexualized language or imagery and unwelcome sexual attention or 26 | advances 27 | * Trolling, insulting/derogatory comments, and personal or political attacks 28 | * Public or private harassment 29 | * Publishing others' private information, such as a physical or electronic 30 | address, without explicit permission 31 | * Other conduct which could reasonably be considered inappropriate in a 32 | professional setting 33 | 34 | ## Our Responsibilities 35 | 36 | Project maintainers are responsible for clarifying the standards of acceptable 37 | behavior and are expected to take appropriate and fair corrective action in 38 | response to any instances of unacceptable behavior. 39 | 40 | Project maintainers have the right and responsibility to remove, edit, or 41 | reject comments, commits, code, wiki edits, issues, and other contributions 42 | that are not aligned to this Code of Conduct, or to ban temporarily or 43 | permanently any contributor for other behaviors that they deem inappropriate, 44 | threatening, offensive, or harmful. 45 | 46 | ## Scope 47 | 48 | This Code of Conduct applies both within project spaces and in public spaces 49 | when an individual is representing the project or its community. Examples of 50 | representing a project or community include using an official project e-mail 51 | address, posting via an official social media account, or acting as an appointed 52 | representative at an online or offline event. Representation of a project may be 53 | further defined and clarified by project maintainers. 54 | 55 | ## Enforcement 56 | 57 | Instances of abusive, harassing, or otherwise unacceptable behavior may be 58 | reported by contacting the project team at terrytangyuan@gmail.com. All 59 | complaints will be reviewed and investigated and will result in a response that 60 | is deemed necessary and appropriate to the circumstances. The project team is 61 | obligated to maintain confidentiality with regard to the reporter of an incident. 62 | Further details of specific enforcement policies may be posted separately. 63 | 64 | Project maintainers who do not follow or enforce the Code of Conduct in good 65 | faith may face temporary or permanent repercussions as determined by other 66 | members of the project's leadership. 67 | 68 | ## Attribution 69 | 70 | This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4, 71 | available at https://www.contributor-covenant.org/version/1/4/code-of-conduct.html 72 | 73 | [homepage]: https://www.contributor-covenant.org 74 | 75 | For answers to common questions about this code of conduct, see 76 | https://www.contributor-covenant.org/faq 77 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contributing Guide 2 | 3 | Welcome to Couler's contributing guide! 4 | 5 | ## Install Dependencies 6 | 7 | You can install all the dependent Python packages for development via the following: 8 | 9 | ```bash 10 | python -m pip install --upgrade pip 11 | python -m pip install -r requirements.txt -r requirements-dev.txt 12 | ``` 13 | 14 | ## Run Unit Tests 15 | 16 | You can execute all the unit tests via the following command: 17 | 18 | ```bash 19 | python setup.py install 20 | python -m pytest 21 | ``` 22 | 23 | ## Run Integration Tests 24 | 25 | The current integration test suite requires: 26 | 27 | - [kubectl](https://kubernetes.io/docs/tasks/tools/install-kubectl/) 28 | - [minikube](https://kubernetes.io/docs/tasks/tools/install-minikube/) 29 | 30 | Star a k8s cluster using minikube: 31 | 32 | ```sh 33 | minikube config set vm-driver docker 34 | minikube config set kubernetes-version 1.18.3 35 | minikube start 36 | ``` 37 | 38 | Install Argo Workflows: 39 | 40 | ```sh 41 | kubectl create ns argo 42 | kubectl apply -n argo -f https://raw.githubusercontent.com/argoproj/argo/v2.11.1/manifests/quick-start-minimal.yaml 43 | ``` 44 | 45 | Run the integration tests: 46 | ```sh 47 | scripts/integration_tests.sh 48 | ``` 49 | 50 | ## Run Sanity Checks 51 | 52 | We use [pre-commit](https://github.com/pre-commit/pre-commit) to check issues on code style and quality. For example, it 53 | runs [black](https://github.com/psf/black) for automatic Python code formatting which should fix most of the issues automatically. 54 | You can execute the following command to run all the sanity checks: 55 | 56 | ```bash 57 | pre-commit run --all 58 | ``` 59 | 60 | ## Run the Documentation Server 61 | 62 | If you have modified the documentation, you may want to run the documentation server 63 | locally to preview the changes. 64 | Couler uses [Material for MkDocs](https://squidfunk.github.io/mkdocs-material/) to build the documentation. 65 | You can run the following command to start a local documentation server: 66 | 67 | ```sh 68 | mkdocs serve 69 | ``` 70 | 71 | This will start the documentation server on the port 8000 by default. 72 | 73 | ## Sign the Contributor License Agreement (CLA) 74 | 75 | If you haven't signed the CLA yet, [@CLAassistant](https://github.com/CLAassistant) will notify you on your pull request. 76 | Then you can simply follow the provided instructions on the pull request and sign the CLA using your GitHub account. 77 | 78 | For your convenience, the content of the CLA can be found [here](https://gist.github.com/terrytangyuan/806ec0627ec54cdf92512936996da986). 79 | -------------------------------------------------------------------------------- /CONTRIBUTORS: -------------------------------------------------------------------------------- 1 | # This is the list of Couler's contributors, including the contributors 2 | # before we open sourced Couler. 3 | 4 | # This does not necessarily list everyone who has contributed code, 5 | # especially since many employees of one corporation may be contributing. 6 | # To see the full list of contributors, see the revision history on 7 | # GitHub. If you've made significant contributions and would like to 8 | # be included here, please submit a pull request to add your name to 9 | # this file. 10 | 11 | # Names should be added to this file as: 12 | # Name or Organization 13 | # The email address is not required for organizations. 14 | # Please keep the list in alphabetical order. 15 | 16 | Bai Huang 17 | Hengda Qi 18 | Yitao Shen 19 | Xiao Xu 20 | Xu Yan 21 | Yi Zhang -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include requirements.txt 2 | -------------------------------------------------------------------------------- /RELEASE.md: -------------------------------------------------------------------------------- 1 | # Couler Releases 2 | 3 | Please see the list of releases [here](https://github.com/couler-proj/couler/releases). 4 | -------------------------------------------------------------------------------- /SECURITY.md: -------------------------------------------------------------------------------- 1 | # Security Policy 2 | 3 | ## Reporting a Vulnerability 4 | 5 | Please report security vulnerabilities by [filing issues on GitHub](https://github.com/couler-proj/couler/issues/new/choose) 6 | or [contacting one of the maintainers](https://github.com/orgs/couler-proj/teams/couler-team). 7 | 8 | ## Public Disclosure 9 | 10 | All security vulnerabilities will be disclosed via [RELEASE.md](RELEASE.md). 11 | 12 | ## Vulnerability Scanning 13 | 14 | See the [pre-commit configuration file](.pre-commit-config.yaml) that includes a list of static code analysis. 15 | -------------------------------------------------------------------------------- /couler/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2021 The Couler Authors. All rights reserved. 2 | # Licensed under the Apache License, Version 2.0 (the "License"); 3 | # you may not use this file except in compliance with the License. 4 | # You may obtain a copy of the License at 5 | # 6 | # http://www.apache.org/licenses/LICENSE-2.0 7 | # 8 | # Unless required by applicable law or agreed to in writing, software 9 | # distributed under the License is distributed on an "AS IS" BASIS, 10 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | # See the License for the specific language governing permissions and 12 | # limitations under the License. 13 | 14 | from couler.argo import * # noqa: F401, F403 15 | -------------------------------------------------------------------------------- /couler/_version.py: -------------------------------------------------------------------------------- 1 | # Copyright 2021 The Couler Authors. All rights reserved. 2 | # Licensed under the Apache License, Version 2.0 (the "License"); 3 | # you may not use this file except in compliance with the License. 4 | # You may obtain a copy of the License at 5 | # 6 | # http://www.apache.org/licenses/LICENSE-2.0 7 | # 8 | # Unless required by applicable law or agreed to in writing, software 9 | # distributed under the License is distributed on an "AS IS" BASIS, 10 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | # See the License for the specific language governing permissions and 12 | # limitations under the License. 13 | 14 | __version__ = "0.1.1rc8" 15 | -------------------------------------------------------------------------------- /couler/core/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2021 The Couler Authors. All rights reserved. 2 | # Licensed under the Apache License, Version 2.0 (the "License"); 3 | # you may not use this file except in compliance with the License. 4 | # You may obtain a copy of the License at 5 | # 6 | # http://www.apache.org/licenses/LICENSE-2.0 7 | # 8 | # Unless required by applicable law or agreed to in writing, software 9 | # distributed under the License is distributed on an "AS IS" BASIS, 10 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | # See the License for the specific language governing permissions and 12 | # limitations under the License. 13 | -------------------------------------------------------------------------------- /couler/core/constants.py: -------------------------------------------------------------------------------- 1 | # Copyright 2021 The Couler Authors. All rights reserved. 2 | # Licensed under the Apache License, Version 2.0 (the "License"); 3 | # you may not use this file except in compliance with the License. 4 | # You may obtain a copy of the License at 5 | # 6 | # http://www.apache.org/licenses/LICENSE-2.0 7 | # 8 | # Unless required by applicable law or agreed to in writing, software 9 | # distributed under the License is distributed on an "AS IS" BASIS, 10 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | # See the License for the specific language governing permissions and 12 | # limitations under the License. 13 | 14 | from enum import Enum 15 | 16 | # For cpu-only containers, we need to overwrite these envs 17 | OVERWRITE_GPU_ENVS = { 18 | "NVIDIA_VISIBLE_DEVICES": "", 19 | "NVIDIA_DRIVER_CAPABILITIES": "", 20 | } 21 | 22 | 23 | class WorkflowCRD(object): 24 | PLURAL = "workflows" 25 | KIND = "Workflow" 26 | GROUP = "argoproj.io" 27 | VERSION = "v1alpha1" 28 | NAME_MAX_LENGTH = 45 29 | NAME_PATTERN = r"[a-z]([-a-z0-9]*[a-z0-9])?" 30 | 31 | 32 | class CronWorkflowCRD(WorkflowCRD): 33 | PLURAL = "cronworkflows" 34 | KIND = "CronWorkflow" 35 | 36 | 37 | class ImagePullPolicy(Enum): 38 | IfNotPresent = "IfNotPresent" 39 | Always = "Always" 40 | Never = "Never" 41 | 42 | @classmethod 43 | def valid(cls, value: str) -> bool: 44 | return value in cls.__members__ 45 | 46 | @classmethod 47 | def values(cls) -> list: 48 | return list(cls.__members__.keys()) 49 | 50 | 51 | class WFStatus(Enum): 52 | Succeeded = "Succeeded" 53 | Failed = "Failed" 54 | Error = "Error" 55 | 56 | 57 | class ArtifactType(object): 58 | LOCAL = "local" 59 | S3 = "s3" 60 | OSS = "oss" 61 | -------------------------------------------------------------------------------- /couler/core/states.py: -------------------------------------------------------------------------------- 1 | # Copyright 2021 The Couler Authors. All rights reserved. 2 | # Licensed under the Apache License, Version 2.0 (the "License"); 3 | # you may not use this file except in compliance with the License. 4 | # You may obtain a copy of the License at 5 | # 6 | # http://www.apache.org/licenses/LICENSE-2.0 7 | # 8 | # Unless required by applicable law or agreed to in writing, software 9 | # distributed under the License is distributed on an "AS IS" BASIS, 10 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | # See the License for the specific language governing permissions and 12 | # limitations under the License. 13 | 14 | from collections import OrderedDict 15 | 16 | from strgen import StringGenerator 17 | from stringcase import spinalcase 18 | 19 | from couler.core import utils 20 | from couler.core.templates import Workflow 21 | 22 | try: 23 | from couler.core.proto_repr import cleanup_proto_workflow 24 | except Exception: 25 | # set cleanup_proto_workflow to an empty function for compatibility 26 | cleanup_proto_workflow = lambda: None # noqa: E731 27 | 28 | _name_salt = StringGenerator(r"[\c\d]{8}") 29 | 30 | 31 | def default_workflow_name_salter(name): 32 | # The maximum length of a workflow name derives from the 33 | # maximum k8s resource name length (workflows are custom resources). 34 | return "{0}-{1}".format(spinalcase(name), _name_salt.render())[:62] 35 | 36 | 37 | _workflow_name_salter = default_workflow_name_salter 38 | default_service_account = None 39 | 40 | _sub_steps = None 41 | # Argo DAG task 42 | _update_steps_lock = True 43 | _run_concurrent_lock = False 44 | _concurrent_func_line = -1 45 | # Identify concurrent functions have the same name 46 | _concurrent_func_id = 0 47 | # We need to fetch the name before triggering atexit, as the atexit handlers 48 | # cannot get the original Python filename. 49 | workflow_filename = utils.workflow_filename() 50 | workflow = Workflow(workflow_filename=workflow_filename) 51 | # '_when_prefix' represents 'when' prefix in Argo YAML. For example, 52 | # https://github.com/argoproj/argo/blob/master/examples/README.md#conditionals 53 | _when_prefix = None 54 | _when_task = None 55 | # '_condition_id' records the line number where the 'couler.when()' is invoked. 56 | _condition_id = None 57 | # '_while_steps' records the step of recursive logic 58 | _while_steps: OrderedDict = OrderedDict() 59 | # '_while_lock' indicts the recursive call start 60 | _while_lock = False 61 | # dependency edges 62 | _upstream_dag_task = None 63 | # Enhanced depends logic 64 | _upstream_dag_depends_logic = None 65 | # dag function caller line 66 | _dag_caller_line = None 67 | # start exit handler 68 | _exit_handler_enable = False 69 | # step output results 70 | _steps_outputs: OrderedDict = OrderedDict() 71 | _secrets: dict = {} 72 | # for passing the artifact implicitly 73 | _outputs_tmp = None 74 | # print yaml at exit 75 | _enable_print_yaml = True 76 | # Whether to overwrite NVIDIA GPU environment variables 77 | # to containers and templates 78 | _overwrite_nvidia_gpu_envs = False 79 | 80 | 81 | def get_step_output(step_name): 82 | # Return the output as a list by default 83 | return _steps_outputs.get(step_name, None) 84 | 85 | 86 | def get_secret(name: str): 87 | """Get secret by name.""" 88 | return _secrets.get(name, None) 89 | 90 | 91 | def _cleanup(): 92 | """Cleanup the cached fields, just used for unit test. 93 | """ 94 | global _secrets, _update_steps_lock, _dag_caller_line, _upstream_dag_task, _upstream_dag_depends_logic, workflow, _steps_outputs # noqa: E501 95 | global _exit_handler_enable, _when_prefix, _when_task, _while_steps, _concurrent_func_line # noqa: E501 96 | _secrets = {} 97 | _update_steps_lock = True 98 | _dag_caller_line = None 99 | _upstream_dag_task = None 100 | _upstream_dag_depends_logic = None 101 | _exit_handler_enable = False 102 | _when_prefix = None 103 | _when_task = None 104 | _while_steps = OrderedDict() 105 | _concurrent_func_line = -1 106 | _steps_outputs = OrderedDict() 107 | workflow.cleanup() 108 | cleanup_proto_workflow() 109 | -------------------------------------------------------------------------------- /couler/core/syntax/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2021 The Couler Authors. All rights reserved. 2 | # Licensed under the Apache License, Version 2.0 (the "License"); 3 | # you may not use this file except in compliance with the License. 4 | # You may obtain a copy of the License at 5 | # 6 | # http://www.apache.org/licenses/LICENSE-2.0 7 | # 8 | # Unless required by applicable law or agreed to in writing, software 9 | # distributed under the License is distributed on an "AS IS" BASIS, 10 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | # See the License for the specific language governing permissions and 12 | # limitations under the License. 13 | 14 | from couler.core.syntax.concurrent import concurrent # noqa: F401 15 | from couler.core.syntax.conditional import when # noqa: F401 16 | from couler.core.syntax.dag import dag, set_dependencies # noqa: F401 17 | from couler.core.syntax.dns import set_dns # noqa: F401 18 | from couler.core.syntax.exit_handler import set_exit_handler # noqa: F401 19 | from couler.core.syntax.image_pull_secret import ( # noqa: F401 20 | add_image_pull_secret, 21 | ) 22 | from couler.core.syntax.loop import map # noqa: F401 23 | from couler.core.syntax.predicates import * # noqa: F401, F403 24 | from couler.core.syntax.recursion import exec_while # noqa: F401 25 | from couler.core.syntax.toleration import add_toleration # noqa: F401 26 | from couler.core.syntax.volume import ( # noqa: F401 27 | add_volume, 28 | create_workflow_volume, 29 | ) 30 | -------------------------------------------------------------------------------- /couler/core/syntax/concurrent.py: -------------------------------------------------------------------------------- 1 | # Copyright 2021 The Couler Authors. All rights reserved. 2 | # Licensed under the Apache License, Version 2.0 (the "License"); 3 | # you may not use this file except in compliance with the License. 4 | # You may obtain a copy of the License at 5 | # 6 | # http://www.apache.org/licenses/LICENSE-2.0 7 | # 8 | # Unless required by applicable law or agreed to in writing, software 9 | # distributed under the License is distributed on an "AS IS" BASIS, 10 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | # See the License for the specific language governing permissions and 12 | # limitations under the License. 13 | 14 | from collections import OrderedDict 15 | 16 | from couler.core import states, utils 17 | from couler.core.step_update_utils import _update_steps 18 | from couler.core.templates import Steps 19 | 20 | 21 | def concurrent(function_list, subtasks=False): 22 | """ 23 | Start different jobs at the same time 24 | subtasks: each function F of function_list contains multiple steps. 25 | Then, for each F, we create a sub-steps template. 26 | """ 27 | if not isinstance(function_list, list): 28 | raise SyntaxError("require input functions as list") 29 | 30 | _, con_caller_line = utils.invocation_location() 31 | 32 | states._concurrent_func_line = con_caller_line 33 | states._run_concurrent_lock = True 34 | 35 | function_rets = [] 36 | for function in function_list: 37 | # In case different parallel steps use the same function name 38 | states._concurrent_func_id = states._concurrent_func_id + 1 39 | if callable(function): 40 | if subtasks is True: 41 | # 1. generate the sub-steps template 42 | # 2. for each step in F, update the sub_steps template 43 | # 3. append the steps into the template 44 | # 4. for F itself, update the main control flow step 45 | states._sub_steps = OrderedDict() 46 | tmp_concurrent_func_id = states._concurrent_func_id 47 | states._run_concurrent_lock = False 48 | ret = function() 49 | states._concurrent_func_id = tmp_concurrent_func_id 50 | func_name = "concurrent-task-%s" % states._concurrent_func_id 51 | template = Steps( 52 | name=func_name, steps=list(states._sub_steps.values()) 53 | ) 54 | states.workflow.add_template(template) 55 | states._sub_steps = None 56 | # TODO: add the args for the sub task 57 | states._run_concurrent_lock = True 58 | _update_steps( 59 | "concurrent_func_name", 60 | con_caller_line, 61 | args=None, 62 | template_name=func_name, 63 | ) 64 | else: 65 | ret = function() 66 | 67 | function_rets.append(ret) 68 | else: 69 | raise TypeError("require loop over a function to run") 70 | 71 | states._run_concurrent_lock = False 72 | states._concurrent_func_id = 0 73 | 74 | return function_rets 75 | -------------------------------------------------------------------------------- /couler/core/syntax/conditional.py: -------------------------------------------------------------------------------- 1 | # Copyright 2021 The Couler Authors. All rights reserved. 2 | # Licensed under the Apache License, Version 2.0 (the "License"); 3 | # you may not use this file except in compliance with the License. 4 | # You may obtain a copy of the License at 5 | # 6 | # http://www.apache.org/licenses/LICENSE-2.0 7 | # 8 | # Unless required by applicable law or agreed to in writing, software 9 | # distributed under the License is distributed on an "AS IS" BASIS, 10 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | # See the License for the specific language governing permissions and 12 | # limitations under the License. 13 | 14 | 15 | from couler.core import states 16 | from couler.core.templates import Step, output 17 | 18 | 19 | def when(condition, function): 20 | """Generates an Argo conditional step. 21 | For example, the coinflip example in 22 | https://github.com/argoproj/argo/blob/master/examples/coinflip.yaml. 23 | """ 24 | pre = condition["pre"] 25 | post = condition["post"] 26 | if pre is None or post is None: 27 | raise SyntaxError("Output of function can not be null") 28 | 29 | condition_suffix = condition["condition"] 30 | 31 | pre_dict = output.extract_step_return(pre) 32 | post_dict = output.extract_step_return(post) 33 | 34 | if "name" in pre_dict: 35 | left_function_id = pre_dict["id"] 36 | if states.workflow.get_step(left_function_id) is None: 37 | states.workflow.add_step( 38 | name=left_function_id, 39 | step=Step(name=left_function_id, template=pre_dict["name"]), 40 | ) 41 | else: 42 | # TODO: fixed if left branch is a variable rather than function 43 | pre_dict["value"] 44 | 45 | post_value = post_dict["value"] 46 | 47 | if states._upstream_dag_task is not None: 48 | step_type = "tasks" 49 | states._when_task = pre_dict["id"] 50 | else: 51 | step_type = "steps" 52 | states._when_prefix = "{{%s.%s.%s}} %s %s" % ( 53 | step_type, 54 | pre_dict["id"], 55 | pre_dict["output"], 56 | condition_suffix, 57 | post_value, 58 | ) 59 | states._condition_id = "%s.%s" % (pre_dict["id"], pre_dict["output"]) 60 | 61 | # Enforce the function to run and lock to add into step 62 | if callable(function): 63 | function() 64 | else: 65 | raise TypeError("condition to run would be a function") 66 | 67 | states._when_prefix = None 68 | states._condition_id = None 69 | -------------------------------------------------------------------------------- /couler/core/syntax/dag.py: -------------------------------------------------------------------------------- 1 | # Copyright 2021 The Couler Authors. All rights reserved. 2 | # Licensed under the Apache License, Version 2.0 (the "License"); 3 | # you may not use this file except in compliance with the License. 4 | # You may obtain a copy of the License at 5 | # 6 | # http://www.apache.org/licenses/LICENSE-2.0 7 | # 8 | # Unless required by applicable law or agreed to in writing, software 9 | # distributed under the License is distributed on an "AS IS" BASIS, 10 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | # See the License for the specific language governing permissions and 12 | # limitations under the License. 13 | 14 | 15 | from couler.core import states, utils 16 | from couler.core.templates import OutputArtifact, OutputJob, OutputParameter 17 | 18 | 19 | def dag(dependency_graph): 20 | """ 21 | Generate a DAG of Argo YAML 22 | Note: couler.set_dependencies() is more preferable. 23 | https://github.com/argoproj/argo/blob/master/examples/dag-coinflip.yaml 24 | """ 25 | if not isinstance(dependency_graph, list): 26 | raise SyntaxError("require input as list") 27 | 28 | states.workflow.enable_dag_mode() 29 | 30 | _, call_line = utils.invocation_location() 31 | 32 | states._dag_caller_line = call_line 33 | 34 | for edges in dependency_graph: 35 | states._upstream_dag_task = None 36 | if isinstance(edges, list): 37 | for node in edges: 38 | if callable(node): 39 | node() 40 | else: 41 | raise TypeError("require loop over a function to run") 42 | 43 | 44 | def set_dependencies(step_function, dependencies=None): 45 | """ 46 | :param step_function: step to run 47 | :param dependencies: the list of dependencies of this step. This can be in 48 | either of the following forms: 49 | 1. a list of step names; 50 | 2. a string representing the enhanced depends logic that specifies 51 | dependencies based on their statuses. See the link below for the 52 | supported syntax: 53 | https://github.com/argoproj/argo/blob/master/docs/enhanced-depends-logic.md 54 | :return: 55 | """ 56 | 57 | if dependencies is not None: 58 | if isinstance(dependencies, list): 59 | # A list of dependencies 60 | states._upstream_dag_task = dependencies 61 | states._upstream_dag_depends_logic = None 62 | elif isinstance(dependencies, str): 63 | # Dependencies using enhanced depends logic 64 | states._upstream_dag_depends_logic = dependencies 65 | states._upstream_dag_task = None 66 | else: 67 | raise SyntaxError("dependencies must be a list or a string") 68 | else: 69 | states._upstream_dag_depends_logic = None 70 | states._upstream_dag_task = None 71 | if not callable(step_function): 72 | raise SyntaxError("require step_function to a function") 73 | 74 | states.workflow.enable_dag_mode() 75 | 76 | states._outputs_tmp = [] 77 | if dependencies is not None and isinstance(dependencies, list): 78 | for step in dependencies: 79 | output = states.get_step_output(step) 80 | 81 | for o in output: 82 | if isinstance(o, (OutputArtifact, OutputParameter, OutputJob)): 83 | states._outputs_tmp.append(o) 84 | 85 | ret = step_function() 86 | states._outputs_tmp = None 87 | return ret 88 | -------------------------------------------------------------------------------- /couler/core/syntax/dns.py: -------------------------------------------------------------------------------- 1 | # Copyright 2021 The Couler Authors. All rights reserved. 2 | # Licensed under the Apache License, Version 2.0 (the "License"); 3 | # you may not use this file except in compliance with the License. 4 | # You may obtain a copy of the License at 5 | # 6 | # http://www.apache.org/licenses/LICENSE-2.0 7 | # 8 | # Unless required by applicable law or agreed to in writing, software 9 | # distributed under the License is distributed on an "AS IS" BASIS, 10 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | # See the License for the specific language governing permissions and 12 | # limitations under the License. 13 | 14 | from couler.core import states 15 | from couler.core.templates.dns import DnsConfig 16 | 17 | 18 | def set_dns(dns_policy: str, dns_config: DnsConfig): 19 | """ 20 | Set dns policy and config to the workflow. 21 | https://argoproj.github.io/argo-workflows/fields/#poddnsconfig 22 | """ 23 | states.workflow.set_dns(dns_policy, dns_config) 24 | -------------------------------------------------------------------------------- /couler/core/syntax/exit_handler.py: -------------------------------------------------------------------------------- 1 | # Copyright 2021 The Couler Authors. All rights reserved. 2 | # Licensed under the Apache License, Version 2.0 (the "License"); 3 | # you may not use this file except in compliance with the License. 4 | # You may obtain a copy of the License at 5 | # 6 | # http://www.apache.org/licenses/LICENSE-2.0 7 | # 8 | # Unless required by applicable law or agreed to in writing, software 9 | # distributed under the License is distributed on an "AS IS" BASIS, 10 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | # See the License for the specific language governing permissions and 12 | # limitations under the License. 13 | 14 | 15 | from couler.core import states 16 | from couler.core.constants import WFStatus 17 | 18 | 19 | def set_exit_handler(status, exit_handler): 20 | """ 21 | Configure the workflow handler 22 | Status would be: Succeeded, Failed, or Error. 23 | Each status invokes one exit_handler function. 24 | https://github.com/argoproj/argo/blob/master/examples/exit-handlers.yaml 25 | """ 26 | if not callable(exit_handler): 27 | raise SyntaxError("require exit handler is a function") 28 | 29 | if not isinstance(status, WFStatus): # noqa: F405 30 | raise SyntaxError( 31 | "require input status to be Succeeded, Failed or Error" 32 | ) 33 | 34 | workflow_status = "{{workflow.status}} == %s" % status.value 35 | 36 | states._exit_handler_enable = True 37 | states._when_prefix = workflow_status 38 | 39 | branch = exit_handler() 40 | if branch is None: 41 | raise SyntaxError("require function return value") 42 | 43 | states._when_prefix = None 44 | states._exit_handler_enable = False 45 | -------------------------------------------------------------------------------- /couler/core/syntax/image_pull_secret.py: -------------------------------------------------------------------------------- 1 | # Copyright 2021 The Couler Authors. All rights reserved. 2 | # Licensed under the Apache License, Version 2.0 (the "License"); 3 | # you may not use this file except in compliance with the License. 4 | # You may obtain a copy of the License at 5 | # 6 | # http://www.apache.org/licenses/LICENSE-2.0 7 | # 8 | # Unless required by applicable law or agreed to in writing, software 9 | # distributed under the License is distributed on an "AS IS" BASIS, 10 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | # See the License for the specific language governing permissions and 12 | # limitations under the License. 13 | 14 | from couler.core import states 15 | from couler.core.templates.image_pull_secret import ImagePullSecret 16 | 17 | 18 | def add_image_pull_secret(image_pull_secret: ImagePullSecret): 19 | """ 20 | Add image pull secret to the workflow. 21 | """ 22 | states.workflow.add_image_pull_secret(image_pull_secret) 23 | -------------------------------------------------------------------------------- /couler/core/syntax/predicates.py: -------------------------------------------------------------------------------- 1 | # Copyright 2021 The Couler Authors. All rights reserved. 2 | # Licensed under the Apache License, Version 2.0 (the "License"); 3 | # you may not use this file except in compliance with the License. 4 | # You may obtain a copy of the License at 5 | # 6 | # http://www.apache.org/licenses/LICENSE-2.0 7 | # 8 | # Unless required by applicable law or agreed to in writing, software 9 | # distributed under the License is distributed on an "AS IS" BASIS, 10 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | # See the License for the specific language governing permissions and 12 | # limitations under the License. 13 | 14 | 15 | def _predicate(pre, post, condition): 16 | """Generates an Argo predicate. 17 | """ 18 | dict_config = {} 19 | if callable(pre): 20 | dict_config["pre"] = pre() 21 | else: 22 | dict_config["pre"] = pre 23 | 24 | if callable(post): 25 | dict_config["post"] = post() 26 | else: 27 | dict_config["post"] = post 28 | 29 | # TODO: check the condition 30 | dict_config["condition"] = condition 31 | 32 | return dict_config 33 | 34 | 35 | def equal(pre, post=None): 36 | if post is not None: 37 | return _predicate(pre, post, "==") 38 | else: 39 | return _predicate(pre, None, "==") 40 | 41 | 42 | def not_equal(pre, post=None): 43 | if post is not None: 44 | return _predicate(pre, post, "!=") 45 | else: 46 | return _predicate(pre, None, "!=") 47 | 48 | 49 | def bigger(pre, post=None): 50 | if post is not None: 51 | return _predicate(pre, post, ">") 52 | else: 53 | return _predicate(pre, None, ">") 54 | 55 | 56 | def smaller(pre, post=None): 57 | if post is not None: 58 | return _predicate(pre, post, "<") 59 | else: 60 | return _predicate(pre, None, "<") 61 | 62 | 63 | def bigger_equal(pre, post=None): 64 | if post is not None: 65 | return _predicate(pre, post, ">=") 66 | else: 67 | return _predicate(pre, None, ">=") 68 | 69 | 70 | def smaller_equal(pre, post=None): 71 | if post is not None: 72 | return _predicate(pre, post, "<=") 73 | else: 74 | return _predicate(pre, None, "<=") 75 | -------------------------------------------------------------------------------- /couler/core/syntax/recursion.py: -------------------------------------------------------------------------------- 1 | # Copyright 2021 The Couler Authors. All rights reserved. 2 | # Licensed under the Apache License, Version 2.0 (the "License"); 3 | # you may not use this file except in compliance with the License. 4 | # You may obtain a copy of the License at 5 | # 6 | # http://www.apache.org/licenses/LICENSE-2.0 7 | # 8 | # Unless required by applicable law or agreed to in writing, software 9 | # distributed under the License is distributed on an "AS IS" BASIS, 10 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | # See the License for the specific language governing permissions and 12 | # limitations under the License. 13 | 14 | from collections import OrderedDict 15 | 16 | from couler.core import states, utils 17 | from couler.core.templates import Step, Steps, output 18 | 19 | 20 | def exec_while(condition, inner_while): 21 | """ 22 | Generate the Argo recursive logic. For example 23 | https://github.com/argoproj/argo/blob/master/examples/README.md#recursion. 24 | """ 25 | # _while_lock means 'exec_while' operation begins to work 26 | # _while_steps stores logic steps inside the recursion logic 27 | states._while_lock = True 28 | 29 | # Enforce inner function of the while-loop to run 30 | if callable(inner_while): 31 | branch = inner_while() 32 | if branch is None: 33 | raise SyntaxError("require function return value") 34 | else: 35 | raise TypeError("condition to run would be a function") 36 | 37 | branch_dict = output.extract_step_return(branch) 38 | recursive_name = "exec-while-" + branch_dict["name"] 39 | recursive_id = "exec-while-" + branch_dict["id"] 40 | if states.workflow.get_template(recursive_name) is None: 41 | template = Steps(name=recursive_name) 42 | else: 43 | raise SyntaxError("Recursive function can not be called twice ") 44 | 45 | # Generate leaving point for recursive 46 | step_out_name = "%s-%s" % (recursive_name, "exit") 47 | pre = condition["pre"] 48 | pre_dict = output.extract_step_return(pre) 49 | condition_suffix = condition["condition"] 50 | 51 | # Generate the recursive go to step 52 | when_prefix = "{{steps.%s.%s}} %s %s" % ( 53 | branch_dict["id"], 54 | branch_dict["output"], 55 | condition_suffix, 56 | pre_dict["value"], 57 | ) 58 | step_out_template = OrderedDict( 59 | { 60 | "name": step_out_name, 61 | "template": recursive_name, 62 | "when": when_prefix, 63 | } 64 | ) 65 | step_out_id = utils.invocation_name(step_out_name, recursive_id) 66 | states._while_steps[step_out_id] = [step_out_template] 67 | 68 | # Add steps inside the recursive logic to recursive template 69 | template.steps = list(states._while_steps.values()) 70 | 71 | # Add this recursive logic to the templates 72 | states.workflow.add_template(template) 73 | 74 | # Add recursive logic to global _steps 75 | recursive_out_step = Step(name=recursive_id, template=recursive_name) 76 | states.workflow.add_step(name=recursive_id, step=recursive_out_step) 77 | 78 | states._while_lock = False 79 | states._while_steps = OrderedDict() 80 | -------------------------------------------------------------------------------- /couler/core/syntax/toleration.py: -------------------------------------------------------------------------------- 1 | # Copyright 2021 The Couler Authors. All rights reserved. 2 | # Licensed under the Apache License, Version 2.0 (the "License"); 3 | # you may not use this file except in compliance with the License. 4 | # You may obtain a copy of the License at 5 | # 6 | # http://www.apache.org/licenses/LICENSE-2.0 7 | # 8 | # Unless required by applicable law or agreed to in writing, software 9 | # distributed under the License is distributed on an "AS IS" BASIS, 10 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | # See the License for the specific language governing permissions and 12 | # limitations under the License. 13 | 14 | from couler.core import states 15 | from couler.core.templates.toleration import Toleration 16 | 17 | 18 | def add_toleration(toleration: Toleration): 19 | """ 20 | Add toleration to the workflow. 21 | """ 22 | states.workflow.add_toleration(toleration) 23 | -------------------------------------------------------------------------------- /couler/core/syntax/volume.py: -------------------------------------------------------------------------------- 1 | # Copyright 2021 The Couler Authors. All rights reserved. 2 | # Licensed under the Apache License, Version 2.0 (the "License"); 3 | # you may not use this file except in compliance with the License. 4 | # You may obtain a copy of the License at 5 | # 6 | # http://www.apache.org/licenses/LICENSE-2.0 7 | # 8 | # Unless required by applicable law or agreed to in writing, software 9 | # distributed under the License is distributed on an "AS IS" BASIS, 10 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | # See the License for the specific language governing permissions and 12 | # limitations under the License. 13 | 14 | from couler.core import states 15 | from couler.core.templates.volume import Volume 16 | from couler.core.templates.volume_claim import VolumeClaimTemplate 17 | 18 | 19 | def add_volume(volume: Volume): 20 | """ 21 | Add existing volume to the workflow. 22 | 23 | Reference: 24 | https://github.com/argoproj/argo/blob/master/examples/volumes-existing.yaml 25 | """ 26 | states.workflow.add_volume(volume) 27 | 28 | 29 | def create_workflow_volume(volume_claim_template: VolumeClaimTemplate): 30 | """ 31 | Create a transient volume for use in the workflow. 32 | 33 | Reference: 34 | https://github.com/argoproj/argo/blob/master/examples/volumes-pvc.yaml 35 | """ 36 | states.workflow.add_pvc_template(volume_claim_template) 37 | -------------------------------------------------------------------------------- /couler/core/templates/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2021 The Couler Authors. All rights reserved. 2 | # Licensed under the Apache License, Version 2.0 (the "License"); 3 | # you may not use this file except in compliance with the License. 4 | # You may obtain a copy of the License at 5 | # 6 | # http://www.apache.org/licenses/LICENSE-2.0 7 | # 8 | # Unless required by applicable law or agreed to in writing, software 9 | # distributed under the License is distributed on an "AS IS" BASIS, 10 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | # See the License for the specific language governing permissions and 12 | # limitations under the License. 13 | 14 | from couler.core.templates.artifact import ( # noqa: F401 15 | Artifact, 16 | LocalArtifact, 17 | OssArtifact, 18 | S3Artifact, 19 | TypedArtifact, 20 | ) 21 | from couler.core.templates.cache import Cache # noqa: F401 22 | from couler.core.templates.container import Container # noqa: F401 23 | from couler.core.templates.job import Job # noqa: F401 24 | from couler.core.templates.output import ( # noqa: F401 25 | Output, 26 | OutputArtifact, 27 | OutputEmpty, 28 | OutputJob, 29 | OutputParameter, 30 | OutputScript, 31 | ) 32 | from couler.core.templates.script import Script # noqa: F401 33 | from couler.core.templates.secret import Secret # noqa: F401 34 | from couler.core.templates.step import Step, Steps # noqa: F401 35 | from couler.core.templates.template import Template # noqa: F401 36 | from couler.core.templates.workflow import Workflow # noqa: F401 37 | -------------------------------------------------------------------------------- /couler/core/templates/cache.py: -------------------------------------------------------------------------------- 1 | # Copyright 2021 The Couler Authors. All rights reserved. 2 | # Licensed under the Apache License, Version 2.0 (the "License"); 3 | # you may not use this file except in compliance with the License. 4 | # You may obtain a copy of the License at 5 | # 6 | # http://www.apache.org/licenses/LICENSE-2.0 7 | # 8 | # Unless required by applicable law or agreed to in writing, software 9 | # distributed under the License is distributed on an "AS IS" BASIS, 10 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | # See the License for the specific language governing permissions and 12 | # limitations under the License. 13 | 14 | from collections import OrderedDict 15 | 16 | 17 | class Cache(object): 18 | def __init__(self, name, key, max_age=""): 19 | self.name = name 20 | self.key = key 21 | self.max_age = max_age 22 | 23 | def to_dict(self): 24 | d = OrderedDict( 25 | { 26 | "key": self.key, 27 | "maxAge": self.max_age, 28 | "cache": {"configMap": {"name": self.name}}, 29 | } 30 | ) 31 | return d 32 | -------------------------------------------------------------------------------- /couler/core/templates/dns.py: -------------------------------------------------------------------------------- 1 | # Copyright 2021 The Couler Authors. All rights reserved. 2 | # Licensed under the Apache License, Version 2.0 (the "License"); 3 | # you may not use this file except in compliance with the License. 4 | # You may obtain a copy of the License at 5 | # 6 | # http://www.apache.org/licenses/LICENSE-2.0 7 | # 8 | # Unless required by applicable law or agreed to in writing, software 9 | # distributed under the License is distributed on an "AS IS" BASIS, 10 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | # See the License for the specific language governing permissions and 12 | # limitations under the License. 13 | 14 | from collections import OrderedDict 15 | 16 | from couler.core import utils 17 | 18 | 19 | class DnsConfig(object): 20 | def __init__(self, nameservers=None, options=None, searches=None): 21 | self.nameservers = nameservers 22 | self.options = options 23 | self.searches = searches 24 | 25 | def to_dict(self): 26 | dt = OrderedDict() 27 | if utils.non_empty(self.nameservers): 28 | dt.update({"nameservers": self.nameservers}) 29 | if utils.non_empty(self.options): 30 | dt.update({"options": self.get_options_to_dict()}) 31 | if utils.non_empty(self.searches): 32 | dt.update({"searches": self.searches}) 33 | return dt 34 | 35 | def get_options_to_dict(self): 36 | t = [] 37 | for option in self.options: 38 | t.append(option.to_dict()) 39 | return t 40 | 41 | 42 | class DnsConfigOption(object): 43 | def __init__(self, name, value): 44 | self.name = name 45 | self.value = value 46 | 47 | def to_dict(self): 48 | return OrderedDict({"name": self.name, "value": self.value}) 49 | -------------------------------------------------------------------------------- /couler/core/templates/image_pull_secret.py: -------------------------------------------------------------------------------- 1 | # Copyright 2021 The Couler Authors. All rights reserved. 2 | # Licensed under the Apache License, Version 2.0 (the "License"); 3 | # you may not use this file except in compliance with the License. 4 | # You may obtain a copy of the License at 5 | # 6 | # http://www.apache.org/licenses/LICENSE-2.0 7 | # 8 | # Unless required by applicable law or agreed to in writing, software 9 | # distributed under the License is distributed on an "AS IS" BASIS, 10 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | # See the License for the specific language governing permissions and 12 | # limitations under the License. 13 | 14 | from collections import OrderedDict 15 | 16 | 17 | class ImagePullSecret(object): 18 | def __init__(self, name): 19 | self.name = name 20 | 21 | def to_dict(self): 22 | return OrderedDict({"name": self.name}) 23 | -------------------------------------------------------------------------------- /couler/core/templates/job.py: -------------------------------------------------------------------------------- 1 | # Copyright 2021 The Couler Authors. All rights reserved. 2 | # Licensed under the Apache License, Version 2.0 (the "License"); 3 | # you may not use this file except in compliance with the License. 4 | # You may obtain a copy of the License at 5 | # 6 | # http://www.apache.org/licenses/LICENSE-2.0 7 | # 8 | # Unless required by applicable law or agreed to in writing, software 9 | # distributed under the License is distributed on an "AS IS" BASIS, 10 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | # See the License for the specific language governing permissions and 12 | # limitations under the License. 13 | 14 | from collections import OrderedDict 15 | 16 | from couler.core import utils 17 | from couler.core.templates.template import Template 18 | 19 | 20 | class Job(Template): 21 | def __init__( 22 | self, 23 | name, 24 | args, 25 | action, 26 | manifest, 27 | set_owner_reference, 28 | success_condition, 29 | failure_condition, 30 | timeout=None, 31 | retry=None, 32 | pool=None, 33 | cache=None, 34 | ): 35 | Template.__init__( 36 | self, 37 | name=name, 38 | timeout=timeout, 39 | retry=retry, 40 | pool=pool, 41 | cache=cache, 42 | ) 43 | self.args = args 44 | self.action = action 45 | self.manifest = manifest 46 | self.set_owner_reference = utils.bool_to_str(set_owner_reference) 47 | self.success_condition = success_condition 48 | self.failure_condition = failure_condition 49 | 50 | def to_dict(self): 51 | template = Template.to_dict(self) 52 | if utils.non_empty(self.args): 53 | template["inputs"] = {"parameters": self.args} 54 | template["resource"] = self.resource_dict() 55 | 56 | # Append outputs to this template 57 | # return the resource job name, job ID, and job object by default 58 | job_outputs = [ 59 | OrderedDict( 60 | { 61 | "name": "job-name", 62 | "valueFrom": {"jsonPath": '"{.metadata.name}"'}, 63 | } 64 | ), 65 | OrderedDict( 66 | { 67 | "name": "job-id", 68 | "valueFrom": {"jsonPath": '"{.metadata.uid}"'}, 69 | } 70 | ), 71 | OrderedDict({"name": "job-obj", "valueFrom": {"jqFilter": '"."'}}), 72 | ] 73 | template["outputs"] = {"parameters": job_outputs} 74 | return template 75 | 76 | def resource_dict(self): 77 | resource = OrderedDict( 78 | { 79 | "action": self.action, 80 | "setOwnerReference": self.set_owner_reference, 81 | "manifest": self.manifest, 82 | } 83 | ) 84 | if self.success_condition: 85 | resource["successCondition"] = self.success_condition 86 | if self.failure_condition: 87 | resource["failureCondition"] = self.failure_condition 88 | return resource 89 | -------------------------------------------------------------------------------- /couler/core/templates/secret.py: -------------------------------------------------------------------------------- 1 | # Copyright 2021 The Couler Authors. All rights reserved. 2 | # Licensed under the Apache License, Version 2.0 (the "License"); 3 | # you may not use this file except in compliance with the License. 4 | # You may obtain a copy of the License at 5 | # 6 | # http://www.apache.org/licenses/LICENSE-2.0 7 | # 8 | # Unless required by applicable law or agreed to in writing, software 9 | # distributed under the License is distributed on an "AS IS" BASIS, 10 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | # See the License for the specific language governing permissions and 12 | # limitations under the License. 13 | 14 | import hashlib 15 | import json 16 | from collections import OrderedDict 17 | 18 | from couler.core import utils 19 | 20 | 21 | class Secret(object): 22 | def __init__( 23 | self, namespace, data, name=None, dry_run=False, use_existing=False, artifact_secret=False 24 | ): 25 | 26 | if not isinstance(data, dict): 27 | raise TypeError("The secret data is required to be a dict") 28 | if not data: 29 | raise ValueError("The secret data is empty") 30 | 31 | self.namespace = namespace 32 | # TO avoid create duplicate secret 33 | cypher_md5 = hashlib.md5( 34 | json.dumps(data, sort_keys=True).encode("utf-8") 35 | ).hexdigest() 36 | if name is None: 37 | self.name = "couler-%s" % cypher_md5 38 | else: 39 | self.name = name 40 | 41 | self.data = {k: utils.encode_base64(v) for k, v in data.items()} 42 | self.dry_run = dry_run 43 | self.use_existing = use_existing 44 | self.artifact_secret = artifact_secret 45 | 46 | def to_yaml(self): 47 | """Covert the secret to a secret CRD specification.""" 48 | d = OrderedDict( 49 | { 50 | "apiVersion": "v1", 51 | "kind": "Secret", 52 | "metadata": {"name": self.name, "namespace": self.namespace} if self.namespace != "default" else {"name": self.name}, 53 | "type": "Opaque", 54 | "data": {}, 55 | } 56 | ) 57 | for k, v in self.data.items(): 58 | d["data"][k] = v 59 | return d 60 | 61 | def to_env_list(self): 62 | """ 63 | Convert the secret to an environment list, and can be attached to 64 | containers. 65 | """ 66 | secret_envs = [] 67 | for key, _ in self.data.items(): 68 | secret_env = { 69 | "name": key, 70 | "valueFrom": {"secretKeyRef": {"name": self.name, "key": key}}, 71 | } 72 | secret_envs.append(secret_env) 73 | return secret_envs 74 | -------------------------------------------------------------------------------- /couler/core/templates/step.py: -------------------------------------------------------------------------------- 1 | # Copyright 2021 The Couler Authors. All rights reserved. 2 | # Licensed under the Apache License, Version 2.0 (the "License"); 3 | # you may not use this file except in compliance with the License. 4 | # You may obtain a copy of the License at 5 | # 6 | # http://www.apache.org/licenses/LICENSE-2.0 7 | # 8 | # Unless required by applicable law or agreed to in writing, software 9 | # distributed under the License is distributed on an "AS IS" BASIS, 10 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | # See the License for the specific language governing permissions and 12 | # limitations under the License. 13 | 14 | from collections import OrderedDict 15 | 16 | from couler.core import utils 17 | from couler.core.templates.template import Template 18 | 19 | 20 | class Step(object): 21 | def __init__( 22 | self, name, template=None, arguments=None, when=None, with_items=None 23 | ): 24 | self.name = name 25 | self.template = template 26 | self.arguments = arguments 27 | self.with_items = with_items 28 | self.when = when 29 | 30 | def to_dict(self): 31 | d = OrderedDict({"name": self.name}) 32 | if self.template is not None: 33 | d.update({"template": self.template}) 34 | if self.when is not None: 35 | d.update({"when": self.when}) 36 | if utils.non_empty(self.arguments): 37 | d.update({"arguments": self.arguments}) 38 | if utils.non_empty(self.with_items): 39 | d.update({"withItems": self.with_items}) 40 | return d 41 | 42 | 43 | class Steps(Template): 44 | def __init__(self, name, steps=None): 45 | Template.__init__(self, name=name) 46 | self.steps = steps 47 | 48 | def to_dict(self): 49 | template = Template.to_dict(self) 50 | template["steps"] = self.steps 51 | return template 52 | -------------------------------------------------------------------------------- /couler/core/templates/template.py: -------------------------------------------------------------------------------- 1 | # Copyright 2021 The Couler Authors. All rights reserved. 2 | # Licensed under the Apache License, Version 2.0 (the "License"); 3 | # you may not use this file except in compliance with the License. 4 | # You may obtain a copy of the License at 5 | # 6 | # http://www.apache.org/licenses/LICENSE-2.0 7 | # 8 | # Unless required by applicable law or agreed to in writing, software 9 | # distributed under the License is distributed on an "AS IS" BASIS, 10 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | # See the License for the specific language governing permissions and 12 | # limitations under the License. 13 | 14 | from collections import OrderedDict 15 | 16 | from couler.core import utils 17 | 18 | 19 | class Template(object): 20 | def __init__( 21 | self, 22 | name, 23 | output=None, 24 | input=None, 25 | timeout=None, 26 | retry=None, 27 | pool=None, 28 | enable_ulogfs=True, 29 | daemon=False, 30 | cache=None, 31 | parallelism=None, 32 | ): 33 | self.name = name 34 | self.output = output 35 | self.input = input 36 | self.timeout = timeout 37 | self.retry = retry 38 | self.pool = pool 39 | self.enable_ulogfs = enable_ulogfs 40 | self.daemon = daemon 41 | self.cache = cache 42 | self.parallelism: int = parallelism 43 | 44 | def to_dict(self): 45 | template = OrderedDict({"name": self.name}) 46 | if self.daemon: 47 | template["daemon"] = True 48 | if self.timeout is not None: 49 | template["activeDeadlineSeconds"] = self.timeout 50 | if self.retry is not None: 51 | template["retryStrategy"] = utils.config_retry_strategy(self.retry) 52 | if self.cache is not None: 53 | template["memoize"] = self.cache.to_dict() 54 | if self.parallelism is not None: 55 | template["parallelism"] = self.parallelism 56 | return template 57 | -------------------------------------------------------------------------------- /couler/core/templates/toleration.py: -------------------------------------------------------------------------------- 1 | # Copyright 2021 The Couler Authors. All rights reserved. 2 | # Licensed under the Apache License, Version 2.0 (the "License"); 3 | # you may not use this file except in compliance with the License. 4 | # You may obtain a copy of the License at 5 | # 6 | # http://www.apache.org/licenses/LICENSE-2.0 7 | # 8 | # Unless required by applicable law or agreed to in writing, software 9 | # distributed under the License is distributed on an "AS IS" BASIS, 10 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | # See the License for the specific language governing permissions and 12 | # limitations under the License. 13 | 14 | from collections import OrderedDict 15 | 16 | 17 | class Toleration(object): 18 | def __init__(self, key, effect, operator): 19 | self.key = key 20 | self.effect = effect 21 | self.operator = operator 22 | 23 | def to_dict(self): 24 | return OrderedDict( 25 | {"key": self.key, "effect": self.effect, "operator": self.operator} 26 | ) 27 | -------------------------------------------------------------------------------- /couler/core/templates/volume.py: -------------------------------------------------------------------------------- 1 | # Copyright 2021 The Couler Authors. All rights reserved. 2 | # Licensed under the Apache License, Version 2.0 (the "License"); 3 | # you may not use this file except in compliance with the License. 4 | # You may obtain a copy of the License at 5 | # 6 | # http://www.apache.org/licenses/LICENSE-2.0 7 | # 8 | # Unless required by applicable law or agreed to in writing, software 9 | # distributed under the License is distributed on an "AS IS" BASIS, 10 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | # See the License for the specific language governing permissions and 12 | # limitations under the License. 13 | 14 | from collections import OrderedDict 15 | 16 | 17 | class Volume(object): 18 | def __init__(self, name, claim_name): 19 | self.name = name 20 | self.claim_name = claim_name 21 | 22 | def to_dict(self): 23 | return OrderedDict( 24 | { 25 | "name": self.name, 26 | "persistentVolumeClaim": {"claimName": self.claim_name}, 27 | } 28 | ) 29 | 30 | 31 | class VolumeMount(object): 32 | def __init__(self, name, mount_path): 33 | self.name = name 34 | self.mount_path = mount_path 35 | 36 | def to_dict(self): 37 | return OrderedDict({"name": self.name, "mountPath": self.mount_path}) 38 | -------------------------------------------------------------------------------- /couler/core/templates/volume_claim.py: -------------------------------------------------------------------------------- 1 | # Copyright 2021 The Couler Authors. All rights reserved. 2 | # Licensed under the Apache License, Version 2.0 (the "License"); 3 | # you may not use this file except in compliance with the License. 4 | # You may obtain a copy of the License at 5 | # 6 | # http://www.apache.org/licenses/LICENSE-2.0 7 | # 8 | # Unless required by applicable law or agreed to in writing, software 9 | # distributed under the License is distributed on an "AS IS" BASIS, 10 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | # See the License for the specific language governing permissions and 12 | # limitations under the License. 13 | 14 | from collections import OrderedDict 15 | from typing import List 16 | 17 | 18 | class VolumeClaimTemplate(object): 19 | def __init__(self, claim_name: str, access_modes: List[str], size: str): 20 | 21 | self.claim_name = claim_name 22 | self.access_modes = access_modes 23 | self.size = size 24 | 25 | def to_dict(self): 26 | return OrderedDict( 27 | { 28 | "metadata": {"name": self.claim_name}, 29 | "spec": { 30 | "accessModes": self.access_modes, 31 | "resources": {"requests": {"storage": self.size}}, 32 | }, 33 | } 34 | ) 35 | -------------------------------------------------------------------------------- /couler/core/workflow_validation_utils.py: -------------------------------------------------------------------------------- 1 | # Copyright 2021 The Couler Authors. All rights reserved. 2 | # Licensed under the Apache License, Version 2.0 (the "License"); 3 | # you may not use this file except in compliance with the License. 4 | # You may obtain a copy of the License at 5 | # 6 | # http://www.apache.org/licenses/LICENSE-2.0 7 | # 8 | # Unless required by applicable law or agreed to in writing, software 9 | # distributed under the License is distributed on an "AS IS" BASIS, 10 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | # See the License for the specific language governing permissions and 12 | # limitations under the License. 13 | 14 | import copy 15 | import json 16 | 17 | try: 18 | from argo.workflows import client 19 | from argo.workflows.client.models import ( 20 | V1alpha1DAGTemplate, 21 | V1alpha1ResourceTemplate, 22 | V1alpha1ScriptTemplate, 23 | V1alpha1WorkflowStep, 24 | ) 25 | 26 | _ARGO_INSTALLED = True 27 | except ImportError: 28 | _ARGO_INSTALLED = False 29 | 30 | 31 | def validate_workflow_yaml(original_wf): 32 | if _ARGO_INSTALLED: 33 | wf = copy.deepcopy(original_wf) 34 | if ( 35 | "spec" not in wf 36 | or "templates" not in wf["spec"] 37 | or len(wf["spec"]["templates"]) <= 0 38 | ): 39 | if wf["kind"] == "CronWorkflow": 40 | if ( 41 | "workflowSpec" not in wf["spec"] 42 | or "templates" not in wf["spec"]["workflowSpec"] 43 | or len(wf["spec"]["workflowSpec"]["templates"]) <= 0 44 | ): 45 | raise Exception( 46 | "CronWorkflow yaml must contain " 47 | "spec.workflowSpec.templates" 48 | ) 49 | else: 50 | raise Exception("Workflow yaml must contain spec.templates") 51 | if wf["kind"] == "CronWorkflow": 52 | templates = wf["spec"]["workflowSpec"]["templates"] 53 | else: 54 | templates = wf["spec"]["templates"] 55 | # Note that currently direct deserialization of `V1alpha1Template` is 56 | # problematic so here we validate them individually instead. 57 | for template in templates: 58 | if "steps" in template: 59 | if template["steps"] is None or len(template["steps"]) <= 0: 60 | raise Exception( 61 | "At least one step definition must exist in steps" 62 | ) 63 | for step in template["steps"]: 64 | _deserialize_wrapper(step, V1alpha1WorkflowStep) 65 | elif "dag" in template: 66 | if ( 67 | template["dag"] is None 68 | or "tasks" not in template["dag"] 69 | or template["dag"]["tasks"] is None 70 | or len(template["dag"]["tasks"]) <= 0 71 | ): 72 | raise Exception( 73 | "At least one task definition must exist in dag.tasks" 74 | ) 75 | _deserialize_wrapper(template["dag"], V1alpha1DAGTemplate) 76 | elif "resource" in template: 77 | _deserialize_wrapper( 78 | template["resource"], V1alpha1ResourceTemplate 79 | ) 80 | elif "script" in template: 81 | _deserialize_wrapper( 82 | template["script"], V1alpha1ScriptTemplate 83 | ) 84 | 85 | 86 | def _deserialize_wrapper(dict_content, response_type): 87 | body = {"data": json.dumps(dict_content)} 88 | attr = type("Response", (), body) 89 | client.ApiClient().deserialize(attr, response_type) 90 | -------------------------------------------------------------------------------- /couler/docker_submitter.py: -------------------------------------------------------------------------------- 1 | # Copyright 2021 The Couler Authors. All rights reserved. 2 | # Licensed under the Apache License, Version 2.0 (the "License"); 3 | # you may not use this file except in compliance with the License. 4 | # You may obtain a copy of the License at 5 | # 6 | # http://www.apache.org/licenses/LICENSE-2.0 7 | # 8 | # Unless required by applicable law or agreed to in writing, software 9 | # distributed under the License is distributed on an "AS IS" BASIS, 10 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | # See the License for the specific language governing permissions and 12 | # limitations under the License. 13 | 14 | import logging 15 | 16 | import docker 17 | 18 | 19 | class DockerSubmitter(object): 20 | @staticmethod 21 | def run_docker_container(image, command): 22 | client = docker.from_env() 23 | container = client.containers.run(image, command, detach=True) 24 | # Streaming the logs 25 | for line in container.logs(stream=True): 26 | logging.info(line.strip()) 27 | -------------------------------------------------------------------------------- /couler/proto/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2021 The Couler Authors. All rights reserved. 2 | # Licensed under the Apache License, Version 2.0 (the "License"); 3 | # you may not use this file except in compliance with the License. 4 | # You may obtain a copy of the License at 5 | # 6 | # http://www.apache.org/licenses/LICENSE-2.0 7 | # 8 | # Unless required by applicable law or agreed to in writing, software 9 | # distributed under the License is distributed on an "AS IS" BASIS, 10 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | # See the License for the specific language governing permissions and 12 | # limitations under the License. -------------------------------------------------------------------------------- /couler/steps/mpi.py: -------------------------------------------------------------------------------- 1 | # Copyright 2021 The Couler Authors. All rights reserved. 2 | # Licensed under the Apache License, Version 2.0 (the "License"); 3 | # you may not use this file except in compliance with the License. 4 | # You may obtain a copy of the License at 5 | # 6 | # http://www.apache.org/licenses/LICENSE-2.0 7 | # 8 | # Unless required by applicable law or agreed to in writing, software 9 | # distributed under the License is distributed on an "AS IS" BASIS, 10 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | # See the License for the specific language governing permissions and 12 | # limitations under the License. 13 | 14 | import copy 15 | import uuid 16 | 17 | import pyaml 18 | 19 | import couler.argo as couler 20 | from couler.core import utils 21 | from couler.steps.pod_utils import _generate_pod_spec 22 | 23 | pod_types = {"Launcher", "Worker"} 24 | 25 | container_template = {"name": "mpi", "image": "", "command": ""} 26 | 27 | pod_template = {"replicas": 1, "template": {"spec": {"containers": []}}} 28 | 29 | manifest_template = { 30 | "apiVersion": '"kubeflow.org/v1"', 31 | "kind": '"MPIJob"', 32 | "metadata": {"name": ""}, 33 | "spec": {"cleanPodPolicy": "", "slotsPerWorker": 1, "mpiReplicaSpecs": {}}, 34 | } 35 | 36 | 37 | def train( 38 | image=None, 39 | command="", 40 | secret=None, 41 | launcher_image=None, 42 | launcher_resources=None, 43 | launcher_command=None, 44 | num_workers=0, 45 | worker_image=None, 46 | worker_resources=None, 47 | worker_command=None, 48 | clean_pod_policy="Running", 49 | timeout=None, 50 | ): 51 | name = "mpi-train-%s" % str(uuid.uuid4()) 52 | success_condition = ( 53 | "status.replicaStatuses.Worker.succeeded == %s" % num_workers 54 | ) 55 | failure_condition = "status.replicaStatuses.Worker.failed > 0" 56 | 57 | manifest = copy.deepcopy(manifest_template) 58 | manifest["metadata"].update({"name": name}) 59 | manifest["spec"].update({"cleanPodPolicy": clean_pod_policy}) 60 | 61 | launcher_image = launcher_image if launcher_image else image 62 | launcher_command = launcher_command if launcher_command else command 63 | 64 | launcher_pod = _generate_pod_spec( 65 | pod_template, 66 | container_template, 67 | allowed_pod_types=pod_types, 68 | pod_type="Launcher", 69 | image=launcher_image, 70 | replicas=1, 71 | secret=secret, 72 | command=launcher_command, 73 | resources=launcher_resources, 74 | ) 75 | 76 | manifest["spec"]["mpiReplicaSpecs"].update({"Launcher": launcher_pod}) 77 | 78 | if num_workers > 0: 79 | worker_image = worker_image if worker_image else image 80 | worker_command = worker_command if worker_command else command 81 | 82 | worker_pod = _generate_pod_spec( 83 | pod_template, 84 | container_template, 85 | allowed_pod_types=pod_types, 86 | pod_type="Worker", 87 | image=worker_image, 88 | replicas=num_workers, 89 | secret=secret, 90 | command=worker_command, 91 | resources=worker_resources, 92 | ) 93 | 94 | manifest["spec"]["mpiReplicaSpecs"].update({"Worker": worker_pod}) 95 | 96 | step_name, _ = utils.invocation_location() 97 | 98 | couler.run_job( 99 | manifest=pyaml.dump(manifest), 100 | success_condition=success_condition, 101 | failure_condition=failure_condition, 102 | step_name=step_name, 103 | timeout=timeout, 104 | ) 105 | -------------------------------------------------------------------------------- /couler/steps/pod_utils.py: -------------------------------------------------------------------------------- 1 | # Copyright 2021 The Couler Authors. All rights reserved. 2 | # Licensed under the Apache License, Version 2.0 (the "License"); 3 | # you may not use this file except in compliance with the License. 4 | # You may obtain a copy of the License at 5 | # 6 | # http://www.apache.org/licenses/LICENSE-2.0 7 | # 8 | # Unless required by applicable law or agreed to in writing, software 9 | # distributed under the License is distributed on an "AS IS" BASIS, 10 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | # See the License for the specific language governing permissions and 12 | # limitations under the License. 13 | 14 | import copy 15 | 16 | import couler.argo as couler 17 | 18 | 19 | def _validate_pod_params(pod_type, allowed_pod_types, image=None, replicas=0): 20 | 21 | if pod_type not in allowed_pod_types: 22 | raise ValueError("Invalid value %s for parameter pod_type." % pod_type) 23 | if replicas == 0: 24 | raise ValueError("Parameter replicas value should be more than 0.") 25 | if image is None: 26 | raise ValueError("Parameter image should not be None.") 27 | if pod_type in ["Master", "Chief", "Launcher"] and replicas > 1: 28 | raise ValueError("Master/Chief/Launcher pod's replicas should be 1.") 29 | 30 | 31 | def _generate_pod_spec( 32 | pod_template, 33 | container_template, 34 | allowed_pod_types, 35 | pod_type=None, 36 | image=None, 37 | replicas=0, 38 | secret=None, 39 | command="", 40 | resources=None, 41 | restart_policy=None, 42 | ): 43 | 44 | _validate_pod_params( 45 | pod_type=pod_type, 46 | allowed_pod_types=allowed_pod_types, 47 | image=image, 48 | replicas=replicas, 49 | ) 50 | 51 | container = copy.deepcopy(container_template) 52 | container.update({"image": image, "command": command}) 53 | 54 | if secret is not None: 55 | secret_envs = couler.states._secrets[secret].to_env_list() 56 | 57 | if "env" not in container.keys(): 58 | container["env"] = secret_envs 59 | else: 60 | container["env"].extend(secret_envs) 61 | 62 | if resources is not None: 63 | # User-defined resource, should be formatted like 64 | # "cpu=1,memory=1024,disk=2048,gpu=1,gpu_type=p100,shared_memory=20480" 65 | try: 66 | kvs = resources.split(",") 67 | limits = {} 68 | for kv in kvs: 69 | k, v = kv.split("=") 70 | if k in ["gpu", "memory", "disk", "shared_memory"]: 71 | v = int(v) 72 | elif k == "cpu": 73 | v = float(v) 74 | 75 | limits[k] = v 76 | 77 | resource_limits = {"limits": limits} 78 | container["resources"] = resource_limits 79 | 80 | except Exception: 81 | raise Exception("Unrecognized resource type %s" % resources) 82 | 83 | pod = copy.deepcopy(pod_template) 84 | pod.update({"replicas": replicas}) 85 | if restart_policy is not None: 86 | pod.update({"restartPolicy": restart_policy}) 87 | pod["template"]["spec"]["containers"].append(container) 88 | 89 | return pod 90 | -------------------------------------------------------------------------------- /couler/steps/pytorch.py: -------------------------------------------------------------------------------- 1 | # Copyright 2021 The Couler Authors. All rights reserved. 2 | # Licensed under the Apache License, Version 2.0 (the "License"); 3 | # you may not use this file except in compliance with the License. 4 | # You may obtain a copy of the License at 5 | # 6 | # http://www.apache.org/licenses/LICENSE-2.0 7 | # 8 | # Unless required by applicable law or agreed to in writing, software 9 | # distributed under the License is distributed on an "AS IS" BASIS, 10 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | # See the License for the specific language governing permissions and 12 | # limitations under the License. 13 | 14 | import copy 15 | import uuid 16 | 17 | import pyaml 18 | 19 | import couler.argo as couler 20 | from couler.core import utils 21 | from couler.steps.pod_utils import _generate_pod_spec 22 | 23 | pod_types = {"Master", "Worker"} 24 | 25 | container_template = {"name": "pytorch", "image": "", "command": ""} 26 | 27 | pod_template = { 28 | "replicas": 1, 29 | "restartPolicy": "", 30 | "template": {"spec": {"containers": []}}, 31 | } 32 | 33 | manifest_template = { 34 | "apiVersion": '"kubeflow.org/v1"', 35 | "kind": '"PyTorchJob"', 36 | "metadata": {"name": ""}, 37 | "spec": {"cleanPodPolicy": "", "pytorchReplicaSpecs": {}}, 38 | } 39 | 40 | 41 | def train( 42 | image=None, 43 | command="", 44 | secret=None, 45 | master_image=None, 46 | master_resources=None, 47 | master_restart_policy="Never", 48 | master_command=None, 49 | num_workers=0, 50 | worker_image=None, 51 | worker_resources=None, 52 | worker_restart_policy="Never", 53 | worker_command=None, 54 | clean_pod_policy="Running", 55 | timeout=None, 56 | ): 57 | name = "pytorch-train-%s" % str(uuid.uuid4()) 58 | success_condition = ( 59 | "status.replicaStatuses.Worker.succeeded == %s" % num_workers 60 | ) 61 | failure_condition = "status.replicaStatuses.Worker.failed > 0" 62 | 63 | manifest = copy.deepcopy(manifest_template) 64 | manifest["metadata"].update({"name": name}) 65 | manifest["spec"].update({"cleanPodPolicy": clean_pod_policy}) 66 | 67 | master_image = master_image if master_image else image 68 | master_command = master_command if master_command else command 69 | 70 | master_pod = _generate_pod_spec( 71 | pod_template, 72 | container_template, 73 | allowed_pod_types=pod_types, 74 | pod_type="Master", 75 | image=master_image, 76 | replicas=1, 77 | secret=secret, 78 | command=master_command, 79 | resources=master_resources, 80 | restart_policy=master_restart_policy, 81 | ) 82 | 83 | manifest["spec"]["pytorchReplicaSpecs"].update({"Master": master_pod}) 84 | 85 | if num_workers > 0: 86 | worker_image = worker_image if worker_image else image 87 | worker_command = worker_command if worker_command else command 88 | 89 | worker_pod = _generate_pod_spec( 90 | pod_template, 91 | container_template, 92 | allowed_pod_types=pod_types, 93 | pod_type="Worker", 94 | image=worker_image, 95 | replicas=num_workers, 96 | secret=secret, 97 | command=worker_command, 98 | resources=worker_resources, 99 | restart_policy=worker_restart_policy, 100 | ) 101 | 102 | manifest["spec"]["pytorchReplicaSpecs"].update({"Worker": worker_pod}) 103 | 104 | step_name, _ = utils.invocation_location() 105 | 106 | couler.run_job( 107 | manifest=pyaml.dump(manifest), 108 | success_condition=success_condition, 109 | failure_condition=failure_condition, 110 | step_name=step_name, 111 | timeout=timeout, 112 | ) 113 | -------------------------------------------------------------------------------- /couler/tests/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2021 The Couler Authors. All rights reserved. 2 | # Licensed under the Apache License, Version 2.0 (the "License"); 3 | # you may not use this file except in compliance with the License. 4 | # You may obtain a copy of the License at 5 | # 6 | # http://www.apache.org/licenses/LICENSE-2.0 7 | # 8 | # Unless required by applicable law or agreed to in writing, software 9 | # distributed under the License is distributed on an "AS IS" BASIS, 10 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | # See the License for the specific language governing permissions and 12 | # limitations under the License. 13 | import couler 14 | 15 | couler.config_defaults(name_salter=lambda x: x) 16 | -------------------------------------------------------------------------------- /couler/tests/argo_yaml_test.py: -------------------------------------------------------------------------------- 1 | # Copyright 2021 The Couler Authors. All rights reserved. 2 | # Licensed under the Apache License, Version 2.0 (the "License"); 3 | # you may not use this file except in compliance with the License. 4 | # You may obtain a copy of the License at 5 | # 6 | # http://www.apache.org/licenses/LICENSE-2.0 7 | # 8 | # Unless required by applicable law or agreed to in writing, software 9 | # distributed under the License is distributed on an "AS IS" BASIS, 10 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | # See the License for the specific language governing permissions and 12 | # limitations under the License. 13 | 14 | import json 15 | import os 16 | import re 17 | 18 | import pyaml 19 | import yaml 20 | 21 | import couler.argo as couler 22 | from couler.tests.argo_test import ArgoBaseTestCase 23 | 24 | _test_data_dir = "test_data" 25 | 26 | 27 | class ArgoYamlTest(ArgoBaseTestCase): 28 | @staticmethod 29 | def mock_dict(x, mock_str="pytest"): 30 | # The following fields are overwritten so we won't be 31 | # comparing them. This is to avoid issues caused by 32 | # different versions of pytest. 33 | if "generateName" in x["metadata"]: 34 | x["metadata"]["generateName"] = mock_str 35 | x["spec"]["entrypoint"] = mock_str 36 | if "templates" in x["spec"]: 37 | x["spec"]["templates"][0]["name"] = mock_str 38 | return x 39 | 40 | def check_argo_yaml(self, expected_fn): 41 | test_data_dir = os.path.join(os.path.dirname(__file__), _test_data_dir) 42 | with open(os.path.join(test_data_dir, expected_fn), "r") as f: 43 | expected = yaml.safe_load(f) 44 | output = yaml.safe_load( 45 | pyaml.dump(couler.workflow_yaml(), string_val_style="plain") 46 | ) 47 | 48 | def dump(x): 49 | x = re.sub( 50 | r"-[0-9]*", "-***", json.dumps(self.mock_dict(x), indent=2) 51 | ) 52 | return x 53 | 54 | output_j, expected_j = dump(output), dump(expected) 55 | 56 | self.maxDiff = None 57 | self.assertEqual(output_j, expected_j) 58 | 59 | def create_callable_cls(self, func): 60 | class A: 61 | def a(self, *args): 62 | return func(*args) 63 | 64 | @classmethod 65 | def b(cls, *args): 66 | return func(*args) 67 | 68 | @staticmethod 69 | def c(*args): 70 | return func(*args) 71 | 72 | def __call__(self, *args): 73 | return func(*args) 74 | 75 | return A 76 | -------------------------------------------------------------------------------- /couler/tests/cluster_config_test.py: -------------------------------------------------------------------------------- 1 | # Copyright 2021 The Couler Authors. All rights reserved. 2 | # Licensed under the Apache License, Version 2.0 (the "License"); 3 | # you may not use this file except in compliance with the License. 4 | # You may obtain a copy of the License at 5 | # 6 | # http://www.apache.org/licenses/LICENSE-2.0 7 | # 8 | # Unless required by applicable law or agreed to in writing, software 9 | # distributed under the License is distributed on an "AS IS" BASIS, 10 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | # See the License for the specific language governing permissions and 12 | # limitations under the License. 13 | 14 | import os 15 | 16 | import couler.argo as couler 17 | from couler.tests.argo_test import ArgoBaseTestCase 18 | 19 | 20 | class ArgoClusterTest(ArgoBaseTestCase): 21 | def test_cluster_config(self): 22 | 23 | couler.config_workflow( 24 | cluster_config_file=os.path.join( 25 | os.path.dirname(__file__), "test_data/dummy_cluster_config.py" 26 | ) 27 | ) 28 | couler.run_container( 29 | image="docker/whalesay:latest", 30 | args=["echo -n hello world"], 31 | command=["bash", "-c"], 32 | step_name="A", 33 | ) 34 | 35 | wf = couler.workflow_yaml() 36 | self.assertTrue(wf["spec"]["hostNetwork"]) 37 | self.assertEqual(wf["spec"]["templates"][1]["tolerations"], []) 38 | couler._cleanup() 39 | -------------------------------------------------------------------------------- /couler/tests/cron_workflow_test.py: -------------------------------------------------------------------------------- 1 | # Copyright 2021 The Couler Authors. All rights reserved. 2 | # Licensed under the Apache License, Version 2.0 (the "License"); 3 | # you may not use this file except in compliance with the License. 4 | # You may obtain a copy of the License at 5 | # 6 | # http://www.apache.org/licenses/LICENSE-2.0 7 | # 8 | # Unless required by applicable law or agreed to in writing, software 9 | # distributed under the License is distributed on an "AS IS" BASIS, 10 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | # See the License for the specific language governing permissions and 12 | # limitations under the License. 13 | 14 | import couler.argo as couler 15 | from couler.tests.argo_yaml_test import ArgoYamlTest 16 | 17 | _test_data_dir = "test_data" 18 | 19 | 20 | class CronJobTest(ArgoYamlTest): 21 | def test_cron_workflow(self): 22 | def tails(): 23 | return couler.run_container( 24 | image="python:3.6", 25 | command=["bash", "-c", 'echo "run schedule job"'], 26 | ) 27 | 28 | tails() 29 | 30 | # schedule to run at one minute past midnight (00:01) every day 31 | cron_config = {"schedule": "1 0 * * *", "suspend": "false"} 32 | 33 | couler.config_workflow(name="pytest", cron_config=cron_config) 34 | 35 | self.check_argo_yaml("cron_workflow_golden.yaml") 36 | -------------------------------------------------------------------------------- /couler/tests/daemon_step_test.py: -------------------------------------------------------------------------------- 1 | # Copyright 2021 The Couler Authors. All rights reserved. 2 | # Licensed under the Apache License, Version 2.0 (the "License"); 3 | # you may not use this file except in compliance with the License. 4 | # You may obtain a copy of the License at 5 | # 6 | # http://www.apache.org/licenses/LICENSE-2.0 7 | # 8 | # Unless required by applicable law or agreed to in writing, software 9 | # distributed under the License is distributed on an "AS IS" BASIS, 10 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | # See the License for the specific language governing permissions and 12 | # limitations under the License. 13 | 14 | import couler.argo as couler 15 | from couler.tests.argo_test import ArgoBaseTestCase 16 | 17 | 18 | class DaemonStepTest(ArgoBaseTestCase): 19 | def setUp(self) -> None: 20 | couler._cleanup() 21 | 22 | def tearDown(self): 23 | couler._cleanup() 24 | 25 | def test_run_daemon_container(self): 26 | self.assertEqual(len(couler.workflow.templates), 0) 27 | couler.run_container( 28 | image="python:3.6", command="echo $uname", daemon=True 29 | ) 30 | self.assertEqual(len(couler.workflow.templates), 1) 31 | template = couler.workflow.get_template( 32 | "test-run-daemon-container" 33 | ).to_dict() 34 | self.assertEqual("test-run-daemon-container", template["name"]) 35 | self.assertTrue(template["daemon"]) 36 | self.assertEqual("python:3.6", template["container"]["image"]) 37 | self.assertEqual(["echo $uname"], template["container"]["command"]) 38 | 39 | def test_run_daemon_script(self): 40 | self.assertEqual(len(couler.workflow.templates), 0) 41 | couler.run_script( 42 | image="python:3.6", command="bash", source="ls", daemon=True 43 | ) 44 | self.assertEqual(len(couler.workflow.templates), 1) 45 | template = couler.workflow.get_template( 46 | "test-run-daemon-script" 47 | ).to_dict() 48 | self.assertEqual("test-run-daemon-script", template["name"]) 49 | self.assertTrue(template["daemon"]) 50 | self.assertEqual("python:3.6", template["script"]["image"]) 51 | self.assertEqual(["bash"], template["script"]["command"]) 52 | self.assertEqual("ls", template["script"]["source"]) 53 | -------------------------------------------------------------------------------- /couler/tests/input_parameter_test.py: -------------------------------------------------------------------------------- 1 | # Copyright 2021 The Couler Authors. All rights reserved. 2 | # Licensed under the Apache License, Version 2.0 (the "License"); 3 | # you may not use this file except in compliance with the License. 4 | # You may obtain a copy of the License at 5 | # 6 | # http://www.apache.org/licenses/LICENSE-2.0 7 | # 8 | # Unless required by applicable law or agreed to in writing, software 9 | # distributed under the License is distributed on an "AS IS" BASIS, 10 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | # See the License for the specific language governing permissions and 12 | # limitations under the License. 13 | 14 | from collections import OrderedDict 15 | 16 | import couler.argo as couler 17 | from couler.tests.argo_yaml_test import ArgoYamlTest 18 | 19 | couler.config_workflow(name="pytest") 20 | 21 | 22 | def random_code(): 23 | import random 24 | 25 | result = random.randint(0, 1) 26 | print(result) 27 | 28 | 29 | def generate_number(): 30 | return couler.run_script(image="python:3.6", source=random_code) 31 | 32 | 33 | def whalesay(hello_param): 34 | return couler.run_container( 35 | image="docker/whalesay", command=["cowsay"], args=hello_param 36 | ) 37 | 38 | 39 | def whalesay_two(hello_param1, hello_param2): 40 | return couler.run_container( 41 | image="docker/whalesay", 42 | command=["cowsay"], 43 | args=[hello_param1, hello_param2], 44 | ) 45 | 46 | 47 | class InputParametersTest(ArgoYamlTest): 48 | def test_input_basic(self): 49 | whalesay("hello1") 50 | inputs = couler.workflow.get_template("whalesay").to_dict()["inputs"] 51 | expected_inputs = OrderedDict( 52 | [("parameters", [{"name": "para-whalesay-0"}])] 53 | ) 54 | self.assertEqual(inputs, expected_inputs) 55 | 56 | def test_input_basic_two_calls(self): 57 | whalesay("hello1") 58 | whalesay("hello2") 59 | 60 | self.check_argo_yaml("input_para_golden_1.yaml") 61 | 62 | def test_input_basic_two_paras_2(self): 63 | whalesay_two("hello1", "hello2") 64 | whalesay_two("x", "y") 65 | 66 | self.check_argo_yaml("input_para_golden_2.yaml") 67 | 68 | def test_input_steps_1(self): 69 | message = "test" 70 | whalesay(message) 71 | message = generate_number() 72 | whalesay(message) 73 | 74 | self.check_argo_yaml("input_para_golden_3.yaml") 75 | 76 | def test_input_arg_as_string(self): 77 | def whalesay(hello_param): 78 | return couler.run_container( 79 | image="docker/whalesay", command=["cowsay"], args=hello_param 80 | ) 81 | 82 | whalesay("test") 83 | 84 | self.check_argo_yaml("input_para_golden_4.yaml") 85 | 86 | def test_input_args_as_othertypes(self): 87 | def whalesay(para_integer, para_boolean, para_float): 88 | return couler.run_container( 89 | image="docker/whalesay", 90 | command=["cowsay"], 91 | args=[para_integer, para_boolean, para_float], 92 | ) 93 | 94 | whalesay(1, True, 1.1) 95 | 96 | wf = couler.workflow_yaml() 97 | parameters = wf["spec"]["templates"][0]["steps"][0][0]["arguments"][ 98 | "parameters" 99 | ] 100 | self.assertEqual(int(parameters[0]["value"].strip(" " "' ")), 1) 101 | self.assertEqual(bool(parameters[1]["value"].strip(" " "' ")), True) 102 | self.assertEqual(float(parameters[2]["value"].strip(" " "' ")), 1.1) 103 | -------------------------------------------------------------------------------- /couler/tests/katib_step_test.py: -------------------------------------------------------------------------------- 1 | # Copyright 2021 The Couler Authors. All rights reserved. 2 | # Licensed under the Apache License, Version 2.0 (the "License"); 3 | # you may not use this file except in compliance with the License. 4 | # You may obtain a copy of the License at 5 | # 6 | # http://www.apache.org/licenses/LICENSE-2.0 7 | # 8 | # Unless required by applicable law or agreed to in writing, software 9 | # distributed under the License is distributed on an "AS IS" BASIS, 10 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | # See the License for the specific language governing permissions and 12 | # limitations under the License. 13 | 14 | import couler.argo as couler 15 | import couler.steps.katib as katib 16 | from couler.tests.argo_test import ArgoBaseTestCase 17 | 18 | xgb_cmd = """ \ 19 | python /opt/katib/train_xgboost.py \ 20 | --train_dataset train.txt \ 21 | --validation_dataset validation.txt \ 22 | --booster gbtree \ 23 | --objective binary:logistic \ 24 | """ 25 | 26 | xgboost_mainifest_template = """ 27 | apiVersion: batch/v1 28 | kind: Job 29 | metadata: 30 | name: {{{{.Trial}}}} 31 | namespace: {{{{.NameSpace}}}} 32 | spec: 33 | template: 34 | spec: 35 | containers: 36 | - name: {{{{.Trial}}}} 37 | image: docker.io/katib/xgboost:v0.1 38 | command: 39 | - "{command}" 40 | {{{{- with .HyperParameters}}}} 41 | {{{{- range .}}}} 42 | - "{{{{.Name}}}}={{{{.Value}}}}" 43 | {{{{- end}}}} 44 | {{{{- end}}}} 45 | restartPolicy: Never 46 | """.format( 47 | command=xgb_cmd 48 | ) 49 | 50 | 51 | class KatibTest(ArgoBaseTestCase): 52 | def test_katib_with_xgboost_training(self): 53 | katib.run( 54 | tuning_params=[ 55 | {"name": "max_depth", "type": "int", "range": [2, 10]}, 56 | {"name": "num_round", "type": "int", "range": [50, 100]}, 57 | ], 58 | objective={ 59 | "type": "maximize", 60 | "goal": 1.01, 61 | "metric_name": "accuracy", 62 | }, 63 | success_condition="status.trialsSucceeded > 4", 64 | failure_condition="status.trialsFailed > 3", 65 | algorithm="random", 66 | raw_template=xgboost_mainifest_template, 67 | parallel_trial_count=4, 68 | max_trial_count=16, 69 | max_failed_trial_count=3, 70 | ) 71 | 72 | wf = couler.workflow_yaml() 73 | 74 | self.assertEqual(len(wf["spec"]["templates"]), 2) 75 | # Check steps template 76 | template0 = wf["spec"]["templates"][0] 77 | self.assertEqual(len(template0["steps"]), 1) 78 | self.assertEqual(len(template0["steps"][0]), 1) 79 | # Check train template 80 | template1 = wf["spec"]["templates"][1] 81 | self.assertTrue(template1["name"] in ["run", "test-run-python-script"]) 82 | -------------------------------------------------------------------------------- /couler/tests/mpi_step_test.py: -------------------------------------------------------------------------------- 1 | # Copyright 2021 The Couler Authors. All rights reserved. 2 | # Licensed under the Apache License, Version 2.0 (the "License"); 3 | # you may not use this file except in compliance with the License. 4 | # You may obtain a copy of the License at 5 | # 6 | # http://www.apache.org/licenses/LICENSE-2.0 7 | # 8 | # Unless required by applicable law or agreed to in writing, software 9 | # distributed under the License is distributed on an "AS IS" BASIS, 10 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | # See the License for the specific language governing permissions and 12 | # limitations under the License. 13 | 14 | from io import StringIO 15 | 16 | import yaml 17 | 18 | import couler.argo as couler 19 | import couler.steps.mpi as mpi 20 | from couler.core import utils 21 | from couler.tests.argo_yaml_test import ArgoYamlTest 22 | 23 | 24 | class MPITestCase(ArgoYamlTest): 25 | def test_mpi_train(self): 26 | access_key_secret = {"access_key": "key1234"} 27 | secret = couler.create_secret(secret_data=access_key_secret) 28 | 29 | mpi.train( 30 | num_workers=3, 31 | image="mpi:1.13", 32 | command="python mpi.py", 33 | worker_resources="cpu=0.5,memory=1024", 34 | clean_pod_policy="Running", 35 | secret=secret, 36 | ) 37 | 38 | secret_yaml = list(couler.states._secrets.values())[0].to_yaml() 39 | self.assertEqual( 40 | secret_yaml["data"]["access_key"], utils.encode_base64("key1234") 41 | ) 42 | 43 | wf = couler.workflow_yaml() 44 | self.assertEqual(len(wf["spec"]["templates"]), 2) 45 | # Check steps template 46 | template0 = wf["spec"]["templates"][0] 47 | self.assertEqual(len(template0["steps"]), 1) 48 | self.assertEqual(len(template0["steps"][0]), 1) 49 | # Check train template 50 | template1 = wf["spec"]["templates"][1] 51 | self.assertEqual(template1["name"], "test-mpi-train") 52 | resource = template1["resource"] 53 | self.assertEqual(resource["action"], "create") 54 | self.assertEqual(resource["setOwnerReference"], "true") 55 | self.assertEqual( 56 | resource["successCondition"], 57 | "status.replicaStatuses.Worker.succeeded == 3", 58 | ) 59 | self.assertEqual( 60 | resource["failureCondition"], 61 | "status.replicaStatuses.Worker.failed > 0", 62 | ) 63 | # Check the MPIJob spec 64 | mpi_job = yaml.load( 65 | StringIO(resource["manifest"]), Loader=yaml.FullLoader 66 | ) 67 | self.assertEqual(mpi_job["kind"], "MPIJob") 68 | self.assertEqual(mpi_job["spec"]["cleanPodPolicy"], "Running") 69 | 70 | master = mpi_job["spec"]["mpiReplicaSpecs"]["Launcher"] 71 | self.assertEqual(master["replicas"], 1) 72 | chief_container = master["template"]["spec"]["containers"][0] 73 | self.assertEqual(chief_container["env"][0]["name"], "access_key") 74 | self.assertEqual( 75 | chief_container["env"][0]["valueFrom"]["secretKeyRef"]["name"], 76 | secret_yaml["metadata"]["name"], 77 | ) 78 | 79 | worker = mpi_job["spec"]["mpiReplicaSpecs"]["Worker"] 80 | self.assertEqual(worker["replicas"], 3) 81 | self.assertEqual(len(worker["template"]["spec"]["containers"]), 1) 82 | worker_container = worker["template"]["spec"]["containers"][0] 83 | self.assertEqual(worker_container["image"], "mpi:1.13") 84 | self.assertEqual(worker_container["command"], "python mpi.py") 85 | 86 | worker_container = worker["template"]["spec"]["containers"][0] 87 | self.assertEqual(worker_container["env"][0]["name"], "access_key") 88 | self.assertEqual( 89 | worker_container["env"][0]["valueFrom"]["secretKeyRef"]["name"], 90 | secret_yaml["metadata"]["name"], 91 | ) 92 | self.assertEqual(worker_container["resources"]["limits"]["cpu"], 0.5) 93 | self.assertEqual( 94 | worker_container["resources"]["limits"]["memory"], 1024 95 | ) 96 | -------------------------------------------------------------------------------- /couler/tests/pytorch_step_test.py: -------------------------------------------------------------------------------- 1 | # Copyright 2021 The Couler Authors. All rights reserved. 2 | # Licensed under the Apache License, Version 2.0 (the "License"); 3 | # you may not use this file except in compliance with the License. 4 | # You may obtain a copy of the License at 5 | # 6 | # http://www.apache.org/licenses/LICENSE-2.0 7 | # 8 | # Unless required by applicable law or agreed to in writing, software 9 | # distributed under the License is distributed on an "AS IS" BASIS, 10 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | # See the License for the specific language governing permissions and 12 | # limitations under the License. 13 | 14 | from io import StringIO 15 | 16 | import yaml 17 | 18 | import couler.argo as couler 19 | import couler.steps.pytorch as pytorch 20 | from couler.core import utils 21 | from couler.tests.argo_yaml_test import ArgoYamlTest 22 | 23 | 24 | class PyTorchTestCase(ArgoYamlTest): 25 | def test_pytorch_train(self): 26 | access_key_secret = {"access_key": "key1234"} 27 | secret = couler.create_secret(secret_data=access_key_secret) 28 | 29 | pytorch.train( 30 | num_workers=3, 31 | image="pytorch:1.13", 32 | command="python pytorch.py", 33 | worker_resources="cpu=0.5,memory=1024", 34 | worker_restart_policy="OnFailure", 35 | clean_pod_policy="Running", 36 | secret=secret, 37 | ) 38 | 39 | secret_yaml = list(couler.states._secrets.values())[0].to_yaml() 40 | self.assertEqual( 41 | secret_yaml["data"]["access_key"], utils.encode_base64("key1234") 42 | ) 43 | 44 | wf = couler.workflow_yaml() 45 | self.assertEqual(len(wf["spec"]["templates"]), 2) 46 | # Check steps template 47 | template0 = wf["spec"]["templates"][0] 48 | self.assertEqual(len(template0["steps"]), 1) 49 | self.assertEqual(len(template0["steps"][0]), 1) 50 | # Check train template 51 | template1 = wf["spec"]["templates"][1] 52 | self.assertEqual(template1["name"], "test-pytorch-train") 53 | resource = template1["resource"] 54 | self.assertEqual(resource["action"], "create") 55 | self.assertEqual(resource["setOwnerReference"], "true") 56 | self.assertEqual( 57 | resource["successCondition"], 58 | "status.replicaStatuses.Worker.succeeded == 3", 59 | ) 60 | self.assertEqual( 61 | resource["failureCondition"], 62 | "status.replicaStatuses.Worker.failed > 0", 63 | ) 64 | # Check the PyTorchJob spec 65 | pytorch_job = yaml.load( 66 | StringIO(resource["manifest"]), Loader=yaml.FullLoader 67 | ) 68 | self.assertEqual(pytorch_job["kind"], "PyTorchJob") 69 | self.assertEqual(pytorch_job["spec"]["cleanPodPolicy"], "Running") 70 | 71 | master = pytorch_job["spec"]["pytorchReplicaSpecs"]["Master"] 72 | self.assertEqual(master["replicas"], 1) 73 | chief_container = master["template"]["spec"]["containers"][0] 74 | self.assertEqual(chief_container["env"][0]["name"], "access_key") 75 | self.assertEqual( 76 | chief_container["env"][0]["valueFrom"]["secretKeyRef"]["name"], 77 | secret_yaml["metadata"]["name"], 78 | ) 79 | 80 | worker = pytorch_job["spec"]["pytorchReplicaSpecs"]["Worker"] 81 | self.assertEqual(worker["replicas"], 3) 82 | self.assertEqual(worker["restartPolicy"], "OnFailure") 83 | self.assertEqual(len(worker["template"]["spec"]["containers"]), 1) 84 | worker_container = worker["template"]["spec"]["containers"][0] 85 | self.assertEqual(worker_container["image"], "pytorch:1.13") 86 | self.assertEqual(worker_container["command"], "python pytorch.py") 87 | 88 | worker_container = worker["template"]["spec"]["containers"][0] 89 | self.assertEqual(worker_container["env"][0]["name"], "access_key") 90 | self.assertEqual( 91 | worker_container["env"][0]["valueFrom"]["secretKeyRef"]["name"], 92 | secret_yaml["metadata"]["name"], 93 | ) 94 | self.assertEqual(worker_container["resources"]["limits"]["cpu"], 0.5) 95 | self.assertEqual( 96 | worker_container["resources"]["limits"]["memory"], 1024 97 | ) 98 | -------------------------------------------------------------------------------- /couler/tests/resource_test.py: -------------------------------------------------------------------------------- 1 | # Copyright 2021 The Couler Authors. All rights reserved. 2 | # Licensed under the Apache License, Version 2.0 (the "License"); 3 | # you may not use this file except in compliance with the License. 4 | # You may obtain a copy of the License at 5 | # 6 | # http://www.apache.org/licenses/LICENSE-2.0 7 | # 8 | # Unless required by applicable law or agreed to in writing, software 9 | # distributed under the License is distributed on an "AS IS" BASIS, 10 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | # See the License for the specific language governing permissions and 12 | # limitations under the License. 13 | 14 | import os 15 | 16 | import pyaml 17 | import yaml 18 | 19 | import couler.argo as couler 20 | from couler.tests.argo_yaml_test import ArgoYamlTest 21 | 22 | 23 | class ResourceTest(ArgoYamlTest): 24 | def test_resource_setup(self): 25 | couler.run_container( 26 | image="docker/whalesay", 27 | command=["cowsay"], 28 | args=["resource test"], 29 | resources={"cpu": "1", "memory": "100Mi"}, 30 | ) 31 | # Because test environment between local and CI is different, 32 | # we can not compare the YAML directly. 33 | _test_data_dir = "test_data" 34 | test_data_dir = os.path.join(os.path.dirname(__file__), _test_data_dir) 35 | with open( 36 | os.path.join(test_data_dir, "resource_config_golden.yaml"), "r" 37 | ) as f: 38 | expected = yaml.safe_load(f) 39 | output = yaml.safe_load( 40 | pyaml.dump(couler.workflow_yaml(), string_val_style="plain") 41 | ) 42 | _resources = output["spec"]["templates"][1]["container"]["resources"] 43 | _expected_resources = expected["spec"]["templates"][1]["container"][ 44 | "resources" 45 | ] 46 | 47 | self.assertEqual(_resources, _expected_resources) 48 | couler._cleanup() 49 | -------------------------------------------------------------------------------- /couler/tests/secret_test.py: -------------------------------------------------------------------------------- 1 | # Copyright 2021 The Couler Authors. All rights reserved. 2 | # Licensed under the Apache License, Version 2.0 (the "License"); 3 | # you may not use this file except in compliance with the License. 4 | # You may obtain a copy of the License at 5 | # 6 | # http://www.apache.org/licenses/LICENSE-2.0 7 | # 8 | # Unless required by applicable law or agreed to in writing, software 9 | # distributed under the License is distributed on an "AS IS" BASIS, 10 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | # See the License for the specific language governing permissions and 12 | # limitations under the License. 13 | 14 | import couler.argo as couler 15 | from couler.core import utils 16 | from couler.tests.argo_yaml_test import ArgoYamlTest 17 | 18 | _test_data_dir = "test_data" 19 | couler.config_workflow(name="pytest") 20 | 21 | 22 | class SecretTest(ArgoYamlTest): 23 | def test_create_secret(self): 24 | # First job with secret1 25 | user_info = {"uname": "abc", "passwd": "def"} 26 | secret1 = couler.create_secret(secret_data=user_info, name="dummy1") 27 | couler.run_container( 28 | image="python:3.6", secret=secret1, command="echo $uname" 29 | ) 30 | 31 | # Second job with secret2 that exists 32 | access_key = ["access_key", "access_value"] 33 | secret2 = couler.obtain_secret( 34 | secret_keys=access_key, namespace="test", name="dummy2" 35 | ) 36 | couler.run_container( 37 | image="python:3.6", secret=secret2, command="echo $access_value" 38 | ) 39 | 40 | # Check the secret yaml 41 | self.assertEqual(len(couler.states._secrets), 2) 42 | secret1_yaml = couler.states._secrets[secret1].to_yaml() 43 | secret2_yaml = couler.states._secrets[secret2].to_yaml() 44 | 45 | self.assertEqual(secret1_yaml["metadata"]["name"], "dummy1") 46 | self.assertEqual(len(secret1_yaml["data"]), 2) 47 | self.assertEqual( 48 | secret1_yaml["data"]["uname"], utils.encode_base64("abc") 49 | ) 50 | self.assertEqual( 51 | secret1_yaml["data"]["passwd"], utils.encode_base64("def") 52 | ) 53 | 54 | self.assertEqual(secret2_yaml["metadata"]["namespace"], "test") 55 | self.assertEqual(secret2_yaml["metadata"]["name"], "dummy2") 56 | self.assertEqual(len(secret2_yaml["data"]), 2) 57 | 58 | def _verify_script_body( 59 | self, script_to_check, image, command, source, env 60 | ): 61 | if env is None: 62 | env = utils.convert_dict_to_env_list( 63 | { 64 | "NVIDIA_VISIBLE_DEVICES": "", 65 | "NVIDIA_DRIVER_CAPABILITIES": "", 66 | } 67 | ) 68 | else: 69 | env.append( 70 | utils.convert_dict_to_env_list( 71 | { 72 | "NVIDIA_VISIBLE_DEVICES": "", 73 | "NVIDIA_DRIVER_CAPABILITIES": "", 74 | } 75 | ) 76 | ) 77 | self.assertEqual(script_to_check.get("image", None), image) 78 | self.assertEqual(script_to_check.get("command", None), command) 79 | self.assertEqual(script_to_check.get("source", None), source) 80 | self.assertEqual(script_to_check.get("env", None), env) 81 | 82 | def test_create_secrete_duplicate(self): 83 | def job_1(): 84 | user_info = {"uname": "abc", "passwd": "def"} 85 | secret1 = couler.create_secret(secret_data=user_info, dry_run=True) 86 | couler.run_container( 87 | image="python:3.6", secret=secret1, command="echo $uname" 88 | ) 89 | 90 | def job_2(): 91 | user_info = {"uname": "abc", "passwd": "def"} 92 | secret1 = couler.create_secret(secret_data=user_info, dry_run=True) 93 | couler.run_container( 94 | image="python:3.6", secret=secret1, command="echo $uname" 95 | ) 96 | 97 | job_1() 98 | job_2() 99 | 100 | self.check_argo_yaml("secret_golden.yaml") 101 | -------------------------------------------------------------------------------- /couler/tests/step_output_test.py: -------------------------------------------------------------------------------- 1 | # Copyright 2021 The Couler Authors. All rights reserved. 2 | # Licensed under the Apache License, Version 2.0 (the "License"); 3 | # you may not use this file except in compliance with the License. 4 | # You may obtain a copy of the License at 5 | # 6 | # http://www.apache.org/licenses/LICENSE-2.0 7 | # 8 | # Unless required by applicable law or agreed to in writing, software 9 | # distributed under the License is distributed on an "AS IS" BASIS, 10 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | # See the License for the specific language governing permissions and 12 | # limitations under the License. 13 | 14 | import couler.argo as couler 15 | from couler.tests.argo_yaml_test import ArgoYamlTest 16 | 17 | couler.config_workflow(name="pytest") 18 | 19 | 20 | def producer(): 21 | output_place = couler.create_parameter_artifact( 22 | path="/mnt/hello_world.txt" 23 | ) 24 | return couler.run_container( 25 | image="docker/whalesay:latest", 26 | args=["echo -n hello world > %s" % output_place.path], 27 | command=["bash", "-c"], 28 | output=output_place, 29 | ) 30 | 31 | 32 | def consumer(message): 33 | couler.run_container( 34 | image="docker/whalesay:latest", command=["cowsay"], args=[message] 35 | ) 36 | 37 | 38 | class StepOutputTest(ArgoYamlTest): 39 | def test_producer_consumer(self): 40 | messge = producer() 41 | consumer(messge) 42 | self.check_argo_yaml("output_golden_1.yaml") 43 | 44 | def test_multiple_outputs(self): 45 | def producer_two(): 46 | output_one = couler.create_parameter_artifact( 47 | path="/mnt/place_one.txt" 48 | ) 49 | output_two = couler.create_parameter_artifact( 50 | path="/mnt/place_two.txt" 51 | ) 52 | c1 = "echo -n output one > %s" % output_one.path 53 | c2 = "echo -n output tw0 > %s" % output_two.path 54 | command = "%s && %s" % (c1, c2) 55 | return couler.run_container( 56 | image="docker/whalesay:latest", 57 | args=command, 58 | output=[output_one, output_two], 59 | command=["bash", "-c"], 60 | ) 61 | 62 | def consume_two(message): 63 | couler.run_container( 64 | image="docker/whalesay:latest", 65 | command=["cowsay"], 66 | args=[message], 67 | ) 68 | 69 | messages = producer_two() 70 | consume_two(messages) 71 | self.check_argo_yaml("output_golden_2.yaml") 72 | -------------------------------------------------------------------------------- /couler/tests/test_data/artifact_passing_golden.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: argoproj.io/v1alpha1 2 | kind: Workflow 3 | metadata: 4 | generateName: pytest- 5 | spec: 6 | entrypoint: pytest 7 | templates: 8 | - name: pytest 9 | dag: 10 | tasks: 11 | - name: A 12 | template: A 13 | arguments: 14 | parameters: 15 | - name: para-A-0 16 | value: 'echo -n hello world > /mnt/t1.txt' 17 | - name: B 18 | dependencies: 19 | - A 20 | template: B 21 | arguments: 22 | artifacts: 23 | - from: "{{steps.A.outputs.artifacts.output-oss-1c392c25}}" 24 | name: para-B-1 25 | parameters: 26 | - name: para-B-0 27 | value: '--test 1' 28 | - name: A 29 | inputs: 30 | parameters: 31 | - name: para-A-0 32 | container: 33 | image: docker/whalesay:latest 34 | command: 35 | - bash 36 | - -c 37 | args: 38 | - "{{inputs.parameters.para-A-0}}" 39 | volumeMounts: 40 | - name: couler-out-dir-0 41 | mountPath: /mnt 42 | outputs: 43 | artifacts: 44 | - name: output-oss-1c392c25 45 | path: /mnt/t1.txt 46 | oss: 47 | endpoint: xyz.com 48 | bucket: test-bucket/ 49 | key: osspath/t1 50 | accessKeySecret: 51 | key: accessKey 52 | name: couler-fd7fe83868ddaf22ddda8bca5f3d83d1 53 | secretKeySecret: 54 | key: secretKey 55 | name: couler-fd7fe83868ddaf22ddda8bca5f3d83d1 56 | - name: B 57 | inputs: 58 | parameters: 59 | - name: para-B-0 60 | artifacts: 61 | - name: output-oss-1c392c25 62 | path: /mnt/t1.txt 63 | oss: 64 | endpoint: xyz.com 65 | bucket: test-bucket/ 66 | key: osspath/t1 67 | accessKeySecret: 68 | key: accessKey 69 | name: couler-fd7fe83868ddaf22ddda8bca5f3d83d1 70 | secretKeySecret: 71 | key: secretKey 72 | name: couler-fd7fe83868ddaf22ddda8bca5f3d83d1 73 | container: 74 | image: docker/whalesay:latest 75 | command: 76 | - 'cat /mnt/t1.txt' 77 | args: 78 | - "{{inputs.parameters.para-B-0}}" 79 | volumes: 80 | - emptyDir: {} 81 | name: couler-out-dir-0 -------------------------------------------------------------------------------- /couler/tests/test_data/cron_workflow_golden.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: argoproj.io/v1alpha1 2 | kind: CronWorkflow 3 | metadata: 4 | name: pytest 5 | spec: 6 | concurrencyPolicy: "Allow" 7 | failedJobsHistoryLimit: 1 8 | schedule: '1 0 * * *' 9 | startingDeadlineSeconds: 10 10 | successfulJobsHistoryLimit: 3 11 | suspend: false 12 | timezone: 'Asia/Shanghai' 13 | workflowSpec: 14 | entrypoint: pytest 15 | templates: 16 | - name: pytest 17 | steps: 18 | - - name: tails-14 19 | template: tails 20 | - name: tails 21 | container: 22 | image: python:3.6 23 | command: 24 | - bash 25 | - -c 26 | - 'echo "run schedule job"' -------------------------------------------------------------------------------- /couler/tests/test_data/dag_golden_1.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: argoproj.io/v1alpha1 2 | kind: Workflow 3 | metadata: 4 | generateName: pytest- 5 | spec: 6 | entrypoint: pytest 7 | templates: 8 | - name: pytest 9 | dag: 10 | tasks: 11 | - name: A 12 | template: A 13 | arguments: 14 | parameters: 15 | - name: para-A-0 16 | value: A 17 | - name: B 18 | dependencies: 19 | - A 20 | template: B 21 | arguments: 22 | parameters: 23 | - name: para-B-0 24 | value: B 25 | - name: C 26 | dependencies: 27 | - A 28 | template: C 29 | arguments: 30 | parameters: 31 | - name: para-C-0 32 | value: C 33 | - name: D 34 | dependencies: 35 | - B 36 | template: D 37 | arguments: 38 | parameters: 39 | - name: para-D-0 40 | value: D 41 | - name: A 42 | inputs: 43 | parameters: 44 | - name: para-A-0 45 | container: 46 | image: docker/whalesay:latest 47 | command: 48 | - cowsay 49 | args: 50 | - "{{inputs.parameters.para-A-0}}" 51 | - name: B 52 | inputs: 53 | parameters: 54 | - name: para-B-0 55 | container: 56 | image: docker/whalesay:latest 57 | command: 58 | - cowsay 59 | args: 60 | - "{{inputs.parameters.para-B-0}}" 61 | - name: C 62 | inputs: 63 | parameters: 64 | - name: para-C-0 65 | container: 66 | image: docker/whalesay:latest 67 | command: 68 | - cowsay 69 | args: 70 | - "{{inputs.parameters.para-C-0}}" 71 | - name: D 72 | inputs: 73 | parameters: 74 | - name: para-D-0 75 | container: 76 | image: docker/whalesay:latest 77 | command: 78 | - cowsay 79 | args: 80 | - "{{inputs.parameters.para-D-0}}" 81 | -------------------------------------------------------------------------------- /couler/tests/test_data/dag_golden_2.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: argoproj.io/v1alpha1 2 | kind: Workflow 3 | metadata: 4 | generateName: pytest- 5 | spec: 6 | entrypoint: pytest- 7 | templates: 8 | - name: pytest- 9 | dag: 10 | tasks: 11 | - name: A 12 | template: A 13 | arguments: 14 | parameters: 15 | - name: para-A-0 16 | value: A 17 | - name: B 18 | dependencies: 19 | - A 20 | template: B 21 | arguments: 22 | parameters: 23 | - name: para-B-0 24 | value: B 25 | - name: flip-coin-0 26 | dependencies: 27 | - B 28 | template: flip-coin-0 29 | - name: heads-0 30 | template: heads-0 31 | when: '{{tasks.flip-coin-0.outputs.result}} == heads' 32 | dependencies: 33 | - flip-coin-0 34 | - name: A 35 | inputs: 36 | parameters: 37 | - name: para-A-0 38 | container: 39 | image: docker/whalesay:latest 40 | command: 41 | - cowsay 42 | args: 43 | - "{{inputs.parameters.para-A-0}}" 44 | - name: B 45 | inputs: 46 | parameters: 47 | - name: para-B-0 48 | container: 49 | image: docker/whalesay:latest 50 | command: 51 | - cowsay 52 | args: 53 | - "{{inputs.parameters.para-B-0}}" 54 | - name: flip-coin-0 55 | script: 56 | image: python:3.6 57 | command: 58 | - python 59 | source: |2 60 | 61 | import random 62 | 63 | result = "heads" if random.randint(0, 1) == 0 else "tails" 64 | print(result) 65 | - name: heads-0 66 | container: 67 | image: python:3.6 68 | command: 69 | - bash 70 | - -c 71 | - 'echo "it was heads"' 72 | -------------------------------------------------------------------------------- /couler/tests/test_data/dummy_cluster_config.py: -------------------------------------------------------------------------------- 1 | # Copyright 2021 The Couler Authors. All rights reserved. 2 | # Licensed under the Apache License, Version 2.0 (the "License"); 3 | # you may not use this file except in compliance with the License. 4 | # You may obtain a copy of the License at 5 | # 6 | # http://www.apache.org/licenses/LICENSE-2.0 7 | # 8 | # Unless required by applicable law or agreed to in writing, software 9 | # distributed under the License is distributed on an "AS IS" BASIS, 10 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | # See the License for the specific language governing permissions and 12 | # limitations under the License. 13 | 14 | 15 | class K8s: 16 | def __init__(self): 17 | pass 18 | 19 | def config_pod(self, template): 20 | template["tolerations"] = list() 21 | return template 22 | 23 | def config_workflow(self, spec): 24 | spec["hostNetwork"] = True 25 | return spec 26 | 27 | 28 | cluster = K8s() 29 | -------------------------------------------------------------------------------- /couler/tests/test_data/input_para_golden_1.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: argoproj.io/v1alpha1 2 | kind: Workflow 3 | metadata: 4 | generateName: pytest- 5 | spec: 6 | entrypoint: pytest 7 | templates: 8 | - name: pytest 9 | steps: 10 | - - name: whalesay-84 11 | template: whalesay 12 | arguments: 13 | parameters: 14 | - name: para-whalesay-0 15 | value: "hello1" 16 | - - name: whalesay-85 17 | template: whalesay 18 | arguments: 19 | parameters: 20 | - name: para-whalesay-0 21 | value: "hello2" 22 | - name: whalesay 23 | inputs: 24 | parameters: 25 | - name: para-whalesay-0 26 | container: 27 | image: docker/whalesay 28 | command: [cowsay] 29 | args: 30 | - "{{inputs.parameters.para-whalesay-0}}" 31 | -------------------------------------------------------------------------------- /couler/tests/test_data/input_para_golden_2.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: argoproj.io/v1alpha1 2 | kind: Workflow 3 | metadata: 4 | generateName: pytest- 5 | spec: 6 | entrypoint: pytest 7 | templates: 8 | - name: pytest 9 | steps: 10 | - - name: whalesay-two-90 11 | template: whalesay-two 12 | arguments: 13 | parameters: 14 | - name: para-whalesay-two-0 15 | value: "hello1" 16 | - name: para-whalesay-two-1 17 | value: "hello2" 18 | - - name: whalesay-two-91 19 | template: whalesay-two 20 | arguments: 21 | parameters: 22 | - name: para-whalesay-two-0 23 | value: "x" 24 | - name: para-whalesay-two-1 25 | value: "y" 26 | - name: whalesay-two 27 | inputs: 28 | parameters: 29 | - name: para-whalesay-two-0 30 | - name: para-whalesay-two-1 31 | container: 32 | image: docker/whalesay 33 | command: 34 | - cowsay 35 | args: 36 | - "{{inputs.parameters.para-whalesay-two-0}}" 37 | - "{{inputs.parameters.para-whalesay-two-1}}" 38 | -------------------------------------------------------------------------------- /couler/tests/test_data/input_para_golden_3.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: argoproj.io/v1alpha1 2 | kind: Workflow 3 | metadata: 4 | generateName: pytest- 5 | spec: 6 | entrypoint: pytest 7 | templates: 8 | - name: pytest 9 | steps: 10 | - - name: whalesay-97 11 | template: whalesay 12 | arguments: 13 | parameters: 14 | - name: para-whalesay-0 15 | value: "test" 16 | - - name: generate-number-98 17 | template: generate-number 18 | - - name: whalesay-99 19 | template: whalesay 20 | arguments: 21 | parameters: 22 | - name: para-whalesay-0 23 | value: "{{steps.generate-number-98.outputs.result}}" 24 | - name: whalesay 25 | inputs: 26 | parameters: 27 | - name: para-whalesay-0 28 | container: 29 | image: docker/whalesay 30 | command: 31 | - cowsay 32 | args: 33 | - "{{inputs.parameters.para-whalesay-0}}" 34 | - name: generate-number 35 | script: 36 | image: python:3.6 37 | command: 38 | - python 39 | source: ' 40 | 41 | import random 42 | 43 | 44 | result = random.randint(0, 1) 45 | 46 | print(result) 47 | 48 | ' 49 | -------------------------------------------------------------------------------- /couler/tests/test_data/input_para_golden_4.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: argoproj.io/v1alpha1 2 | kind: Workflow 3 | metadata: 4 | generateName: pytest- 5 | spec: 6 | entrypoint: pytest 7 | templates: 8 | - name: pytest 9 | steps: 10 | - - name: whalesay-109 11 | template: whalesay 12 | arguments: 13 | parameters: 14 | - name: para-whalesay-0 15 | value: test 16 | - name: whalesay 17 | inputs: 18 | parameters: 19 | - name: para-whalesay-0 20 | container: 21 | image: docker/whalesay 22 | command: 23 | - cowsay 24 | args: 25 | - "{{inputs.parameters.para-whalesay-0}}" 26 | -------------------------------------------------------------------------------- /couler/tests/test_data/output_golden_1.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: argoproj.io/v1alpha1 2 | kind: Workflow 3 | metadata: 4 | generateName: pytest- 5 | spec: 6 | entrypoint: pytest 7 | templates: 8 | - name: pytest 9 | steps: 10 | - - name: producer-23 11 | template: producer 12 | arguments: 13 | parameters: 14 | - name: para-producer-0 15 | value: 'echo -n hello world > /mnt/hello_world.txt' 16 | - - name: consumer-24 17 | template: consumer 18 | arguments: 19 | parameters: 20 | - name: para-consumer-0 21 | value: "{{steps.producer-23.outputs.parameters.output-id-6}}" 22 | - name: producer 23 | inputs: 24 | parameters: 25 | - name: para-producer-0 26 | container: 27 | image: docker/whalesay:latest 28 | command: 29 | - bash 30 | - -c 31 | args: 32 | - "{{inputs.parameters.para-producer-0}}" 33 | volumeMounts: 34 | - name: couler-out-dir-0 35 | mountPath: /mnt 36 | outputs: 37 | parameters: 38 | - name: output-id-6 39 | valueFrom: 40 | path: /mnt/hello_world.txt 41 | - name: consumer 42 | inputs: 43 | parameters: 44 | - name: para-consumer-0 45 | container: 46 | image: docker/whalesay:latest 47 | command: 48 | - cowsay 49 | args: 50 | - "{{inputs.parameters.para-consumer-0}}" 51 | volumes: 52 | - emptyDir: {} 53 | name: couler-out-dir-0 54 | -------------------------------------------------------------------------------- /couler/tests/test_data/output_golden_2.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: argoproj.io/v1alpha1 2 | kind: Workflow 3 | metadata: 4 | generateName: pytest- 5 | spec: 6 | entrypoint: pytest 7 | templates: 8 | - name: pytest 9 | steps: 10 | - - name: producer-two-48 11 | template: producer-two 12 | arguments: 13 | parameters: 14 | - name: para-producer-two-0 15 | value: 'echo -n output one > /mnt/place_one.txt && echo -n output 16 | tw0 > /mnt/place_two.txt' 17 | - - name: consume-two-49 18 | template: consume-two 19 | arguments: 20 | parameters: 21 | - name: para-consume-two-0 22 | value: "{{steps.producer-two-48.outputs.parameters.output-id-29}}" 23 | - name: para-consume-two-1 24 | value: "{{steps.producer-two-48.outputs.parameters.output-id-30}}" 25 | - name: producer-two 26 | inputs: 27 | parameters: 28 | - name: para-producer-two-0 29 | container: 30 | image: docker/whalesay:latest 31 | command: 32 | - bash 33 | - -c 34 | args: 35 | - "{{inputs.parameters.para-producer-two-0}}" 36 | volumeMounts: 37 | - name: couler-out-dir-0 38 | mountPath: /mnt 39 | outputs: 40 | parameters: 41 | - name: output-id-29 42 | valueFrom: 43 | path: /mnt/place_one.txt 44 | - name: output-id-30 45 | valueFrom: 46 | path: /mnt/place_two.txt 47 | - name: consume-two 48 | inputs: 49 | parameters: 50 | - name: para-consume-two-0 51 | - name: para-consume-two-1 52 | container: 53 | image: docker/whalesay:latest 54 | command: 55 | - cowsay 56 | args: 57 | - "{{inputs.parameters.para-consume-two-0}}" 58 | - "{{inputs.parameters.para-consume-two-1}}" 59 | volumes: 60 | - emptyDir: {} 61 | name: couler-out-dir-0 62 | -------------------------------------------------------------------------------- /couler/tests/test_data/parameter_passing_golden.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: argoproj.io/v1alpha1 2 | kind: Workflow 3 | metadata: 4 | generateName: pytest- 5 | spec: 6 | entrypoint: pytest 7 | templates: 8 | - name: pytest 9 | dag: 10 | tasks: 11 | - name: A 12 | template: A 13 | arguments: 14 | parameters: 15 | - name: para-A-0 16 | value: 'echo -n A > /mnt/t1.txt && echo -n B > /mnt/t2.txt' 17 | - name: B 18 | dependencies: 19 | - A 20 | template: B 21 | arguments: 22 | parameters: 23 | - name: para-B-0 24 | value: '--input: x' 25 | - name: para-B-1 26 | value: "{{tasks.A.outputs.parameters.output-id-208}}" 27 | - name: para-B-2 28 | value: "{{tasks.A.outputs.parameters.output-id-211}}" 29 | - name: A 30 | inputs: 31 | parameters: 32 | - name: para-A-0 33 | container: 34 | image: docker/whalesay:latest 35 | command: 36 | - bash 37 | - -c 38 | args: 39 | - "{{inputs.parameters.para-A-0}}" 40 | volumeMounts: 41 | - name: couler-out-dir-0 42 | mountPath: /mnt 43 | outputs: 44 | parameters: 45 | - name: output-id-208 46 | valueFrom: 47 | path: /mnt/t1.txt 48 | - name: output-id-211 49 | valueFrom: 50 | path: /mnt/t2.txt 51 | - name: B 52 | inputs: 53 | parameters: 54 | - name: para-B-0 55 | - name: para-B-1 56 | - name: para-B-2 57 | container: 58 | image: docker/whalesay:latest 59 | command: 60 | - echo 61 | args: 62 | - "{{inputs.parameters.para-B-0}}" 63 | - "{{inputs.parameters.para-B-1}}" 64 | - "{{inputs.parameters.para-B-2}}" 65 | volumes: 66 | - emptyDir: {} 67 | name: couler-out-dir-0 68 | -------------------------------------------------------------------------------- /couler/tests/test_data/resource_config_golden.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: argoproj.io/v1alpha1 2 | kind: Workflow 3 | metadata: 4 | generateName: pytest- 5 | spec: 6 | entrypoint: pytest 7 | templates: 8 | - name: pytest 9 | steps: 10 | - - name: test-resource-setup-605 11 | template: test-resource-setup 12 | arguments: 13 | parameters: 14 | - name: para-test-resource-setup-0 15 | value: 'resource test' 16 | - name: test-resource-setup 17 | inputs: 18 | parameters: 19 | - name: para-test-resource-setup-0 20 | container: 21 | image: docker/whalesay 22 | command: 23 | - bash 24 | - -c 25 | - cowsay 26 | args: 27 | - "{{inputs.parameters.para-test-resource-setup-0}}" 28 | resources: 29 | requests: 30 | cpu: 1 31 | memory: 100Mi 32 | limits: 33 | cpu: 1 34 | memory: 100Mi -------------------------------------------------------------------------------- /couler/tests/test_data/run_concurrent_golden.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: argoproj.io/v1alpha1 2 | kind: Workflow 3 | metadata: 4 | generateName: pytest- 5 | spec: 6 | entrypoint: pytest 7 | templates: 8 | - name: pytest 9 | steps: 10 | - - name: whalesay-605 11 | template: whalesay 12 | arguments: 13 | parameters: 14 | - name: para-whalesay-0 15 | value: hello1 16 | - name: heads-605 17 | template: heads 18 | - name: tails-605 19 | template: tails 20 | - name: whalesay 21 | inputs: 22 | parameters: 23 | - name: para-whalesay-0 24 | container: 25 | image: docker/whalesay 26 | command: 27 | - bash 28 | - -c 29 | - cowsay 30 | args: 31 | - "{{inputs.parameters.para-whalesay-0}}" 32 | - name: heads 33 | container: 34 | image: docker/whalesay 35 | command: 36 | - bash 37 | - -c 38 | - 'echo "it was heads"' 39 | env: 40 | - name: "NVIDIA_VISIBLE_DEVICES" 41 | value: null 42 | - name: "NVIDIA_DRIVER_CAPABILITIES" 43 | value: null 44 | - name: tails 45 | container: 46 | image: docker/whalesay 47 | command: 48 | - bash 49 | - -c 50 | - 'echo "it was tails"' 51 | -------------------------------------------------------------------------------- /couler/tests/test_data/run_concurrent_golden_2.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: argoproj.io/v1alpha1 2 | kind: Workflow 3 | metadata: 4 | generateName: pytest- 5 | spec: 6 | entrypoint: pytest 7 | templates: 8 | - name: pytest 9 | steps: 10 | - - name: whalesay-615-1 11 | template: whalesay 12 | arguments: 13 | parameters: 14 | - name: para-whalesay-0 15 | value: hello1 16 | - name: whalesay-615-2 17 | template: whalesay 18 | arguments: 19 | parameters: 20 | - name: para-whalesay-0 21 | value: hello1 22 | - name: tails-615-3 23 | template: tails 24 | - name: whalesay 25 | inputs: 26 | parameters: 27 | - name: para-whalesay-0 28 | container: 29 | image: docker/whalesay 30 | command: 31 | - bash 32 | - -c 33 | - cowsay 34 | args: 35 | - "{{inputs.parameters.para-whalesay-0}}" 36 | - name: tails 37 | container: 38 | image: docker/whalesay 39 | command: 40 | - bash 41 | - -c 42 | - 'echo "it was tails"' 43 | -------------------------------------------------------------------------------- /couler/tests/test_data/run_concurrent_golden_3.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: argoproj.io/v1alpha1 2 | kind: Workflow 3 | metadata: 4 | generateName: pytest- 5 | spec: 6 | entrypoint: pytest 7 | templates: 8 | - name: pytest 9 | steps: 10 | - - name: job-one-615-1 11 | template: job-one 12 | arguments: 13 | parameters: 14 | - name: para-job-one-0 15 | value: 'echo -n step one > /tmp/job_one.txt' 16 | - name: job-two-615-2 17 | template: job-two 18 | arguments: 19 | parameters: 20 | - name: para-job-two-0 21 | value: 'echo -n step two > /tmp/job_two.txt' 22 | - - name: summary-96 23 | template: summary 24 | arguments: 25 | parameters: 26 | - name: para-summary-0 27 | value: "{{steps.job-one-615-1.outputs.parameters.output-id-95}}" 28 | - name: para-summary-1 29 | value: "{{steps.job-two-615-2.outputs.parameters.output-id-95}}" 30 | - name: job-one 31 | inputs: 32 | parameters: 33 | - name: para-job-one-0 34 | container: 35 | image: python:3.6 36 | command: 37 | - bash 38 | - -c 39 | args: 40 | - "{{inputs.parameters.para-job-one-0}}" 41 | outputs: 42 | parameters: 43 | - name: output-id-95 44 | valueFrom: 45 | path: /tmp/job_one.txt 46 | - name: job-two 47 | inputs: 48 | parameters: 49 | - name: para-job-two-0 50 | container: 51 | image: python:3.6 52 | command: 53 | - bash 54 | - -c 55 | args: 56 | - "{{inputs.parameters.para-job-two-0}}" 57 | outputs: 58 | parameters: 59 | - name: output-id-95 60 | valueFrom: 61 | path: /tmp/job_two.txt 62 | - name: summary 63 | inputs: 64 | parameters: 65 | - name: para-summary-0 66 | - name: para-summary-1 67 | container: 68 | image: docker/whalesay 69 | command: 70 | - bash 71 | - -c 72 | - cowsay 73 | args: 74 | - "{{inputs.parameters.para-summary-0}}" 75 | - "{{inputs.parameters.para-summary-1}}" 76 | -------------------------------------------------------------------------------- /couler/tests/test_data/run_concurrent_subtasks_golden.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: argoproj.io/v1alpha1 2 | kind: Workflow 3 | metadata: 4 | generateName: pytest- 5 | spec: 6 | entrypoint: pytest 7 | templates: 8 | - name: pytest 9 | steps: 10 | - - name: whalesay-106 11 | template: whalesay 12 | arguments: 13 | parameters: 14 | - name: para-whalesay-0 15 | value: 'workflow start' 16 | - - name: concurrent-task-1-615-1 17 | template: concurrent-task-1 18 | - name: concurrent-task-2-615-2 19 | template: concurrent-task-2 20 | - - name: whalesay-108 21 | template: whalesay 22 | arguments: 23 | parameters: 24 | - name: para-whalesay-0 25 | value: 'workflow finish' 26 | - name: whalesay 27 | inputs: 28 | parameters: 29 | - name: para-whalesay-0 30 | container: 31 | image: docker/whalesay 32 | command: 33 | - cowsay 34 | args: 35 | - "{{inputs.parameters.para-whalesay-0}}" 36 | - name: concurrent-task-1 37 | steps: 38 | - - name: whalesay-99 39 | template: whalesay 40 | arguments: 41 | parameters: 42 | - name: para-whalesay-0 43 | value: 'workflow one' 44 | - - name: whalesay-100 45 | template: whalesay 46 | arguments: 47 | parameters: 48 | - name: para-whalesay-0 49 | value: t1 50 | - - name: whalesay-101 51 | template: whalesay 52 | arguments: 53 | parameters: 54 | - name: para-whalesay-0 55 | value: t2 56 | - name: concurrent-task-2 57 | steps: 58 | - - name: whalesay-104 59 | template: whalesay 60 | arguments: 61 | parameters: 62 | - name: para-whalesay-0 63 | value: 'workflow two' 64 | -------------------------------------------------------------------------------- /couler/tests/test_data/secret_golden.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: argoproj.io/v1alpha1 2 | kind: Workflow 3 | metadata: 4 | generateName: pytest- 5 | spec: 6 | entrypoint: pytest 7 | templates: 8 | - name: pytest 9 | steps: 10 | - - name: job-1-90 11 | template: job-1 12 | - - name: job-2-91 13 | template: job-2 14 | - name: job-1 15 | container: 16 | image: python:3.6 17 | command: 18 | - 'echo $uname' 19 | env: 20 | - name: uname 21 | valueFrom: 22 | secretKeyRef: 23 | key: uname 24 | name: couler-60270dab25f1b09127c16efc3907d44e 25 | - name: passwd 26 | valueFrom: 27 | secretKeyRef: 28 | key: passwd 29 | name: couler-60270dab25f1b09127c16efc3907d44e 30 | - name: job-2 31 | container: 32 | image: python:3.6 33 | command: 34 | - 'echo $uname' 35 | env: 36 | - name: uname 37 | valueFrom: 38 | secretKeyRef: 39 | key: uname 40 | name: couler-60270dab25f1b09127c16efc3907d44e 41 | - name: passwd 42 | valueFrom: 43 | secretKeyRef: 44 | key: passwd 45 | name: couler-60270dab25f1b09127c16efc3907d44e 46 | 47 | -------------------------------------------------------------------------------- /couler/tests/test_data/while_golden.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: argoproj.io/v1alpha1 2 | kind: Workflow 3 | metadata: 4 | generateName: pytest- 5 | spec: 6 | entrypoint: pytest 7 | templates: 8 | - name: pytest 9 | steps: 10 | - - name: exec-while-flip-coin-18 11 | template: exec-while-flip-coin 12 | - name: flip-coin 13 | script: 14 | image: python:3.6 15 | command: 16 | - python 17 | source: ' 18 | 19 | import random 20 | 21 | 22 | result = "heads" if random.randint(0, 1) == 0 else "tails" 23 | 24 | print(result) 25 | 26 | ' 27 | - name: exec-while-flip-coin 28 | steps: 29 | - - name: flip-coin-18 30 | template: flip-coin 31 | - - name: exec-while-flip-coin-exit 32 | template: exec-while-flip-coin 33 | when: '{{steps.flip-coin-18.outputs.result}} == tails' 34 | -------------------------------------------------------------------------------- /couler/tests/test_data/workflow_basic_golden.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: argoproj.io/v1alpha1 2 | kind: Workflow 3 | metadata: 4 | generateName: pytest- 5 | spec: 6 | entrypoint: pytest 7 | onExit: exit-handler 8 | templates: 9 | - name: pytest 10 | steps: 11 | - - name: flip-coin-167 12 | template: flip-coin 13 | - name: flip-coin 14 | script: 15 | image: python:3.6 16 | command: 17 | - python 18 | source: |2 19 | 20 | import random 21 | 22 | result = "heads" if random.randint(0, 1) == 0 else "tails" 23 | print(result) 24 | - name: heads 25 | container: 26 | image: python:3.6 27 | command: 28 | - bash 29 | - -c 30 | - 'echo "it was heads"' 31 | - name: tails 32 | container: 33 | image: python:3.6 34 | command: 35 | - bash 36 | - -c 37 | - 'echo "it was tails"' 38 | - name: exit-handler 39 | steps: 40 | - - name: heads-637 41 | template: heads 42 | when: '{{workflow.status}} == Succeeded' 43 | - - name: tails-637 44 | template: tails 45 | when: '{{workflow.status}} == Failed' 46 | -------------------------------------------------------------------------------- /couler/tests/utils_test.py: -------------------------------------------------------------------------------- 1 | # Copyright 2021 The Couler Authors. All rights reserved. 2 | # Licensed under the Apache License, Version 2.0 (the "License"); 3 | # you may not use this file except in compliance with the License. 4 | # You may obtain a copy of the License at 5 | # 6 | # http://www.apache.org/licenses/LICENSE-2.0 7 | # 8 | # Unless required by applicable law or agreed to in writing, software 9 | # distributed under the License is distributed on an "AS IS" BASIS, 10 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | # See the License for the specific language governing permissions and 12 | # limitations under the License. 13 | 14 | import base64 15 | 16 | from couler.core import utils 17 | from couler.tests.argo_test import ArgoBaseTestCase 18 | 19 | 20 | class PyfuncTest(ArgoBaseTestCase): 21 | def test_argo_safe_name(self): 22 | self.assertIsNone(utils.argo_safe_name(None)) 23 | self.assertEqual(utils.argo_safe_name("a_b"), "a-b") 24 | self.assertEqual(utils.argo_safe_name("a.b"), "a-b") 25 | self.assertEqual(utils.argo_safe_name("a_.b"), "a--b") 26 | self.assertEqual(utils.argo_safe_name("_abc."), "-abc-") 27 | 28 | def test_body(self): 29 | # Check None 30 | self.assertIsNone(utils.body(None)) 31 | # A real function 32 | code = """ 33 | func_name = utils.workflow_filename() 34 | # Here we assume that we are using `pytest` or `python -m pytest` 35 | # to trigger the unit tests. 36 | self.assertTrue(func_name in ["pytest", "runpy"]) 37 | """ 38 | self.assertEqual(code, utils.body(self.test_get_root_caller_filename)) 39 | 40 | def test_get_root_caller_filename(self): 41 | func_name = utils.workflow_filename() 42 | # Here we assume that we are using `pytest` or `python -m pytest` 43 | # to trigger the unit tests. 44 | self.assertTrue(func_name in ["pytest", "runpy"]) 45 | 46 | def test_invocation_location(self): 47 | def inner_func(): 48 | func_name, _ = utils.invocation_location() 49 | self.assertEqual("test-invocation-location", func_name) 50 | 51 | inner_func() 52 | 53 | def test_encode_base64(self): 54 | s = "test encode string" 55 | encode = utils.encode_base64(s) 56 | decode = str(base64.b64decode(encode), "utf-8") 57 | self.assertEqual(s, decode) 58 | 59 | def test_check_gpu(self): 60 | with self.assertRaises(TypeError): 61 | utils.gpu_requested("cpu=1") 62 | self.assertFalse(utils.gpu_requested(None)) 63 | self.assertFalse(utils.gpu_requested({})) 64 | self.assertFalse(utils.gpu_requested({"cpu": 1})) 65 | self.assertFalse(utils.gpu_requested({"cpu": 1, "memory": 2})) 66 | self.assertTrue(utils.gpu_requested({"gpu": 1})) 67 | self.assertTrue(utils.gpu_requested({" gpu ": 1})) 68 | self.assertTrue(utils.gpu_requested({"GPU": 1})) 69 | self.assertTrue(utils.gpu_requested({"cpu": 1, "memory": 2, "gpu": 1})) 70 | 71 | def test_non_empty(self): 72 | self.assertFalse(utils.non_empty(None)) 73 | self.assertFalse(utils.non_empty([])) 74 | self.assertFalse(utils.non_empty({})) 75 | self.assertTrue(utils.non_empty(["a"])) 76 | self.assertTrue(utils.non_empty({"a": "b"})) 77 | -------------------------------------------------------------------------------- /couler/tests/while_test.py: -------------------------------------------------------------------------------- 1 | # Copyright 2021 The Couler Authors. All rights reserved. 2 | # Licensed under the Apache License, Version 2.0 (the "License"); 3 | # you may not use this file except in compliance with the License. 4 | # You may obtain a copy of the License at 5 | # 6 | # http://www.apache.org/licenses/LICENSE-2.0 7 | # 8 | # Unless required by applicable law or agreed to in writing, software 9 | # distributed under the License is distributed on an "AS IS" BASIS, 10 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | # See the License for the specific language governing permissions and 12 | # limitations under the License. 13 | 14 | import couler.argo as couler 15 | from couler.tests.argo_yaml_test import ArgoYamlTest 16 | 17 | 18 | def random_code(): 19 | import random 20 | 21 | result = "heads" if random.randint(0, 1) == 0 else "tails" 22 | print(result) 23 | 24 | 25 | def flip_coin(): 26 | return couler.run_script(image="python:3.6", source=random_code) 27 | 28 | 29 | class RecursionTest(ArgoYamlTest): 30 | def test_while(self): 31 | couler.exec_while(couler.equal("tails"), lambda: flip_coin()) 32 | self.check_argo_yaml("while_golden.yaml") 33 | 34 | def test_while_condition_callable(self): 35 | cls = self.create_callable_cls(lambda: "tails") 36 | instance = cls() 37 | func_names = ["a", "b", "c", "self"] 38 | for func_name in func_names: 39 | if func_name == "self": 40 | couler.exec_while(couler.equal(instance), lambda: flip_coin()) 41 | else: 42 | couler.exec_while( 43 | couler.equal(getattr(instance, func_name)), 44 | lambda: flip_coin(), 45 | ) 46 | self.check_argo_yaml("while_golden.yaml") 47 | couler._cleanup() 48 | 49 | def test_while_callable(self): 50 | cls = self.create_callable_cls(lambda: flip_coin()) 51 | instance = cls() 52 | func_names = ["a", "b", "c", "self"] 53 | for func_name in func_names: 54 | if func_name == "self": 55 | couler.exec_while(couler.equal("tails"), instance) 56 | else: 57 | couler.exec_while( 58 | couler.equal("tails"), getattr(instance, func_name) 59 | ) 60 | 61 | self.check_argo_yaml("while_golden.yaml") 62 | couler._cleanup() 63 | -------------------------------------------------------------------------------- /docs/NL-to-Unified-Programming-Interface/Algorithm.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/couler-proj/couler/5c437c2f1552656b68dec1e4b562b57b027efe25/docs/NL-to-Unified-Programming-Interface/Algorithm.png -------------------------------------------------------------------------------- /docs/NL-to-Unified-Programming-Interface/Method.md: -------------------------------------------------------------------------------- 1 | In this file, we introduce the application of LLMs for converting Natural Language (NL) to Unified Programming Interface. Traditional methods involve defining workflows using various techniques and submitting them to a cluster. Lately, LLMs have demonstrated remarkable performance across a wide array of inference tasks. However, upon direct application of LLMs for unified programming code generation, certain challenges arise: Firstly, the overall workflow complexity hampers the performance of LLMs in complete workflow conversion. Secondly, LLMs possess limited knowledge regarding Couler's unified programming interface. 2 | 3 | To address these challenges, we introduce a method that leverages LLMs to automatically translate natural language into unified programming code via the crafting of task-specific prompts. This approach enables users to articulate their desired workflows in natural language, which are then automatically translated into executable unified programming code. As a result, our method simplifies the Couler workflow creation process and improves usability for individuals with limited programming experience. The transition from NL descriptions to Couler code encompasses four pivotal steps: 4 | 5 | **Step 1: Modular Decomposition:** 6 | 7 | Initially, we employ a chain of thought strategy to decompose natural language descriptions into smaller, more concise task modules, such as data loading, data processing, model generation, and evaluation metrics. Each module should encapsulate a singular, coherent task to ensure the precision and correctness of the generated Couler code. A series of predefined task types can be established to identify and extract pertinent tasks based on the input of natural language descriptions automatically. They provide a structured approach to ensure the precision and correctness of the code generated. 8 | 9 | **Step 2: Code Generation:** 10 | 11 | For each independent subtask, we utilize LLMs to generate code. Considering that LLMs have limited knowledge about Couler, we construct a Code Lake containing code for various functions. We search for relevant code from the Code Lake for each subtask and provide it to LLMs for reference. This significantly improves the ability for unified programming code generation. 12 | 13 | **Step 3: Self-calibration:** 14 | 15 | After generating the code for each subtask, we integrate a self-calibration strategy to optimize the generated code. This strategy evaluates the generated code by having LLMs critique it. Initially, we define a baseline score as the standard evaluation score. We use LLMs to evaluate the generated code for a score between 0 and 1, and if, we will provide feedback of LLMs and repeat the code generation. After this self-calibration, we will have improved code for each subtask. 16 | 17 | **Step 4: User Feedback:** 18 | 19 | Finally, users can review and validate the generated workflow code. If the generated code fails to meet the users' requirements, they have the opportunity to provide feedback and suggestions in textual format. The system will leverage this feedback to optimize the code and enhance the precision of code generation. 20 | 21 | 22 | -------------------------------------------------------------------------------- /docs/NL-to-Unified-Programming-Interface/Method_overview.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/couler-proj/couler/5c437c2f1552656b68dec1e4b562b57b027efe25/docs/NL-to-Unified-Programming-Interface/Method_overview.pdf -------------------------------------------------------------------------------- /docs/NL-to-Unified-Programming-Interface/Running_example.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/couler-proj/couler/5c437c2f1552656b68dec1e4b562b57b027efe25/docs/NL-to-Unified-Programming-Interface/Running_example.pdf -------------------------------------------------------------------------------- /docs/TEMPLATE.md: -------------------------------------------------------------------------------- 1 | > Note to proposers: Please keep this document as brief as possible, preferably not more than two pages. 2 | 3 | ## Motivation 4 | A high level description of the problem or opportunity being addressed. 5 | 6 | ## Goals 7 | Specific new functionalities or other changes. 8 | 9 | ## Non-Goals 10 | Issues or changes not being addressed by this proposal. 11 | 12 | ## Design 13 | Description of new software design and any major changes to existing software. Should include figures or diagrams where appropriate. Backward compatibility must be considered. 14 | 15 | ## Alternatives Considered 16 | Description of possible alternative solutions and the reasons they were not chosen. 17 | -------------------------------------------------------------------------------- /docs/Technical-Report-of-Couler/README.md: -------------------------------------------------------------------------------- 1 | Machine Learning (ML) has become ubiquitous, fueling data-driven applications across various organizations. Contrary to the traditional perception of ML in research, ML workflows can be complex, resource-intensive, and time-consuming. 2 | Expanding an ML workflow to encompass a wider range of data infrastructure and data types may lead to larger workloads and increased deployment costs. 3 | Currently, numerous workflow engines are available (with over ten being widely recognized). This variety poses a challenge for end-users in terms of mastering different engine APIs. While efforts have primarily focused on optimizing ML Operations (MLOps) for a specific workflow engine, current methods largely overlook workflow optimization across different engines. 4 | 5 | In this work, we design and implement Couler, a system designed for unified ML workflow optimization in the cloud. 6 | Our main insight lies in the ability to generate an ML workflow using natural language (NL) descriptions. 7 | We integrate Large Language Models (LLMs) into workflow generation, and provide a unified programming interface for various workflow engines. This approach alleviates the need to understand various workflow engines' APIs. Moreover, Couler enhances workflow computation efficiency by introducing automated caching at multiple stages, enabling large workflow auto-parallelization and automatic hyperparameters tuning. These enhancements minimize redundant computational costs and improve fault tolerance during deep learning workflow training. 8 | -------------------------------------------------------------------------------- /docs/Technical-Report-of-Couler/Tech-Report-of-Couler-Unified-Machine-Learning-Workflow-Optimization-in-Cloud.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/couler-proj/couler/5c437c2f1552656b68dec1e4b562b57b027efe25/docs/Technical-Report-of-Couler/Tech-Report-of-Couler-Unified-Machine-Learning-Workflow-Optimization-in-Cloud.pdf -------------------------------------------------------------------------------- /docs/adopters.md: -------------------------------------------------------------------------------- 1 | --8<-- "ADOPTERS.md" 2 | -------------------------------------------------------------------------------- /docs/assets/stylesheets/extra.css: -------------------------------------------------------------------------------- 1 | :root { 2 | --md-primary-fg-color: #4185F1; 3 | } 4 | 5 | .md-header__topic { 6 | display: none; 7 | } 8 | 9 | .md-header__button.md-logo img { 10 | width: 3.3rem; 11 | } 12 | -------------------------------------------------------------------------------- /docs/contributing.md: -------------------------------------------------------------------------------- 1 | --8<-- "CONTRIBUTING.md" 2 | -------------------------------------------------------------------------------- /docs/couler-step-zoo.md: -------------------------------------------------------------------------------- 1 | # Couler Step Zoo 2 | 3 | This document introduces the Couler Step Zoo, which consists of a collection of pre-defined and reusable steps that can 4 | be used directly as part of a workflow defined using Couler. 5 | 6 | 7 | ## Existing Steps 8 | 9 | Currently, Couler Step Zoo consists of the following pre-defined steps: 10 | 11 | | Name | Description | API | 12 | | ---- | ----------- | --- | 13 | | MPI | Submit MPI-based distributed jobs via Kubeflow MPI Operator | `couler.steps.mpi.train()` | 14 | | TensorFlow | Submit TensorFlow distributed training jobs via Kubeflow TF Operator | `couler.steps.tensorflow.train()` | 15 | | PyTorch | Submit PyTorch distributed training jobs via Kubeflow PyTorch Operator | `couler.steps.pytorch.train()` | 16 | | Katib | Submit AutoML experiments (e.g. hyperparameter tuning and neural architecture search) via Kubeflow Katib | `couler.steps.katib.run()` | 17 | 18 | If you'd like to add a new pre-defined steps to the Couler Step Zoo, please see the following sections for 19 | the specific requirements and instructions. 20 | 21 | 22 | ## Requirements on New Steps 23 | 24 | In order to provide a consistent and friendly experience to our users, below are the requirements for a new pre-defined 25 | step to be eligible for inclusion in Couler Step Zoo: 26 | 27 | * It should be completely implemented using the core [Couler APIs](couler-api-design.md). 28 | * It should expose backend specific configurations instead of hard-coding them. 29 | * It should have a clear set of dependencies that can be easily installed with sufficient instructions. 30 | * When possible, proving a minimal set of unit tests and integration tests to make sure the step functions correctly. 31 | 32 | 33 | ## Adding a Pre-defined Step 34 | 35 | To add a pre-defined step to the Couler Step Zoo, please follow the instructions below. 36 | 37 | 1. Make sure the step meets the list of requirements in the above section. 38 | 1. Add the step implementation to [couler.steps module](../couler/steps). The interface would look like the following: 39 | 40 | ```python 41 | def random_code(): 42 | import random 43 | 44 | res = "heads" if random.randint(0, 1) == 0 else "tails" 45 | print(res) 46 | 47 | def run_heads( 48 | image="alpine:3.6", 49 | command=["sh", "-c", 'echo "it was heads"'], 50 | ): 51 | result = random_code() 52 | couler.when( 53 | couler.equal(result, "heads"), 54 | lambda: couler.run_step( 55 | command=command, 56 | image=image, 57 | )) 58 | ``` 59 | 60 | Here we implemented a pre-defined step `run_heads()` that could run a specified command such as `["sh", "-c", 'echo "it was heads"']` 61 | if `random_code()` returns `"heads"`. Note that the step should be completely implemented using the 62 | core [Couler APIs](couler-api-design.md) in order to work well with different Couler backends. 63 | 64 | You can find reference step implementations [here](../couler/steps). 65 | 66 | 1. Provide minimal set of [unit test](../couler/tests) and [integration test](../integration_tests) when possible. 67 | 1. Provide necessary user-facing documentation in the API docstring, including documentation on each arguments in the 68 | step signature and the system dependencies. 69 | 70 | 71 | ## Alternatives Considered 72 | 73 | Some workflow engines or frameworks provide their own library for distributing the set of reusable steps/tasks/components, for example: 74 | 75 | * [Argo Workflows catalog](https://github.com/argoproj-labs/argo-workflows-catalog) 76 | * [Prefect tasks library](https://docs.prefect.io/core/task_library/overview.html#task-library-in-action) 77 | * [KFP components](https://github.com/kubeflow/pipelines/tree/master/components) 78 | * [Tekton tasks/pipelines catalog](https://github.com/tektoncd/catalog) 79 | 80 | Even though it's relatively easier to provide wrappers around the existing reusable libraries, there are some issues 81 | with that approach: 82 | 83 | * It's hard to maintain and keep the wrappers up-to-date. 84 | * It's non-trivial to provide a consistent interface that would work across different backends. 85 | * This would introduce bad user experience due to feature parity across those reusable libraries implemented for different backends. 86 | -------------------------------------------------------------------------------- /docs/couler-tekton-design.md: -------------------------------------------------------------------------------- 1 | # Couler Tekton Design 2 | 3 | This document outlines the design of tekton backend for core Couler APIs. 4 | 5 | ## Goals 6 | 7 | * Design the tekton backend implementation for core Couler APIs. 8 | 9 | ## Non-Goals 10 | 11 | * Provide implementation details. 12 | 13 | ## Design 14 | 15 | Based on the core [Couler APIs design](couler-api-design.md), we can see the definitions in different workflow engines. 16 | 17 | | Couler | Argo | Tekton | 18 | | ---- | ---- | ---- | 19 | | Step | Step | Step | 20 | | Reusable step | Template | Task | 21 | | Workflow | Workflow | Pipeline | 22 | 23 | * `Step` is the smallest unit and remain consistent across different backends so the core operation `run_step(step_def)` will also keep consistent. In tekton, the definition of `step` and `container` keeps the same, which means that if users pass a python function to step, the backend should wrap the function in a python base image automatically. 24 | 25 | ### Control Flow 26 | 27 | #### `when(cond, if_op, else_op)` 28 | 29 | In the latest v0.16 version of Tekton, the condition field is deprecated and Tekton uses [whenExpression](https://github.com/tektoncd/pipeline/blob/master/docs/pipelines.md#guard-task-execution-using-whenexpressions) instead. Here is an example: 30 | 31 | ```yaml 32 | tasks: 33 | - name: echo-file-exists 34 | when: 35 | - input: "$(tasks.check-file.results.exists)" 36 | operator: in 37 | values: ["yes"] 38 | taskRef: 39 | name: echo-file-exists 40 | ``` 41 | 42 | A `whenExpression` is made of `Input`, `Operator` and `Values`. The `Input` can be static inputs or variables (Parameters or Results in Tekton). The `Operator` can be either `in` or `notin`. 43 | As we can see, there is no `else_op` in Tekton, so we need to convert the `else` in the Tekton backend, which is pretty simple since there is only `in` and `notin` operator, we can simply change the operator in the "else" task like: 44 | ```yaml 45 | tasks: 46 | - name: echo-file-not-exists 47 | when: 48 | - input: "$(tasks.check-file.results.exists)" 49 | operator: notin 50 | values: ["yes"] 51 | taskRef: 52 | name: echo-file-not-exists 53 | ``` 54 | 55 | 56 | #### `while_loop(cond, func, *args, **kwargs)` 57 | 58 | There is no native support for loops in Tekton for now. See this [issue](https://github.com/tektoncd/pipeline/issues/2050) for more information. 59 | 60 | ### Couler Utilities: 61 | 62 | The `get_status`, `get_logs` remains the same as the Argo design. 63 | 64 | * `submit(config=workflow_config(schedule="* * * * 1"))`: There is no native support for cron in Tekton, see this [issue](https://github.com/tektoncd/triggers/issues/69) for more information. The community recommend cronjob in kubernetes, see [example](https://github.com/tektoncd/triggers/blob/master/examples/cron/README.md) here. For couler, we can wrap the `cron` in the workflow and let backend to handle the work like create `CronJob` in Kubernetes. 65 | * `delete_workflow(workflow_name)`: To be discussed: should we delete the tasks when delete the pipeline? 66 | * `list_workflow_records(workflow_name)`: After submitting the workflow(create PipelineRun in Tekton), there will be numbers of workflow records. A list function is necessary for users to show all the records. 67 | 68 | ### Other Tekton Utilities 69 | 70 | * [Finally](https://github.com/tektoncd/pipeline/blob/master/docs/pipelines.md#adding-finally-to-the-pipeline): Finally is a unique feature of Tekton. With finally, the Final tasks are guaranteed to be executed in parallel after all PipelineTasks under tasks have completed regardless of success or error. For example: 71 | ```yaml 72 | spec: 73 | tasks: 74 | - name: tests 75 | taskRef: 76 | Name: integration-test 77 | finally: 78 | - name: cleanup-test 79 | taskRef: 80 | Name: cleanup 81 | ``` 82 | 83 | In Couler, we can have `run_finally(finally_def)` function which is only available when the backend is Tekton. -------------------------------------------------------------------------------- /docs/examples.md: -------------------------------------------------------------------------------- 1 | # Examples 2 | 3 | ### Hello World! 4 | 5 | Let's start by running a very simple workflow template to echo "hello world" using the `docker/whalesay` container image from DockerHub. 6 | 7 | You can run this directly from your shell with a simple docker command: 8 | 9 | ``` 10 | $ docker run docker/whalesay cowsay "hello world" 11 | _____________ 12 | < hello world > 13 | ------------- 14 | \ 15 | \ 16 | \ 17 | ## . 18 | ## ## ## == 19 | ## ## ## ## === 20 | /""""""""""""""""___/ === 21 | ~~~ {~~ ~~~~ ~~~ ~~~~ ~~ ~ / ===- ~~~ 22 | \______ o __/ 23 | \ \ __/ 24 | \____\______/ 25 | ``` 26 | 27 | With Couler, we can run the same container on a Kubernetes cluster 28 | using an Argo Workflows as the workflow engine. 29 | 30 | ```python 31 | --8<-- "examples/hello_world.py" 32 | ``` 33 | 34 | ### Coin Flip 35 | 36 | This example combines the use of a Python function result, along with conditionals, 37 | to take a dynamic path in the workflow. In this example, depending on the result 38 | of the first step defined in `flip_coin()`, the template will either run the 39 | `heads()` step or the `tails()` step. 40 | 41 | Steps can be defined via either `couler.run_script()` 42 | for Python functions or `couler.run_container()` for containers. In addition, 43 | the conditional logic to decide whether to flip the coin in this example 44 | is defined via the combined use of `couler.when()` and `couler.equal()`. 45 | 46 | ```python 47 | --8<-- "examples/coin_flip.py" 48 | ``` 49 | 50 | ### DAG 51 | 52 | This example demonstrates different ways to define the workflow as a directed-acyclic graph (DAG) by specifying the 53 | dependencies of each task via `couler.set_dependencies()` and `couler.dag()`. Please see the code comments for the 54 | specific shape of DAG that we've defined in `linear()` and `diamond()`. 55 | 56 | ```python 57 | --8<-- "examples/dag.py" 58 | ``` 59 | 60 | Note that the current version only works with Argo Workflows but we are actively working on the design of the unified 61 | interface that is extensible to additional workflow engines. Please stay tuned for more updates and we welcome 62 | any feedback and contributions from the community. 63 | -------------------------------------------------------------------------------- /docs/getting-started.md: -------------------------------------------------------------------------------- 1 | ## Installation 2 | 3 | * Couler currently only supports Argo Workflows. Please see instructions [here](https://argoproj.github.io/argo/quick-start/#install-argo-workflows) 4 | to install Argo Workflows on your Kubernetes cluster. 5 | * Install Python 3.6+ 6 | * Install Couler Python SDK via the following `pip` command: 7 | 8 | ```bash 9 | pip install git+https://github.com/couler-proj/couler 10 | ``` 11 | Alternatively, you can clone this repository and then run the following to install: 12 | 13 | ```bash 14 | python setup.py install 15 | ``` 16 | 17 | After installing Couler, run the hello world example to submit your first workflow: 18 | 19 | ```python 20 | --8<-- "examples/hello_world.py" 21 | ``` 22 | 23 | Once the workflow is successfully submitted, the following logs will be shown: 24 | 25 | ``` 26 | INFO:root:Found local kubernetes config. Initialized with kube_config. 27 | INFO:root:Checking workflow name/generatedName runpy- 28 | INFO:root:Submitting workflow to Argo 29 | INFO:root:Workflow runpy-ddc2m has been submitted in "default" namespace! 30 | ``` 31 | -------------------------------------------------------------------------------- /examples/coin_flip.py: -------------------------------------------------------------------------------- 1 | import couler.argo as couler 2 | from couler.argo_submitter import ArgoSubmitter 3 | 4 | 5 | def random_code(): 6 | import random 7 | 8 | res = "heads" if random.randint(0, 1) == 0 else "tails" 9 | print(res) 10 | 11 | 12 | def flip_coin(): 13 | return couler.run_script(image="python:alpine3.6", source=random_code) 14 | 15 | 16 | def heads(): 17 | return couler.run_container( 18 | image="alpine:3.6", command=["sh", "-c", 'echo "it was heads"'] 19 | ) 20 | 21 | 22 | def tails(): 23 | return couler.run_container( 24 | image="alpine:3.6", command=["sh", "-c", 'echo "it was tails"'] 25 | ) 26 | 27 | 28 | result = flip_coin() 29 | couler.when(couler.equal(result, "heads"), lambda: heads()) 30 | couler.when(couler.equal(result, "tails"), lambda: tails()) 31 | 32 | submitter = ArgoSubmitter() 33 | couler.run(submitter=submitter) 34 | -------------------------------------------------------------------------------- /examples/dag.py: -------------------------------------------------------------------------------- 1 | import couler.argo as couler 2 | from couler.argo_submitter import ArgoSubmitter 3 | 4 | 5 | def job(name): 6 | couler.run_container( 7 | image="docker/whalesay:latest", 8 | command=["cowsay"], 9 | args=[name], 10 | step_name=name, 11 | ) 12 | 13 | 14 | # A 15 | # / \ 16 | # B C 17 | # / 18 | # D 19 | def linear(): 20 | couler.set_dependencies(lambda: job(name="A"), dependencies=None) 21 | couler.set_dependencies(lambda: job(name="B"), dependencies=["A"]) 22 | couler.set_dependencies(lambda: job(name="C"), dependencies=["A"]) 23 | couler.set_dependencies(lambda: job(name="D"), dependencies=["B"]) 24 | 25 | 26 | # A 27 | # / \ 28 | # B C 29 | # \ / 30 | # D 31 | def diamond(): 32 | couler.dag( 33 | [ 34 | [lambda: job(name="A")], 35 | [lambda: job(name="A"), lambda: job(name="B")], # A -> B 36 | [lambda: job(name="A"), lambda: job(name="C")], # A -> C 37 | [lambda: job(name="B"), lambda: job(name="D")], # B -> D 38 | [lambda: job(name="C"), lambda: job(name="D")], # C -> D 39 | ] 40 | ) 41 | 42 | 43 | linear() 44 | submitter = ArgoSubmitter() 45 | couler.run(submitter=submitter) 46 | -------------------------------------------------------------------------------- /examples/default_submitter.py: -------------------------------------------------------------------------------- 1 | import couler.argo as couler 2 | from couler.argo_submitter import ArgoSubmitter 3 | 4 | submitter = ArgoSubmitter(namespace="my_namespace") 5 | 6 | # Instead of setting up and passing a submitter each time, 7 | # you can just set it as default! 8 | couler.set_default_submitter(submitter) 9 | 10 | couler.run_container( 11 | image="docker/whalesay", 12 | command=["cowsay"], 13 | args=["hello world with default submitter"], 14 | ) 15 | # no submitter need be passed now! 16 | couler.run() 17 | 18 | couler.run_container( 19 | image="docker/whalesay", 20 | command=["cowsay"], 21 | args=["yet another hello world with default submitter"], 22 | ) 23 | # You can still pass a submitter to use, 24 | # overriding the default submitter if you want to 25 | couler.run(ArgoSubmitter(namespace="other_namespace")) 26 | -------------------------------------------------------------------------------- /examples/depends.py: -------------------------------------------------------------------------------- 1 | """ 2 | This example demonstrates how to use "depends". 3 | 4 | "depends" is an extension on "dependencies" to leverage the 5 | Workflow State as conditions: 6 | https://argoproj.github.io/argo-workflows/enhanced-depends-logic/ 7 | """ 8 | 9 | import sys 10 | import couler.argo as couler 11 | 12 | from couler.argo_submitter import ArgoSubmitter 13 | from typing import Callable 14 | 15 | def random_code() -> None: 16 | """ 17 | Randomly generate a 'success' or 'fail' 18 | to let sys.exit emulate a task final state 19 | """ 20 | import random 21 | task = ['success', 'fail'] 22 | res = random.randint(0, 1) 23 | res = task[res] 24 | print(f'{res}') 25 | if res == 'fail': 26 | sys.exit(2) 27 | 28 | 29 | def job(name: str, source: Callable) -> Callable: 30 | """ 31 | Create a Workflow run_script job template 32 | """ 33 | return couler.run_script( 34 | image="python:alpine3.6", 35 | source=source, 36 | step_name=name, 37 | ) 38 | 39 | # called if any task succeeded 40 | def any_succeeded() -> None: 41 | print('A Task Succeeded') 42 | 43 | # called if all tasks fail 44 | def all_failed() -> None: 45 | print('All Tasks Failed') 46 | 47 | 48 | def run_dag() -> None: 49 | """ 50 | Create a DAG to submit to Argo Workflows 51 | """ 52 | 53 | couler.set_dependencies( 54 | lambda: job(name='task1', source=random_code), 55 | dependencies=None 56 | ) 57 | 58 | couler.set_dependencies( 59 | lambda: job(name='task2', source=random_code), 60 | dependencies='task1.Failed' 61 | ) 62 | 63 | couler.set_dependencies( 64 | lambda: job(name='task3', source=random_code), 65 | dependencies='task2.Failed' 66 | ) 67 | 68 | couler.set_dependencies( 69 | lambda: job(name='task4', source=random_code), 70 | dependencies='task3.Failed' 71 | ) 72 | 73 | couler.set_dependencies( 74 | lambda: job(name='allfail', source=all_failed), 75 | dependencies='task1.Failed && task2.Failed && task3.Failed && task4.Failed' 76 | ) 77 | 78 | couler.set_dependencies( 79 | lambda: job(name='anysucceeded', source=any_succeeded), 80 | dependencies='task1.Succeeded || task2.Succeeded || task3.Succeeded || task4.Succeeded' 81 | ) 82 | 83 | 84 | if __name__ == '__main__': 85 | run_dag() 86 | submitter = ArgoSubmitter() 87 | couler.run(submitter=submitter) 88 | -------------------------------------------------------------------------------- /examples/hello_world.py: -------------------------------------------------------------------------------- 1 | import couler.argo as couler 2 | from couler.argo_submitter import ArgoSubmitter 3 | 4 | couler.run_container( 5 | image="docker/whalesay", command=["cowsay"], args=["hello world"] 6 | ) 7 | 8 | submitter = ArgoSubmitter() 9 | result = couler.run(submitter=submitter) 10 | -------------------------------------------------------------------------------- /examples/node_assign.py: -------------------------------------------------------------------------------- 1 | import couler.argo as couler 2 | from couler.argo_submitter import ArgoSubmitter 3 | from couler.core.templates.toleration import Toleration 4 | 5 | def job(name): 6 | # Define a nodes Toleration 7 | toleration = Toleration( 8 | key='', 9 | operator='', # Exists 10 | effect='', # NoSchedule, NoExecute, PreferNoSchedule 11 | ) 12 | # Add the toleration to the workflow 13 | couler.add_toleration(toleration) # pipeline/nodepool=pipe:NoSchedule 14 | couler.run_container( 15 | image="docker/whalesay:latest", 16 | command=["cowsay"], 17 | args=[name], 18 | step_name=name, 19 | node_selector={'':''} # Node Selector 20 | ) 21 | 22 | def simple(): 23 | couler.set_dependencies(lambda: job(name='Whale-Noise'), dependencies=None) 24 | 25 | simple() 26 | 27 | submitter = ArgoSubmitter(namespace='') 28 | couler.run(submitter=submitter) 29 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/couler-proj/couler 2 | 3 | go 1.13 4 | 5 | require ( 6 | github.com/alecthomas/assert v0.0.0-20170929043011-405dbfeb8e38 7 | github.com/alecthomas/colour v0.1.0 // indirect 8 | github.com/alecthomas/repr v0.0.0-20201120212035-bb82daffcca2 // indirect 9 | github.com/argoproj/argo v0.0.0-20210125193418-4cb5b7eb8075 10 | github.com/golang/protobuf v1.4.3 11 | github.com/google/go-cmp v0.5.2 // indirect 12 | github.com/mattn/go-isatty v0.0.12 // indirect 13 | golang.org/x/net v0.0.0-20201110031124-69a78807bb2b // indirect 14 | golang.org/x/sys v0.0.0-20201112073958-5cba982894dd // indirect 15 | golang.org/x/text v0.3.4 // indirect 16 | golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect 17 | google.golang.org/protobuf v1.25.0 // indirect 18 | k8s.io/api v0.18.2 19 | k8s.io/apimachinery v0.18.2 20 | k8s.io/client-go v0.18.2 21 | k8s.io/kube-openapi v0.0.0-20201113171705-d219536bb9fd // indirect 22 | ) 23 | 24 | replace ( 25 | k8s.io/api => k8s.io/api v0.17.8 26 | k8s.io/apimachinery => k8s.io/apimachinery v0.17.8 27 | k8s.io/client-go => k8s.io/client-go v0.17.8 28 | ) 29 | -------------------------------------------------------------------------------- /go/couler/commands/submit.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "C" 5 | "fmt" 6 | "github.com/couler-proj/couler/go/couler/conversion" 7 | pb "github.com/couler-proj/couler/go/couler/proto/couler/v1" 8 | "github.com/couler-proj/couler/go/couler/submitter" 9 | "github.com/golang/protobuf/proto" 10 | "io/ioutil" 11 | "os/user" 12 | "path/filepath" 13 | ) 14 | 15 | // Submit is the main function that submits a workflow protobuf. 16 | //export Submit 17 | func Submit(cProtoPath, cNamespace, cNamePrefix *C.char) *C.char { 18 | protoPath := C.GoString(cProtoPath) 19 | namespace := C.GoString(cNamespace) 20 | namePrefix := C.GoString(cNamePrefix) 21 | 22 | in, err := ioutil.ReadFile(protoPath) 23 | if err != nil { 24 | return wrapError(fmt.Errorf("failed to read workflow protobuf file: %w", err)) 25 | } 26 | pbWf := &pb.Workflow{} 27 | err = proto.Unmarshal(in, pbWf) 28 | if err != nil { 29 | return wrapError(fmt.Errorf("failed to unmarshal workflow pb: %w", err)) 30 | } 31 | argoWf, err := conversion.ConvertToArgoWorkflow(pbWf, namePrefix) 32 | if err != nil { 33 | return wrapError(fmt.Errorf("failed to convert to Argo Workflow: %w", err)) 34 | } 35 | 36 | // get current user to determine home directory 37 | usr, err := user.Current() 38 | if err != nil { 39 | return wrapError(fmt.Errorf("failed to get the current user: %w", err)) 40 | } 41 | sub := submitter.New(namespace, filepath.Join(usr.HomeDir, ".kube", "config")) 42 | submittedArgoWf, err := sub.Submit(argoWf, true) 43 | if err != nil && submittedArgoWf != nil { 44 | var errMsg string 45 | errMsg += fmt.Sprintf("Workflow read from protobuf %s failed due to: %v. \nStatuses of each workflow nodes:\n", submittedArgoWf.Name, err) 46 | for _, node := range submittedArgoWf.Status.Nodes { 47 | errMsg += fmt.Sprintf("Node %s %s. Message: %s\n", node.Name, node.Phase, node.Message) 48 | } 49 | return C.CString(errMsg) 50 | } 51 | return C.CString("Success") 52 | } 53 | 54 | func wrapError(err error) *C.char { 55 | return C.CString(err.Error()) 56 | } 57 | 58 | func main() {} 59 | -------------------------------------------------------------------------------- /go/couler/optimization/optimization.go: -------------------------------------------------------------------------------- 1 | package optimization 2 | 3 | import ( 4 | "fmt" 5 | 6 | pb "github.com/couler-proj/couler/go/couler/proto/couler/v1" 7 | ) 8 | 9 | // Pass is the interface of the optimization pass 10 | type Pass interface { 11 | Run(w *pb.Workflow) (*pb.Workflow, error) 12 | } 13 | 14 | // ComposedPass composed a sequence of optimization passes 15 | type ComposedPass struct { 16 | passes []Pass 17 | } 18 | 19 | // Run all optimization passes 20 | func (c *ComposedPass) Run(w *pb.Workflow) (*pb.Workflow, error) { 21 | var err error 22 | for i, pass := range c.passes { 23 | w, err = pass.Run(w) 24 | if err != nil { 25 | return nil, fmt.Errorf("optimization failed on %d-th pass", i) 26 | } 27 | } 28 | return w, nil 29 | } 30 | 31 | // Compose sequence optimization passes 32 | func Compose(passes ...Pass) *ComposedPass { 33 | return &ComposedPass{passes: passes} 34 | } 35 | -------------------------------------------------------------------------------- /go/couler/optimization/optimization_test.go: -------------------------------------------------------------------------------- 1 | package optimization 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/alecthomas/assert" 7 | pb "github.com/couler-proj/couler/go/couler/proto/couler/v1" 8 | ) 9 | 10 | type EndNodePass struct{} 11 | 12 | func (p *EndNodePass) Run(w *pb.Workflow) (*pb.Workflow, error) { 13 | stepList := &pb.ConcurrentSteps{ 14 | Steps: []*pb.Step{{Name: "endnode"}}, 15 | } 16 | w.Steps = append(w.Steps, stepList) 17 | return w, nil 18 | } 19 | 20 | func TestComposePass(t *testing.T) { 21 | a := assert.New(t) 22 | w := &pb.Workflow{} 23 | w.Steps = []*pb.ConcurrentSteps{ 24 | { 25 | Steps: []*pb.Step{{Name: "node1"}}, 26 | }, 27 | } 28 | 29 | // EndNodePass would add a node with name "endnode" at the tail 30 | opt := Compose(&EndNodePass{}) 31 | w, e := opt.Run(w) 32 | a.NoError(e) 33 | a.Equal(len(w.Steps), 2) 34 | a.Equal(w.Steps[1].Steps[0].Name, "endnode") 35 | } 36 | -------------------------------------------------------------------------------- /go/couler/proto/couler/v1/proto.go: -------------------------------------------------------------------------------- 1 | //go:generate protoc --go_out=../../. ../../../../../proto/couler.proto -I ../../../../../proto 2 | 3 | package v1 4 | -------------------------------------------------------------------------------- /go/couler/submitter/argo_submitter.go: -------------------------------------------------------------------------------- 1 | package submitter 2 | 3 | import ( 4 | "fmt" 5 | "k8s.io/client-go/rest" 6 | 7 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 8 | "k8s.io/apimachinery/pkg/fields" 9 | "k8s.io/client-go/tools/clientcmd" 10 | 11 | wfv1 "github.com/argoproj/argo/pkg/apis/workflow/v1alpha1" 12 | wfclientset "github.com/argoproj/argo/pkg/client/clientset/versioned" 13 | ) 14 | 15 | // ArgoWorkflowSubmitter holds configurations used for workflow submission 16 | type ArgoWorkflowSubmitter struct { 17 | namespace string 18 | kubeConfigPath string 19 | } 20 | 21 | // New returns ArgoWorkflowSubmitter struct 22 | func New(namespace, kubeConfigPath string) *ArgoWorkflowSubmitter { 23 | return &ArgoWorkflowSubmitter{ 24 | namespace: namespace, 25 | kubeConfigPath: kubeConfigPath, 26 | } 27 | } 28 | 29 | // Submit takes an Argo Workflow object and submit it to Kubernetes cluster 30 | func (submitter *ArgoWorkflowSubmitter) Submit(wf wfv1.Workflow, watch bool) (*wfv1.Workflow, error) { 31 | // Use the current context in kubeconfig 32 | config, err := clientcmd.BuildConfigFromFlags("", submitter.kubeConfigPath) 33 | if err != nil { 34 | fmt.Printf("failed to get the configuration from in the kubeconfig file %s: %s", submitter.kubeConfigPath, err) 35 | config, err = rest.InClusterConfig() 36 | if err != nil { 37 | return nil, fmt.Errorf("failed to get in-cluster configuration: %w", err) 38 | } 39 | } 40 | 41 | // Create the workflow client 42 | wfClient := wfclientset.NewForConfigOrDie(config).ArgoprojV1alpha1().Workflows(submitter.namespace) 43 | 44 | // Submit the workflow 45 | createdWf, err := wfClient.Create(&wf) 46 | if err != nil { 47 | return createdWf, fmt.Errorf("failed to create the workflow %s: %s", wf.Name, err) 48 | } 49 | fmt.Printf("Workflow %s successfully submitted\n", createdWf.Name) 50 | 51 | if watch { 52 | // Wait for the workflow to complete 53 | fieldSelector := fields.ParseSelectorOrDie(fmt.Sprintf("metadata.name=%s", createdWf.Name)) 54 | watchIf, err := wfClient.Watch(metav1.ListOptions{FieldSelector: fieldSelector.String()}) 55 | if err != nil { 56 | return createdWf, fmt.Errorf("failed establish a watch") 57 | } 58 | defer watchIf.Stop() 59 | for next := range watchIf.ResultChan() { 60 | wf, ok := next.Object.(*wfv1.Workflow) 61 | if !ok { 62 | continue 63 | } 64 | if !wf.Status.FinishedAt.IsZero() { 65 | fmt.Printf("Workflow %s %s at %v. Message: %s\n", wf.Name, wf.Status.Phase, wf.Status.FinishedAt, wf.Status.Message) 66 | if wf.Status.Phase == wfv1.NodeError || wf.Status.Phase == wfv1.NodeFailed { 67 | return wf, fmt.Errorf(wf.Status.Message) 68 | } 69 | return wf, nil 70 | } 71 | } 72 | } 73 | return createdWf, nil 74 | } 75 | -------------------------------------------------------------------------------- /go/couler/submitter/argo_submitter_test.go: -------------------------------------------------------------------------------- 1 | package submitter 2 | 3 | import ( 4 | "fmt" 5 | "github.com/alecthomas/assert" 6 | wfv1 "github.com/argoproj/argo/pkg/apis/workflow/v1alpha1" 7 | "github.com/couler-proj/couler/go/couler/conversion" 8 | pb "github.com/couler-proj/couler/go/couler/proto/couler/v1" 9 | "os" 10 | "os/user" 11 | "path/filepath" 12 | "testing" 13 | ) 14 | 15 | func TestArgoWorkflowSubmitter(t *testing.T) { 16 | toTest := os.Getenv("E2E_TEST") 17 | if toTest == "" || toTest == "false" { 18 | t.Skip("Skipping end-to-end tests") 19 | } 20 | pbWf := &pb.Workflow{} 21 | containerStep := &pb.Step{ 22 | Name: "container-test-step", 23 | TmplName: "container-test", ContainerSpec: &pb.ContainerSpec{ 24 | Image: "docker/whalesay:latest", 25 | Command: []string{"cowsay", "hello world"}, 26 | }} 27 | scriptStep := &pb.Step{ 28 | Name: "script-test-step", 29 | TmplName: "script-test", Script: "print(3)", ContainerSpec: &pb.ContainerSpec{ 30 | Image: "python:alpine3.6", 31 | Command: []string{"python"}, 32 | }} 33 | exitHandlerStep := &pb.Step{ 34 | Name: "exit-handler-step", 35 | TmplName: "exit-handler-step-template", 36 | ContainerSpec: &pb.ContainerSpec{ 37 | Image: "docker/whalesay:latest", 38 | Command: []string{"cowsay", "exiting"}, 39 | }, 40 | When: "{{workflow.status}} == Failed", 41 | } 42 | pbWf.ExitHandlerSteps = append(pbWf.ExitHandlerSteps, exitHandlerStep) 43 | // TODO (terrytangyuan): Debug why this step keeps running forever. 44 | //manifest := ` 45 | // apiVersion: v1 46 | // kind: Pod 47 | // metadata: 48 | // generateName: pi-job- 49 | // spec: 50 | // containers: 51 | // - name: pi 52 | // image: perl 53 | // command: ["perl", "-Mbignum=bpi", "-wle", "print bpi(2000)"]` 54 | //resourceStep := &pb.Step{ 55 | // Name: "resource-test-step", 56 | // TmplName: "resource-test", ResourceSpec: &pb.ResourceSpec{ 57 | // Manifest: manifest, 58 | // SuccessCondition: "status.phase == Succeeded", 59 | // FailureCondition: "status.phase == Failed", 60 | // SetOwnerReference: true, 61 | // Action: "create", 62 | // }, 63 | //} 64 | pbWf.Steps = []*pb.ConcurrentSteps{ 65 | {Steps: []*pb.Step{containerStep}}, 66 | {Steps: []*pb.Step{scriptStep}}, 67 | //{Steps: []*pb.Step{resourceStep}}, 68 | } 69 | 70 | argoWf, err := conversion.ConvertToArgoWorkflow(pbWf, "hello-world-") 71 | assert.NoError(t, err) 72 | 73 | // get current user to determine home directory 74 | usr, err := user.Current() 75 | assert.NoError(t, err) 76 | 77 | submitter := ArgoWorkflowSubmitter{ 78 | namespace: "argo", 79 | kubeConfigPath: filepath.Join(usr.HomeDir, ".kube", "config"), 80 | } 81 | finishedArgoWf, err := submitter.Submit(argoWf, true) 82 | if err != nil && finishedArgoWf != nil { 83 | fmt.Printf("Workflow %s failed due to %s. \nStatuses of each workflow nodes:\n", finishedArgoWf.Name, err) 84 | for _, node := range finishedArgoWf.Status.Nodes { 85 | fmt.Printf("Node %s %s. Message: %s\n", node.Name, node.Phase, node.Message) 86 | } 87 | } 88 | assert.NotNil(t, finishedArgoWf) 89 | assert.NoError(t, err) 90 | assert.Equal(t, wfv1.NodeSucceeded, finishedArgoWf.Status.Phase) 91 | assert.False(t, finishedArgoWf.Status.FinishedAt.IsZero()) 92 | 93 | unfinishedArgoWf, err := conversion.ConvertToArgoWorkflow(pbWf, "unfinished-hello-world-") 94 | assert.NoError(t, err) 95 | submittedUnfinishedArgoWf, err := submitter.Submit(unfinishedArgoWf, false) 96 | assert.NoError(t, err) 97 | assert.True(t, submittedUnfinishedArgoWf.Status.FinishedAt.IsZero()) 98 | } 99 | -------------------------------------------------------------------------------- /integration_tests/dag_depends_example.py: -------------------------------------------------------------------------------- 1 | # Copyright 2021 The Couler Authors. All rights reserved. 2 | # Licensed under the Apache License, Version 2.0 (the "License"); 3 | # you may not use this file except in compliance with the License. 4 | # You may obtain a copy of the License at 5 | # 6 | # http://www.apache.org/licenses/LICENSE-2.0 7 | # 8 | # Unless required by applicable law or agreed to in writing, software 9 | # distributed under the License is distributed on an "AS IS" BASIS, 10 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | # See the License for the specific language governing permissions and 12 | # limitations under the License. 13 | 14 | import couler.argo as couler 15 | from couler.argo_submitter import ArgoSubmitter 16 | 17 | if __name__ == "__main__": 18 | couler.config_workflow(timeout=3600, time_to_clean=3600 * 1.5) 19 | 20 | def pass_step(name): 21 | return couler.run_container( 22 | image="alpine:3.6", command=["sh", "-c", "exit 0"], step_name=name 23 | ) 24 | 25 | def fail_step(name): 26 | return couler.run_container( 27 | image="alpine:3.6", command=["sh", "-c", "exit 1"], step_name=name 28 | ) 29 | 30 | couler.set_dependencies(lambda: pass_step("A"), dependencies=None) 31 | couler.set_dependencies(lambda: pass_step("B"), dependencies="A") 32 | couler.set_dependencies(lambda: fail_step("C"), dependencies="A") 33 | couler.set_dependencies( 34 | lambda: pass_step("should-execute-1"), 35 | dependencies="A && (C.Succeeded || C.Failed)", 36 | ) 37 | couler.set_dependencies( 38 | lambda: pass_step("should-execute-2"), dependencies="B || C" 39 | ) 40 | couler.set_dependencies( 41 | lambda: pass_step("should-not-execute"), dependencies="B && C" 42 | ) 43 | couler.set_dependencies( 44 | lambda: pass_step("should-execute-3"), 45 | dependencies="should-execute-2.Succeeded || should-not-execute", 46 | ) 47 | 48 | submitter = ArgoSubmitter(namespace="argo") 49 | wf = couler.run(submitter=submitter) 50 | wf_name = wf["metadata"]["name"] 51 | print("Workflow %s has been submitted for DAG depends example" % wf_name) 52 | -------------------------------------------------------------------------------- /integration_tests/dag_example.py: -------------------------------------------------------------------------------- 1 | # Copyright 2021 The Couler Authors. All rights reserved. 2 | # Licensed under the Apache License, Version 2.0 (the "License"); 3 | # you may not use this file except in compliance with the License. 4 | # You may obtain a copy of the License at 5 | # 6 | # http://www.apache.org/licenses/LICENSE-2.0 7 | # 8 | # Unless required by applicable law or agreed to in writing, software 9 | # distributed under the License is distributed on an "AS IS" BASIS, 10 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | # See the License for the specific language governing permissions and 12 | # limitations under the License. 13 | 14 | import os 15 | 16 | import couler.argo as couler 17 | from couler.argo_submitter import ( 18 | _SUBMITTER_IMPL_ENV_VAR_KEY, 19 | ArgoSubmitter, 20 | _SubmitterImplTypes, 21 | ) 22 | 23 | 24 | def job(name): 25 | couler.run_container( 26 | image="docker/whalesay:latest", 27 | command=["cowsay"], 28 | args=[name], 29 | step_name=name, 30 | ) 31 | 32 | 33 | def exit_handler_succeeded(): 34 | return couler.run_container( 35 | image="alpine:3.6", 36 | command=["sh", "-c", 'echo "succeeded"'], 37 | step_name="success-exit", 38 | ) 39 | 40 | 41 | def exit_handler_failed(): 42 | return couler.run_container( 43 | image="alpine:3.6", 44 | command=["sh", "-c", 'echo "failed"'], 45 | step_name="failure-exit", 46 | ) 47 | 48 | 49 | def random_code(): 50 | import random 51 | 52 | res = "heads" if random.randint(0, 1) == 0 else "tails" 53 | print(res) 54 | 55 | 56 | def conditional_parent(): 57 | return couler.run_script( 58 | image="python:3.6", source=random_code, step_name="condition-parent" 59 | ) 60 | 61 | 62 | def conditional_child(): 63 | return couler.run_container( 64 | image="python:3.6", 65 | command=["bash", "-c", 'echo "child is triggered based on condition"'], 66 | step_name="condition-child", 67 | ) 68 | 69 | 70 | # A 71 | # / \ 72 | # B C 73 | # / 74 | # D 75 | def linear(): 76 | couler.set_dependencies(lambda: job(name="A"), dependencies=None) 77 | couler.set_dependencies(lambda: job(name="B"), dependencies=["A"]) 78 | couler.set_dependencies(lambda: job(name="C"), dependencies=["A"]) 79 | couler.set_dependencies(lambda: job(name="D"), dependencies=["B"]) 80 | 81 | 82 | # A 83 | # / \ 84 | # B C 85 | # \ / 86 | # D 87 | def diamond(): 88 | couler.dag( 89 | [ 90 | [lambda: job(name="A")], 91 | [lambda: job(name="A"), lambda: job(name="B")], # A -> B 92 | [lambda: job(name="A"), lambda: job(name="C")], # A -> C 93 | [lambda: job(name="B"), lambda: job(name="D")], # B -> D 94 | [lambda: job(name="C"), lambda: job(name="D")], # C -> D 95 | ] 96 | ) 97 | 98 | 99 | if __name__ == "__main__": 100 | for impl_type in [_SubmitterImplTypes.GO, _SubmitterImplTypes.PYTHON]: 101 | os.environ[_SUBMITTER_IMPL_ENV_VAR_KEY] = impl_type 102 | print( 103 | "Submitting DAG example workflow via %s implementation" % impl_type 104 | ) 105 | couler.config_workflow( 106 | name="dag-%s" % impl_type.lower(), 107 | timeout=3600, 108 | time_to_clean=3600 * 1.5, 109 | ) 110 | 111 | # 1) Add a linear DAG. 112 | linear() 113 | # 2) Add another step that depends on D and flips a coin. 114 | # 3) If the result is "heads", another child step is also 115 | # added to the entire workflow. 116 | couler.set_dependencies( 117 | lambda: couler.when( 118 | couler.equal(conditional_parent(), "heads"), 119 | lambda: conditional_child(), 120 | ), 121 | dependencies=["D"], 122 | ) 123 | # 4) Add an exit handler that runs when the workflow succeeds. 124 | couler.set_exit_handler( 125 | couler.WFStatus.Succeeded, exit_handler_succeeded 126 | ) 127 | # 5) Add an exit handler that runs when the workflow failed. 128 | couler.set_exit_handler(couler.WFStatus.Failed, exit_handler_failed) 129 | submitter = ArgoSubmitter(namespace="argo") 130 | couler.run(submitter=submitter) 131 | -------------------------------------------------------------------------------- /integration_tests/flip_coin_example.py: -------------------------------------------------------------------------------- 1 | # Copyright 2021 The Couler Authors. All rights reserved. 2 | # Licensed under the Apache License, Version 2.0 (the "License"); 3 | # you may not use this file except in compliance with the License. 4 | # You may obtain a copy of the License at 5 | # 6 | # http://www.apache.org/licenses/LICENSE-2.0 7 | # 8 | # Unless required by applicable law or agreed to in writing, software 9 | # distributed under the License is distributed on an "AS IS" BASIS, 10 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | # See the License for the specific language governing permissions and 12 | # limitations under the License. 13 | 14 | import os 15 | 16 | import couler.argo as couler 17 | from couler.argo_submitter import ( 18 | _SUBMITTER_IMPL_ENV_VAR_KEY, 19 | ArgoSubmitter, 20 | _SubmitterImplTypes, 21 | ) 22 | 23 | 24 | def random_code(): 25 | import random 26 | 27 | res = "heads" if random.randint(0, 1) == 0 else "tails" 28 | print(res) 29 | 30 | 31 | def flip_coin(): 32 | return couler.run_script(image="python:alpine3.6", source=random_code) 33 | 34 | 35 | def heads(): 36 | return couler.run_container( 37 | image="alpine:3.6", command=["sh", "-c", 'echo "it was heads"'] 38 | ) 39 | 40 | 41 | def tails(): 42 | return couler.run_container( 43 | image="alpine:3.6", command=["sh", "-c", 'echo "it was tails"'] 44 | ) 45 | 46 | 47 | if __name__ == "__main__": 48 | for impl_type in [_SubmitterImplTypes.GO, _SubmitterImplTypes.PYTHON]: 49 | os.environ[_SUBMITTER_IMPL_ENV_VAR_KEY] = impl_type 50 | print( 51 | "Submitting flip coin example workflow via %s implementation" 52 | % impl_type 53 | ) 54 | couler.config_workflow( 55 | name="flip-coin-%s" % impl_type.lower(), 56 | timeout=3600, 57 | time_to_clean=3600 * 1.5, 58 | ) 59 | result = flip_coin() 60 | couler.when(couler.equal(result, "heads"), lambda: heads()) 61 | couler.when(couler.equal(result, "tails"), lambda: tails()) 62 | 63 | submitter = ArgoSubmitter(namespace="argo") 64 | couler.run(submitter=submitter) 65 | -------------------------------------------------------------------------------- /integration_tests/flip_coin_security_context_example.py: -------------------------------------------------------------------------------- 1 | # Copyright 2021 The Couler Authors. All rights reserved. 2 | # Licensed under the Apache License, Version 2.0 (the "License"); 3 | # you may not use this file except in compliance with the License. 4 | # You may obtain a copy of the License at 5 | # 6 | # http://www.apache.org/licenses/LICENSE-2.0 7 | # 8 | # Unless required by applicable law or agreed to in writing, software 9 | # distributed under the License is distributed on an "AS IS" BASIS, 10 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | # See the License for the specific language governing permissions and 12 | # limitations under the License. 13 | 14 | import os 15 | 16 | import couler.argo as couler 17 | from couler.argo_submitter import ( 18 | _SUBMITTER_IMPL_ENV_VAR_KEY, 19 | ArgoSubmitter, 20 | _SubmitterImplTypes, 21 | ) 22 | 23 | 24 | def random_code(): 25 | import random 26 | 27 | res = "heads" if random.randint(0, 1) == 0 else "tails" 28 | print(res) 29 | 30 | 31 | def flip_coin(): 32 | return couler.run_script(image="python:alpine3.6", source=random_code) 33 | 34 | 35 | def heads(): 36 | return couler.run_container( 37 | image="alpine:3.6", command=["sh", "-c", 'echo "it was heads"'] 38 | ) 39 | 40 | 41 | def tails(): 42 | return couler.run_container( 43 | image="alpine:3.6", command=["sh", "-c", 'echo "it was tails"'] 44 | ) 45 | 46 | 47 | if __name__ == "__main__": 48 | for impl_type in [_SubmitterImplTypes.PYTHON]: 49 | os.environ[_SUBMITTER_IMPL_ENV_VAR_KEY] = impl_type 50 | print( 51 | "Submitting flip coin example workflow via %s implementation" 52 | % impl_type 53 | ) 54 | couler.config_workflow( 55 | name="flip-coin-sc-%s" % impl_type.lower(), 56 | timeout=3600, 57 | time_to_clean=3600 * 1.5, 58 | ) 59 | result = flip_coin() 60 | couler.when(couler.equal(result, "heads"), lambda: heads()) 61 | couler.when(couler.equal(result, "tails"), lambda: tails()) 62 | 63 | couler.states.workflow.set_security_context( 64 | security_context={"seLinuxOptions": {"level": "s0:c123,c456"}} 65 | ) 66 | 67 | submitter = ArgoSubmitter(namespace="argo") 68 | couler.run(submitter=submitter) 69 | -------------------------------------------------------------------------------- /integration_tests/memoization_example.py: -------------------------------------------------------------------------------- 1 | # Copyright 2021 The Couler Authors. All rights reserved. 2 | # Licensed under the Apache License, Version 2.0 (the "License"); 3 | # you may not use this file except in compliance with the License. 4 | # You may obtain a copy of the License at 5 | # 6 | # http://www.apache.org/licenses/LICENSE-2.0 7 | # 8 | # Unless required by applicable law or agreed to in writing, software 9 | # distributed under the License is distributed on an "AS IS" BASIS, 10 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | # See the License for the specific language governing permissions and 12 | # limitations under the License. 13 | 14 | import os 15 | 16 | import couler.argo as couler 17 | from couler.argo_submitter import ( 18 | _SUBMITTER_IMPL_ENV_VAR_KEY, 19 | ArgoSubmitter, 20 | _SubmitterImplTypes, 21 | ) 22 | 23 | if __name__ == "__main__": 24 | for impl_type in [_SubmitterImplTypes.GO, _SubmitterImplTypes.PYTHON]: 25 | os.environ[_SUBMITTER_IMPL_ENV_VAR_KEY] = impl_type 26 | print( 27 | "Submitting memoization example workflow via %s implementation" 28 | % impl_type 29 | ) 30 | couler.config_workflow( 31 | name="memoization-%s" % impl_type.lower(), 32 | timeout=3600, 33 | time_to_clean=3600 * 1.5, 34 | ) 35 | couler.run_container( 36 | image="alpine:3.6", 37 | command=["sh", "-c", 'echo "Hello world"'], 38 | cache=couler.Cache( 39 | name="cache-name", key="cache-key", max_age="60s" 40 | ), 41 | ) 42 | 43 | submitter = ArgoSubmitter(namespace="argo") 44 | couler.run(submitter=submitter) 45 | -------------------------------------------------------------------------------- /integration_tests/mpi_example.py: -------------------------------------------------------------------------------- 1 | # Copyright 2021 The Couler Authors. All rights reserved. 2 | # Licensed under the Apache License, Version 2.0 (the "License"); 3 | # you may not use this file except in compliance with the License. 4 | # You may obtain a copy of the License at 5 | # 6 | # http://www.apache.org/licenses/LICENSE-2.0 7 | # 8 | # Unless required by applicable law or agreed to in writing, software 9 | # distributed under the License is distributed on an "AS IS" BASIS, 10 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | # See the License for the specific language governing permissions and 12 | # limitations under the License. 13 | 14 | import os 15 | 16 | import couler.argo as couler 17 | from couler.argo_submitter import ( 18 | _SUBMITTER_IMPL_ENV_VAR_KEY, 19 | ArgoSubmitter, 20 | _SubmitterImplTypes, 21 | ) 22 | from couler.steps import mpi 23 | 24 | if __name__ == "__main__": 25 | for impl_type in [_SubmitterImplTypes.GO, _SubmitterImplTypes.PYTHON]: 26 | os.environ[_SUBMITTER_IMPL_ENV_VAR_KEY] = impl_type 27 | print( 28 | "Submitting DAG example workflow via %s implementation" % impl_type 29 | ) 30 | couler.config_workflow( 31 | "mpijob-%s" % impl_type.lower(), 32 | timeout=3600, 33 | time_to_clean=3600 * 1.5, 34 | ) 35 | 36 | mpi.train( 37 | image="alpine:3.6", 38 | command=["sh", "-c", 'echo "running"; exit 0'], 39 | num_workers=1, 40 | ) 41 | submitter = ArgoSubmitter(namespace="argo") 42 | couler.run(submitter=submitter) 43 | -------------------------------------------------------------------------------- /integration_tests/volume_example.py: -------------------------------------------------------------------------------- 1 | # Copyright 2021 The Couler Authors. All rights reserved. 2 | # Licensed under the Apache License, Version 2.0 (the "License"); 3 | # you may not use this file except in compliance with the License. 4 | # You may obtain a copy of the License at 5 | # 6 | # http://www.apache.org/licenses/LICENSE-2.0 7 | # 8 | # Unless required by applicable law or agreed to in writing, software 9 | # distributed under the License is distributed on an "AS IS" BASIS, 10 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | # See the License for the specific language governing permissions and 12 | # limitations under the License. 13 | 14 | import os 15 | 16 | import couler.argo as couler 17 | from couler.argo_submitter import ( 18 | _SUBMITTER_IMPL_ENV_VAR_KEY, 19 | ArgoSubmitter, 20 | _SubmitterImplTypes, 21 | ) 22 | from couler.core.templates.volume import VolumeMount 23 | 24 | if __name__ == "__main__": 25 | for impl_type in [_SubmitterImplTypes.PYTHON]: 26 | os.environ[_SUBMITTER_IMPL_ENV_VAR_KEY] = impl_type 27 | print( 28 | "Submitting volume example workflow via %s implementation" 29 | % impl_type 30 | ) 31 | couler.config_workflow( 32 | name="volume-%s" % impl_type.lower(), 33 | timeout=3600, 34 | time_to_clean=3600 * 1.5, 35 | ) 36 | # 2) Add a container to the workflow. 37 | couler.run_container( 38 | image="debian:latest", 39 | command=["/bin/bash", "-c"], 40 | args=[ 41 | ' vol_found=`mount | grep /tmp` && \ 42 | if [[ -n $vol_found ]]; \ 43 | then echo "Volume mounted and found"; \ 44 | else exit -1; fi ' 45 | ], 46 | volume_mounts=[VolumeMount("apppath", "/tmp")], 47 | ) 48 | submitter = ArgoSubmitter(namespace="argo") 49 | couler.run(submitter=submitter) 50 | -------------------------------------------------------------------------------- /mkdocs.yml: -------------------------------------------------------------------------------- 1 | site_name: Couler 2 | strict: true 3 | copyright: Copyright © 2021 The Couler Authors 4 | site_url: https://couler-proj.github.io/ 5 | site_description: >- 6 | Couler aims to provide a unified interface for constructing and managing workflows on different workflow engines, 7 | such as Argo Workflows, Tekton Pipelines, and Apache Airflow. 8 | 9 | repo_name: couler-proj/couler 10 | repo_url: https://github.com/couler-proj/couler 11 | 12 | nav: 13 | - Overview: README.md 14 | - Getting Started: getting-started.md 15 | - Examples: examples.md 16 | - Developer Guide: 17 | - Contributing: contributing.md 18 | - API Design: couler-api-design.md 19 | - Step Zoo: couler-step-zoo.md 20 | - Tekton Design: couler-tekton-design.md 21 | - Template: TEMPLATE.md 22 | - Adopters: adopters.md 23 | 24 | theme: 25 | logo: assets/logo-white.svg 26 | favicon: assets/logo.svg 27 | name: material 28 | font: 29 | text: Work Sans 30 | palette: 31 | - scheme: default 32 | toggle: 33 | icon: material/weather-night 34 | name: Switch to dark mode 35 | - scheme: slate 36 | toggle: 37 | icon: material/weather-sunny 38 | name: Switch to light mode 39 | 40 | extra: 41 | social: 42 | - icon: fontawesome/brands/github 43 | link: https://github.com/couler-proj/couler 44 | - icon: fontawesome/brands/twitter 45 | link: https://twitter.com/CoulerProject 46 | - icon: fontawesome/brands/slack 47 | link: https://join.slack.com/t/couler/shared_invite/zt-i0m7ziol-1gz4_p_aXmWM7KVpxJ4k9Q 48 | 49 | extra_css: 50 | - assets/stylesheets/extra.css 51 | 52 | markdown_extensions: 53 | - codehilite 54 | - admonition 55 | - pymdownx.superfences 56 | - pymdownx.tabbed 57 | - pymdownx.details 58 | - pymdownx.snippets 59 | - toc: 60 | permalink: true 61 | -------------------------------------------------------------------------------- /proto/couler.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | package couler.v1; 4 | 5 | import "google/protobuf/any.proto"; 6 | 7 | 8 | // Declare the Go package name here since the generated Go package with `couler_v1` is 9 | // not acceptable for `golint`. Reference: https://developers.google.com/protocol-buffers/docs/reference/go-generated 10 | option go_package = "couler/v1"; 11 | 12 | message Parameter { 13 | string name = 1; 14 | // value can be used in templates and steps 15 | // in templates: value is how the parameter can be referred to 16 | // in steps: actual value to be passed to the step 17 | string value = 2; 18 | string global_name = 3; 19 | } 20 | 21 | message Secret { 22 | string name = 1; 23 | string key = 2; 24 | string value = 3; // BASE64 encoded secret value 25 | } 26 | 27 | message Artifact { 28 | string name = 1; 29 | // value can be only used steps as the "from" field: 30 | // https://argoproj.github.io/argo/examples/#artifacts 31 | string value = 2; 32 | string type = 3; // HTTP/GIT/... 33 | string local_path = 4; 34 | string remote_path = 5; 35 | Secret access_key = 6; 36 | Secret secret_key = 7; 37 | string endpoint = 8; 38 | string bucket = 9; 39 | string global_name = 10; 40 | } 41 | 42 | message StdOut { 43 | string name = 1; 44 | } 45 | 46 | message StepIO { 47 | string name = 1; 48 | int32 source = 2; 49 | oneof step_io { 50 | Parameter parameter = 3; 51 | Artifact artifact = 4; 52 | StdOut stdout = 5; 53 | } 54 | } 55 | 56 | message VolumeMount { 57 | string name = 1; 58 | string path = 2; 59 | } 60 | 61 | message Cache { 62 | string name = 1; 63 | string key = 2; 64 | string max_age = 3; 65 | } 66 | 67 | message ContainerSpec { 68 | string image = 1; 69 | repeated string command = 2; 70 | map env = 3; 71 | map resources = 6; 72 | repeated VolumeMount volume_mounts = 7; 73 | } 74 | 75 | message ResourceSpec { 76 | string manifest = 1; 77 | string success_condition = 2; 78 | string failure_condition = 3; 79 | string action = 4; 80 | bool set_owner_reference = 5; 81 | } 82 | 83 | message CannedStepSpec{ 84 | string name = 1; 85 | map args = 2; 86 | repeated StepIO inputs = 3; 87 | repeated StepIO outputs = 4; 88 | } 89 | 90 | 91 | message Step { 92 | int32 id = 1; 93 | string name = 2; // name for reference 94 | string tmpl_name = 3; // name for generating template 95 | ContainerSpec container_spec = 4; 96 | ResourceSpec resource_spec = 5; 97 | CannedStepSpec canned_step_spec = 6; 98 | string script = 7; 99 | repeated StepIO args = 8; 100 | repeated string dependencies = 9; 101 | string when = 10; 102 | map attrs = 11; // attributes for step 103 | repeated Secret secrets = 12; 104 | Cache cache = 13; 105 | } 106 | 107 | message ConcurrentSteps { 108 | repeated Step steps = 1; 109 | } 110 | 111 | message StepTemplate { 112 | string name = 1; 113 | repeated StepIO inputs = 2; 114 | repeated StepIO outputs = 3; 115 | // TODO(typhoonzero): 116 | // add timeout, retry, daemon 117 | } 118 | 119 | message Workflow { 120 | string name = 1; 121 | // sequential steps: [ [step], [step], [step], ...] 122 | // concurrent steps: [ [step], [step, step, step], ...] 123 | // if dependencies was set, no matter what structure the 124 | // "steps" field stores, generate a DAG. 125 | repeated ConcurrentSteps steps = 2; 126 | map templates = 3; 127 | int32 parallelism = 4; 128 | string secret = 5; 129 | repeated Step exit_handler_steps = 6; 130 | map attrs = 7; // Workflow-level attributes 131 | } 132 | -------------------------------------------------------------------------------- /requirements-dev.txt: -------------------------------------------------------------------------------- 1 | pre-commit 2 | pytest 3 | pytest-cov 4 | Deprecated 5 | protobuf 6 | argo-workflows==3.5.1 7 | mkdocs==1.2.4 8 | mkdocs-material==7.1.3 9 | jinja2==3.0.0 -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | pyaml 2 | kubernetes>=11.0.0 3 | docker>=4.1.0 4 | Deprecated 5 | stringcase>=1.2.0 6 | StringGenerator>=0.4.4 7 | -------------------------------------------------------------------------------- /scripts/integration_tests.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # Copyright 2020 The Couler Authors. All rights reserved. 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | set -e 15 | 16 | # TODO(terrytangyuan): Temporarily disabled due to timeout. To be investigated. 17 | # python -m integration_tests.mpi_example 18 | python -m integration_tests.dag_example 19 | python -m integration_tests.dag_depends_example 20 | python -m integration_tests.flip_coin_example 21 | python -m integration_tests.flip_coin_security_context_example 22 | python -m integration_tests.memoization_example 23 | python -m integration_tests.volume_example 24 | 25 | # Validate workflow statuses 26 | kubectl -n argo get workflows 27 | for WF_NAME in $(kubectl -n argo get workflows --no-headers -o custom-columns=":metadata.name") 28 | do 29 | bash scripts/validate_workflow_statuses.sh ${WF_NAME} 30 | done 31 | -------------------------------------------------------------------------------- /scripts/test_go.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -e 3 | 4 | pb_file=./go/couler/proto/couler/v1/couler.pb.go 5 | cp $pb_file /tmp/old.pb.go 6 | 7 | go generate ./... 8 | 9 | # TODO: This check is temporarily disabled as it's flaky on GitHub Actions 10 | #pb_diff=$(diff /tmp/old.pb.go $pb_file) 11 | #if [[ ! -z "$pb_diff" ]]; then 12 | # echo "should commit generated protobuf files:" $pb_file 13 | # exit 1 14 | #fi 15 | 16 | go test ./... -v -------------------------------------------------------------------------------- /scripts/test_python.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -e 3 | 4 | pb_file=./couler/proto/couler_pb2.py 5 | cp $pb_file /tmp/old_pb2.py 6 | 7 | python setup.py proto 8 | 9 | # TODO: This check is temporarily disabled as it's flaky on GitHub Actions 10 | #pb_diff=$(diff /tmp/old_pb2.py $pb_file) 11 | #if [[ ! -z "$pb_diff" ]]; then 12 | # echo "should commit generated protobuf files:" $pb_file 13 | # exit 1 14 | #fi 15 | 16 | python setup.py install 17 | python -m pytest -------------------------------------------------------------------------------- /scripts/validate_workflow_statuses.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # Copyright 2020 The Couler Authors. All rights reserved. 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | # We intentionally `set +e` here since we want to allow certain kinds of known failures that can be ignored. 16 | # For example, workflows may not have been created yet so neither `kubectl get workflows` nor 17 | # `kubectl delete workflow` would be successful at earlier stages of this script. 18 | set +e 19 | 20 | WF_NAME=$1 21 | CHECK_INTERVAL_SECS=10 22 | 23 | function get_workflow_status { 24 | local wf_status=$(kubectl -n argo get workflow $1 -o jsonpath='{.status.phase}') 25 | echo ${wf_status} 26 | } 27 | 28 | for i in {1..70}; do 29 | WF_STATUS=$(get_workflow_status ${WF_NAME}) 30 | 31 | if [[ "$WF_STATUS" == "Succeeded" ]]; then 32 | echo "Workflow ${WF_NAME} succeeded." 33 | kubectl -n argo delete workflow ${WF_NAME} 34 | exit 0 35 | elif [[ "$WF_STATUS" == "Failed" ]] || 36 | [[ "$WF_STATUS" == "Error" ]]; then 37 | echo "Workflow ${WF_NAME} failed." 38 | kubectl -n argo describe workflow ${WF_NAME} 39 | exit 1 40 | else 41 | echo "Workflow ${WF_NAME} status: ${WF_STATUS}. Continue checking..." 42 | sleep ${CHECK_INTERVAL_SECS} 43 | fi 44 | done 45 | 46 | kubectl -n argo describe workflow ${WF_NAME} 47 | echo "Workflow ${WF_NAME} timed out." 48 | 49 | exit 1 50 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | # Copyright 2021 The Couler Authors. All rights reserved. 2 | # Licensed under the Apache License, Version 2.0 (the "License"); 3 | # you may not use this file except in compliance with the License. 4 | # You may obtain a copy of the License at 5 | # 6 | # http://www.apache.org/licenses/LICENSE-2.0 7 | # 8 | # Unless required by applicable law or agreed to in writing, software 9 | # distributed under the License is distributed on an "AS IS" BASIS, 10 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | # See the License for the specific language governing permissions and 12 | # limitations under the License. 13 | 14 | import distutils.cmd 15 | import io 16 | import os 17 | 18 | from setuptools import find_packages, setup 19 | 20 | 21 | class ProtocCommand(distutils.cmd.Command): 22 | description = "run protoc to compile protobuf files" 23 | user_options = [] 24 | 25 | def initialize_options(self): 26 | """Set default values for options.""" 27 | pass 28 | 29 | def finalize_options(self): 30 | """Post-process options.""" 31 | pass 32 | 33 | def run(self): 34 | os.system("mkdir -p couler/proto") 35 | os.system("protoc --python_out=couler proto/couler.proto") 36 | os.system("touch couler/proto/__init__.py") 37 | self.announce( 38 | "proto file generated under couler/proto.", 39 | level=distutils.log.INFO, 40 | ) 41 | 42 | 43 | with open("requirements.txt") as f: 44 | required_deps = f.read().splitlines() 45 | 46 | extras = {} 47 | with open("requirements-dev.txt") as f: 48 | extras["develop"] = f.read().splitlines() 49 | 50 | version = {} # type: dict 51 | with io.open(os.path.join("couler", "_version.py")) as fp: 52 | exec(fp.read(), version) 53 | 54 | setup( 55 | name="couler", 56 | description="Unified Interface for Constructing and Managing Workflows", 57 | long_description="Unified Interface for" 58 | " Constructing and Managing Workflows", 59 | version=version["__version__"], 60 | include_package_data=True, 61 | install_requires=required_deps, 62 | extras_require=extras, 63 | python_requires=">=3.6", 64 | packages=find_packages(exclude=["*test*"]), 65 | package_data={"": ["requirements.txt"]}, 66 | cmdclass={"proto": ProtocCommand}, 67 | ) 68 | -------------------------------------------------------------------------------- /templates/LICENSE.txt: -------------------------------------------------------------------------------- 1 | Copyright 2021 The Couler Authors. All rights reserved. 2 | Licensed under the Apache License, Version 2.0 (the "License"); 3 | you may not use this file except in compliance with the License. 4 | You may obtain a copy of the License at 5 | 6 | http://www.apache.org/licenses/LICENSE-2.0 7 | 8 | Unless required by applicable law or agreed to in writing, software 9 | distributed under the License is distributed on an "AS IS" BASIS, 10 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | See the License for the specific language governing permissions and 12 | limitations under the License. --------------------------------------------------------------------------------