├── .github └── workflows │ ├── build.yml │ └── ci.yml ├── .gitignore ├── .spec-docs.json ├── CONTRIBUTING.md ├── GOVERNANCE.md ├── LICENSE ├── README.md ├── USECASES.md └── openapi └── task_execution_service.openapi.yaml /.github/workflows/build.yml: -------------------------------------------------------------------------------- 1 | name: Build OpenAPI docs 2 | on: 3 | push: 4 | branches: 5 | - main 6 | - develop 7 | - 'release*' 8 | pull_request: 9 | types: [opened, reopened, sychronize] 10 | jobs: 11 | docs-build: 12 | runs-on: ubuntu-latest 13 | steps: 14 | - name: Setup Node.js environment 15 | uses: actions/setup-node@v4 16 | with: 17 | node-version: 14.x 18 | # Comes with npm 6. For newer Node, encountered: https://github.com/npm/cli/issues/3359 19 | - run: npm install -g @redocly/openapi-cli && npm install -g redoc-cli 20 | - run: npm install -g gh-openapi-docs 21 | - name: Check out repository code 22 | uses: actions/checkout@v4 23 | - run: gh-openapi-docs 24 | - name: Deploy 🚀 25 | uses: JamesIves/github-pages-deploy-action@v4 26 | with: 27 | branch: gh-pages 28 | clean: true # Clean only if on the main branch 29 | folder: . 30 | -------------------------------------------------------------------------------- /.github/workflows/ci.yml: -------------------------------------------------------------------------------- 1 | name: Lint and validate OpenAPI specs 2 | on: 3 | - push 4 | - pull_request_target 5 | jobs: 6 | lint: 7 | name: Lint OpenAPI definition 8 | runs-on: ubuntu-latest 9 | steps: 10 | - name: Check out head branch 11 | uses: actions/checkout@v4 12 | - name: Run OpenAPI Lint Action 13 | uses: mhiew/redoc-lint-github-action@v4 14 | with: 15 | args: 'openapi/task_execution_service.openapi.yaml' 16 | validate: 17 | name: Validate OpenAPI definition 18 | runs-on: ubuntu-latest 19 | steps: 20 | - name: Check out head branch 21 | uses: actions/checkout@v4 22 | - name: Run OpenAPI Validate Action 23 | uses: swaggerexpert/swagger-editor-validate@v1 24 | with: 25 | definition-file: openapi/task_execution_service.openapi.yaml 26 | 27 | diff: 28 | name: Show OpenAPI differences relative to target branch 29 | runs-on: ubuntu-latest 30 | outputs: 31 | diff_generated: ${{ steps.upload-log.outputs.artifact_id }} 32 | steps: 33 | - name: Check out head branch 34 | uses: actions/checkout@v4 35 | with: 36 | ref: ${{ github.ref }} 37 | path: head 38 | - name: Check out base branch 39 | uses: actions/checkout@v4 40 | with: 41 | ref: ${{ github.event.pull_request.base.ref }} 42 | path: base 43 | - name: Create output directory 44 | run: mkdir -p diff-artifacts/ 45 | - name: Pull Docker Image 46 | run: docker pull openapitools/openapi-diff:2.0.1 47 | - name: Run openapi-diff tool 48 | run: | 49 | docker run --rm \ 50 | -v $(pwd)/head:/head:ro \ 51 | -v $(pwd)/base:/base:ro \ 52 | -v $(pwd)/diff-artifacts:/local \ 53 | openapitools/openapi-diff:2.0.1 \ 54 | /base/openapi/task_execution_service.openapi.yaml \ 55 | /head/openapi/task_execution_service.openapi.yaml \ 56 | --markdown /local/diff.md 2> diff-artifacts/error.log 57 | - name: Get PR number 58 | id: get-pr-number 59 | run: | 60 | echo "${{ github.event.pull_request.number }}" > diff-artifacts/pr_number 61 | - name: Upload artifacts 62 | uses: actions/upload-artifact@v4 63 | with: 64 | name: diff-artifacts 65 | path: diff-artifacts/ 66 | if-no-files-found: ignore 67 | - name: Check if OpenAPI Diff failed 68 | id: check-diff 69 | run: | 70 | if [ -s diff-artifacts/error.log ]; then 71 | echo "The diff failed. Please see artifact error.log." 72 | exit 1 73 | fi 74 | 75 | permissions: 76 | contents: read 77 | pull-requests: write 78 | issues: write 79 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | bin 2 | pkg 3 | src 4 | -------------------------------------------------------------------------------- /.spec-docs.json: -------------------------------------------------------------------------------- 1 | { 2 | "apiSpecPath": "openapi/task_execution_service.openapi.yaml", 3 | "redocTheme": "ga4gh" 4 | } -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | 2 | # CONTRIBUTING 3 | 4 | This schema is developed by the [Cloud Work 5 | Stream]([https://ga4gh.cloud](https://www.ga4gh.org/work_stream/cloud/)) of the 6 | [Global Alliance for Genomics and Health](https://ga4gh.org). 7 | 8 | ## Semantic Versioning 9 | 10 | The Task Execution Service (TES) API uses [semantic 11 | versioning](https://semver.org/) for TES. Please consider that breaking changes 12 | imply a new major version release, which is associated with considerable 13 | administrative work and therefore happen only rarely. 14 | 15 | ## Suggesting Changes 16 | 17 | Changes to TES can be initiated as 18 | [**issues**](https://github.com/ga4gh/task-execution-schemas/issues) or 19 | [**pull requests**](https://github.com/ga4gh/task-execution-schemas/pulls) to 20 | allow for discussion and review. For considerable changes, we generally 21 | recommend opening issues first in order to discuss scope and feasibility. 22 | 23 | When creating pull requests, please do so from your own fork - even if you have 24 | write access to the repository. In this way, when the main repository is forked 25 | again, the new fork is created with a minimum of extraneous, volatile branches. 26 | 27 | > To facilitate the review of external pull requests, users are encouraged to 28 | > activate [**GitHub Actions**](https://github.com/features/actions) to monitor 29 | > the build status of their fork. By following the documentation for [deployment 30 | > to GitHub 31 | > Pages](https://docs.github.com/en/pages/getting-started-with-github-pages/configuring-a-publishing-source-for-your-github-pages-site), 32 | > pushes to the forked repository will be viewable at 33 | > `https://[user-or-org].github.io/task-execution-service/preview//`, 34 | > e.g.,: 35 | > - `https://[user-or-org].github.io/task-execution-schemas/preview/\/docs/` 36 | > - `https://[user-or-org].github.io/task-execution-schemas/preview/\/swagger-ui/` 37 | > - `https://[user-or-org].github.io/task-execution-schemas/preview/\/swagger.json` 38 | > - `https://[user-or-org].github.io/task-execution-schemas/preview/\/swagger.yaml` 39 | 40 | Providing this base URL in the pull request comment is appreciated, but not 41 | required. 42 | 43 | If a security vulnerability is identified with the specification, please send an 44 | email to detailing your concerns. 45 | 46 | For more information please refer to the [**governance 47 | documentation**](GOVERNANCE.md). 48 | -------------------------------------------------------------------------------- /GOVERNANCE.md: -------------------------------------------------------------------------------- 1 | TES Governance and Process 2 | ============================== 3 | 4 | The Task Execution Service (TES) operates under a community-driven development model, where advancements to the specification occur exclusively through community contributions. The standard is governed by a core team of Product Leads that fosters collaboration and consensus among contributors. This team is tasked with overseeing the design and development processes, setting priorities, managing the release schedule, and making decisions in instances where consensus cannot be reached. 5 | 6 | Current TES Product Leads are: 7 | 8 | | Name | Organization | github | 9 | |:-------------------|:-------------|:------------------ 10 | | Kyle Ellrot | [Oregon Health and Science University](https://www.ohsu.edu/) | [kellrott](https://github.com/kellrott) | 11 | | Venkat Malladi | [Microsoft](https://www.microsoft.com/en-us/genomics/) | [vsmalladi](https://github.com/vsmalladi) | 12 | | Alex Kanitz | [Swiss Institute of Bioinformatics / ELIXIR Switzerland](https://www.sib.swiss/) | [uniqueg](https://github.com/uniqueg) | 13 | 14 | 15 | ## Voting Procedure 16 | 17 | Proposed changes and releases are voted on by Driver Project champions and other key stakeholders, such as product implementers (client- and sever-side). Product Leads review proposed changes on a regular product-specific call, then solicit community feedback during regular Cloud Work Stream calls and through the [product-specific](mailto:ga4gh-cloud-tes+subscribe@ga4gh.org) mailing list for a specified time period. Proposed releases are further shared through official GA4GH channels (Slack board and mailing list with wider scope) and with a longer feedback period. Product Leads, in close connection with the Cloud Work Stream leadership, will always strive to reach broad consensus, but may accept simple majority decisions if broad consensus cannot be reached. 18 | 19 | Sign up for the [**GA4GH TES API mailing list**](mailto:ga4gh-cloud-tes+subscribe@ga4gh.org) to stay updated about the latest news and developments around the TES API, in particular for soliciting comments on proposed specification changes and for notifications about TES subgroup meetings. 20 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "{}" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright 2019 Global Alliance for Genomics and Health 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | GA4GH Logo 2 | 3 | [![License: Apache 2.0](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://www.apache.org/licenses/LICENSE-2.0) 4 | [![release_badge](https://img.shields.io/github/v/tag/ga4gh/task-execution-schemas?label=latest%20release&logo=github&style=flat)](https://github.com/ga4gh/task-execution-schemas/releases) 5 | 6 | | Branch | CI Docs | CI Specs | Swagger Validator | 7 | |---|---|---|---| 8 | | `main` | [![Build Status](https://github.com/ga4gh/task-execution-schemas/actions/workflows/build.yml/badge.svg?branch=main)](https://github.com/ga4gh/task-execution-schemas/actions/workflows/build.yml?query=branch%3Amain) | [![CI Status](https://github.com/ga4gh/task-execution-schemas/actions/workflows/ci.yml/badge.svg?branch=main)](https://github.com/ga4gh/task-execution-schemas/actions/workflows/ci.yml?query=branch%3Amain) | [![Swagger Validator](https://img.shields.io/swagger/valid/3.0?specUrl=https%3A%2F%2Fraw.githubusercontent.com%2Fga4gh%2Ftask-execution-schemas%2Fmain%2Fopenapi%2Ftask_execution_service.openapi.yaml&label=OpenAPI)](https://editor.swagger.io/?url=https://raw.githubusercontent.com/ga4gh/task-execution-schemas/refs/heads/main/openapi/task_execution_service.openapi.yaml) | 9 | | `develop` | [![Build Status](https://github.com/ga4gh/task-execution-schemas/actions/workflows/build.yml/badge.svg?branch=develop)](https://github.com/ga4gh/task-execution-schemas/actions/workflows/build.yml?query=branch%3Adevelop) | [![CI Status](https://github.com/ga4gh/task-execution-schemas/actions/workflows/ci.yml/badge.svg?branch=develop)](https://github.com/ga4gh/task-execution-schemas/actions/workflows/ci.yml?query=branch%3Adevelop) | [![Swagger Validator](https://img.shields.io/swagger/valid/3.0?specUrl=https%3A%2F%2Fraw.githubusercontent.com%2Fga4gh%2Ftask-execution-schemas%2Fdevelop%2Fopenapi%2Ftask_execution_service.openapi.yaml&label=OpenAPI)](https://editor.swagger.io/?url=https://raw.githubusercontent.com/ga4gh/task-execution-schemas/refs/heads/develop/openapi/task_execution_service.openapi.yaml) | 10 | 11 | 12 | 13 | Task Execution Service (TES) API 14 | ================================ 15 | 16 | This repository is the home for the schema of the Task Execution Service (TES) 17 | API defined by the [Cloud Work Stream](https://www.ga4gh.org/work_stream/cloud/) 18 | of the [Global Alliance for Genomics and Health (GA4GH)](https://ga4gh.org/). 19 | The goal of the API standard is to provide a uniform way to executing batch 20 | computing tasks. 21 | 22 | [GA4GH](https://ga4gh.org/) is an international coalition, formed to enable the 23 | sharing and processing of genomic data. 24 | 25 | The [Cloud Work Stream](https://www.ga4gh.org/work_stream/cloud/) helps the 26 | genomics and health communities take full advantage of modern cloud 27 | environments. Our initial focus is on “bringing the algorithms to the data”, by 28 | creating standards for defining, sharing, and executing portable workflows. 29 | 30 | We work with platform development partners and industry leaders to develop 31 | standards that will facilitate interoperability. 32 | 33 | What is TES? 34 | ============ 35 | 36 | The Task Execution Service (TES) API is an effort to define a standardized 37 | schema and API for describing batch execution tasks. A task defines a set of 38 | commands to run, a set of (Docker) containers to run them _in_, sets of input 39 | and output files, required resources, as well as some other metadata, e.g., for 40 | capturing provenance information. 41 | 42 | API Definition 43 | -------------- 44 | 45 | See the human-readable [**reference 46 | documentation**](https://ga4gh.github.io/task-execution-schemas/docs/). 47 | 48 | > The documentation hosted at 49 | > reflects the latest official API release from the `main` branch. To explore 50 | > the documentation from a development branch, append 51 | > `preview//docs/` to the base URL. For example, to view the 52 | > documentation for the latest **development version of the specification**, 53 | > visit . 54 | 55 | You can also examine the specification in the [**Swagger 56 | Editor**](https://editor.swagger.io/?url=https://raw.githubusercontent.com/ga4gh/task-execution-schemas/refs/heads/main/openapi/task_execution_service.openapi.yaml). 57 | 58 | > If you want to explore a version from a development branch, please load the 59 | > corresponding specification file (in 60 | > `openapi/task_execution_service.openapi.yaml`) manually into the [Swagger 61 | > Editor](https://editor.swagger.io/). 62 | 63 | TES Compliant Implementations 64 | ----------------------------- 65 | 66 | In alignment with GA4GH policies and regulations, security reviews are conducted 67 | for each major version release of the API. However, **no security guarantees are 68 | provided for any implementation of the API, including those linked from this 69 | page or the associated documentation**. Users are advised to proceed at their 70 | own risk and should arrange for a security audit of their application to ensure 71 | compliance with relevant regulatory and security standards, particularly when 72 | handling personal data. 73 | 74 | ### Client 75 | - [cwl-tes](https://github.com/ohsu-comp-bio/cwl-tes) 76 | - [Cromwell](https://cromwell.readthedocs.io/en/stable/backends/TES/) 77 | - [ELIXIR Cloud Components](https://elixir-cloud-components.vercel.app/) 78 | - [py-tes](https://github.com/ohsu-comp-bio/py-tes) 79 | - [Nextflow](https://www.nextflow.io/docs/stable/executor.html#ga4gh-tes) 80 | - [Snakemake](https://snakemake.readthedocs.io/en/stable/executing/cloud.html#executing-a-snakemake-workflow-via-ga4gh-tes) 81 | - [Toil](https://toil.readthedocs.io/en/latest/) 82 | - [St. Jude Rust Labs](https://github.com/stjude-rust-labs/tes) 83 | 84 | ### Server 85 | - [Funnel](https://ohsu-comp-bio.github.io/funnel/) 86 | - [TESK](https://github.com/EMBL-EBI-TSI/TESK) 87 | - [TES Azure](https://github.com/microsoft/ga4gh-tes) 88 | - [Pulsar](https://pulsar.readthedocs.io/) 89 | - [proTES](https://github.com/elixir-cloud-aai/proTES) 90 | 91 | ### Compatibility Matrix 92 | 93 | > Compatibility is assumed based on available documentation and limited tests 94 | > performed on latest versions of implementations available in December 2020. 95 | > Information may be outdated. 96 | 97 | | | cwl-tes | Cromwell | Nextflow | Snakemake | 98 | | ----------- | ------------- | --------- | ---------| --------- | 99 | | **Funnel** | [Compatible](https://github.com/ohsu-comp-bio/cwl-tes) | Compatible | [Compatible](https://www.nextflow.io/docs/latest/executor.html#ga4gh-tes) | [Compatible](https://snakemake.readthedocs.io/en/stable/executing/cloud.html#executing-a-snakemake-workflow-via-ga4gh-tes) | 100 | | **TESK** | [Compatible](https://github.com/ohsu-comp-bio/cwl-tes/pull/25) | [Compatible](https://cromwell.readthedocs.io/en/stable/backends/TES/) | [Compatible](https://github.com/EMBL-EBI-TSI/tesk-core/pull/19) | [Compatible](https://github.com/EMBL-EBI-TSI/tesk-core/pull/19) | [Compatible](https://github.com/elixir-cloud-aai/demo-tes-hybrid-cloud/tree/main) | 101 | | **TES Azure** | Not tested | [Compatible](https://github.com/microsoft/CromwellOnAzure) | [Compatible](https://www.nextflow.io/docs/stable/executor.html#ga4gh-tes) | [Compatible](https://github.com/microsoft/ga4gh-tes/blob/main/snakemake/examples/snakemake/README.md) | 102 | 103 | 104 | TES Service Examples 105 | -------------------- 106 | 107 | The API specification is available 108 | [here](openapi/task_execution_service.openapi.yaml) in [OpenAPI 109 | v3.0.1](https://github.com/OAI/OpenAPI-Specification/blob/master/versions/3.0.1.md). 110 | Clients may use JSON and REST to communicate with a service implementing the TES 111 | API. 112 | 113 | ### Creating a task 114 | 115 | Here's an example of a complete task message, defining a task which calculates 116 | an MD5 checksum on an input file and uploads the output: 117 | 118 | ```JSON 119 | { 120 | "name": "MD5 example", 121 | "description": "Task which runs md5sum on the input file.", 122 | "tags": { 123 | "custom-tag": "tag-value" 124 | }, 125 | "inputs": [ 126 | { 127 | "name": "infile", 128 | "description": "md5sum input file", 129 | "url": "/path/to/input_file", 130 | "path": "/container/input", 131 | "type": "FILE" 132 | } 133 | ], 134 | "outputs": [ 135 | { 136 | "name": "outfile", 137 | "url": "/path/to/output_file", 138 | "path": "/container/output" 139 | } 140 | ], 141 | "resources": { 142 | "cpuCores": 1, 143 | "ramGb": 1, 144 | "diskGb": 100, 145 | "preemptible": false 146 | }, 147 | "executors": [ 148 | { 149 | "image": "ubuntu", 150 | "command": [ 151 | "md5sum", 152 | "/container/input" 153 | ], 154 | "stdout": "/container/output", 155 | "stderr": "/container/stderr", 156 | "workdir": "/tmp" 157 | } 158 | ] 159 | } 160 | ``` 161 | 162 | A minimal version of the same task, including only the required fields looks 163 | like: 164 | 165 | ```JSON 166 | { 167 | "inputs": [ 168 | { 169 | "url": "/path/to/input_file", 170 | "path": "/container/input" 171 | } 172 | ], 173 | "outputs" : [ 174 | { 175 | "url" : "/path/to/output_file", 176 | "path" : "/container/output" 177 | } 178 | ], 179 | "executors" : [ 180 | { 181 | "image" : "ubuntu", 182 | "command" : ["md5sum", "/container/input"], 183 | "stdout" : "/container/output" 184 | } 185 | ] 186 | } 187 | ``` 188 | 189 | To create the task, send an HTTP `POST` request to the `/tasks` endpoint: 190 | 191 | ```HTTP 192 | POST /ga4gh/tes/v1/tasks 193 | ``` 194 | 195 | The response indicates an identifier for the created task resource: 196 | 197 | ```JSON 198 | { 199 | "id": "task-1234" 200 | } 201 | ``` 202 | 203 | ### Fetching a task 204 | 205 | To get a task by its identifier, send an HTTP `GET` request to the `/tasks/{id}` 206 | endpoint: 207 | 208 | ```HTTP 209 | GET /ga4gh/tes/v1/tasks/task-1234 210 | ``` 211 | 212 | The default minimal response will include the task state: 213 | 214 | ```JSON 215 | { 216 | "id": "task-1234", 217 | "state": "RUNNING" 218 | } 219 | ``` 220 | 221 | To get more information, you can change the task view using the `view` URL query 222 | parameter. 223 | 224 | The `BASIC` view will include all task fields except a few which might be large 225 | strings (stdout/stderr, system logging, input parameter contents): 226 | 227 | ```HTTP 228 | GET /ga4gh/tes/v1/tasks/task-1234?view=BASIC 229 | ``` 230 | 231 | ```JSON 232 | { 233 | "id": "task123", 234 | "name": "Sample Task", 235 | "description": "This is a sample task description.", 236 | "state": "COMPLETED", 237 | "inputs": [ 238 | { 239 | "name": "infile", 240 | "description": "Input file for the task.", 241 | "url": "/path/to/input_file", 242 | "path": "/container/input", 243 | "type": "FILE" 244 | } 245 | ], 246 | "outputs": [ 247 | { 248 | "name": "outfile", 249 | "url": "/path/to/output_file", 250 | "path": "/container/output" 251 | } 252 | ], 253 | "resources": { 254 | "cpuCores": 1, 255 | "ramGb": 2.0, 256 | "diskGb": 10.0, 257 | "preemptible": false 258 | }, 259 | "executors": [ 260 | { 261 | "image": "ubuntu:latest", 262 | "command": ["command", "arg1", "arg2"], 263 | "stdout": "/container/output", 264 | "stderr": "/container/stderr", 265 | "workdir": "/tmp" 266 | } 267 | ], 268 | "created": "2024-10-24T12:00:00Z", 269 | "updated": "2024-10-24T12:30:00Z" 270 | } 271 | ``` 272 | 273 | The `FULL` view includes stdout/stderr, system logs and full input parameters: 274 | 275 | ```HTTP 276 | GET /ga4gh/tes/v1/tasks/task-1234?view=FULL 277 | ``` 278 | 279 | ```JSON 280 | { 281 | "id": "job-0012345", 282 | "state": "COMPLETE", 283 | "name": "MD5 Checksum Task", 284 | "description": "This task computes the MD5 checksum of the input file.", 285 | "inputs": [ 286 | { 287 | "url": "s3://my-object-store/file1", 288 | "path": "/data/file1" 289 | } 290 | ], 291 | "outputs": [ 292 | { 293 | "path": "/data/outfile", 294 | "url": "s3://my-object-store/outfile-1", 295 | "type": "FILE" 296 | } 297 | ], 298 | "resources": { 299 | "cpu_cores": 4, 300 | "preemptible": false, 301 | "ram_gb": 8, 302 | "disk_gb": 40, 303 | "zones": "us-west-1", 304 | "backend_parameters": { 305 | "VmSize": "Standard_D64_v3" 306 | }, 307 | "backend_parameters_strict": false 308 | }, 309 | "executors": [ 310 | { 311 | "image": "ubuntu:20.04", 312 | "command": [ 313 | "/bin/md5", 314 | "/data/file1" 315 | ], 316 | "workdir": "/data/", 317 | "stdin": "/data/file1", 318 | "stdout": "/tmp/stdout.log", 319 | "stderr": "/tmp/stderr.log", 320 | "ignore_error": true 321 | } 322 | ], 323 | "volumes": [ 324 | "/vol/A/" 325 | ], 326 | "tags": { 327 | "WORKFLOW_ID": "cwl-01234", 328 | "PROJECT_GROUP": "alice-lab" 329 | }, 330 | "logs": [ 331 | { 332 | "logs": [ 333 | { 334 | "start_time": "2020-10-02T10:00:00-05:00", 335 | "end_time": "2020-10-02T11:00:00-05:00", 336 | "stdout": "MD5 checksum calculation completed successfully.", 337 | "stderr": "", 338 | "exit_code": 0 339 | } 340 | ], 341 | "metadata": { 342 | "host": "worker-001", 343 | "slurmm_id": 123456 344 | }, 345 | "start_time": "2020-10-02T10:00:00-05:00", 346 | "end_time": "2020-10-02T11:00:00-05:00", 347 | "outputs": [ 348 | { 349 | "url": "s3://my-object-store/outfile-1", 350 | "path": "/data/outfile", 351 | "size_bytes": 1024 352 | } 353 | ], 354 | "system_logs": [ 355 | "Task executed successfully without any issues." 356 | ] 357 | } 358 | ], 359 | "creation_time": "2020-10-02T10:00:00-05:00" 360 | } 361 | ``` 362 | 363 | ### Listing tasks 364 | 365 | To list all available tasks, send an HTTP `GET` requests to the `/tasks` 366 | endpoint: 367 | 368 | ```HTTP 369 | GET /ga4gh/tes/v1/tasks 370 | ``` 371 | 372 | ```JSON 373 | { 374 | "tasks": [ 375 | { 376 | "id": "job-0012345", 377 | "state": "COMPLETE" 378 | }, 379 | { 380 | "id": "job-0012346", 381 | "state": "RUNNING" 382 | }, 383 | { 384 | "id": "job-0012347", 385 | "state": "FAILED" 386 | } 387 | ] 388 | } 389 | ``` 390 | 391 | Similar to getting a task by ID, you may change the task view: 392 | 393 | ```HTTP 394 | GET /ga4gh/tes/v1/tasks?view=BASIC 395 | ``` 396 | 397 | ### Cancelling a task 398 | 399 | To cancel a task, send an HTTP `POST` request to the `tasks/{id}:cancel` 400 | endpoint: 401 | 402 | ```HTTP 403 | POST /ga4gh/tes/v1/tasks/task-1234:cancel 404 | ``` 405 | 406 | How to Contribute Changes 407 | ------------------------- 408 | 409 | ### Community Contributions and Spec Advancement 410 | 411 | The advancement of the GA4GH Task Execution Service (TES) API relies on active 412 | community engagement and contributions. While submitting issues is an effective 413 | way to report bugs or foster discussions about existing or proposed features, it 414 | is important to note that these actions alone typically do not lead to 415 | modifications in the specification. **The most effective method for implementing 416 | changes is through the submission of a pull request (PR).** 417 | 418 | For detailed guidance on how to contribute, please refer to the 419 | [**contributing documentation**](CONTRIBUTING.md). 420 | 421 | If a security issue is identified with the specification, please send an email to 422 | detailing your concerns. 423 | 424 | ### Governance 425 | 426 | The development of the TES specification is entirely community driven. However, 427 | development is overseen by a governance committee. For more information please 428 | refer to the [**governance documentation**](GOVERNANCE.md). 429 | -------------------------------------------------------------------------------- /USECASES.md: -------------------------------------------------------------------------------- 1 | 2 | The Cromwell group has 4 instances via 2 projects where TES is being considered. These are all similar enough to effectively be a single use case. The idea is to present an institution's local compute as a private cloud via a standardized interface in a similar manner to Google's Pipelines API. These would all fit PAPI's "docker as a service" model. In all cases some combination of polling or pubsub to detect job completion is necessary. For all of these use cases the intention is to handle situations where the data must stay local so merely describing where the local input/outputs should be is sufficient, but something more complex can be made to work. We will need to extract stdout/stderr and system logs. We will need authN/authZ as specified by the host site for all of these. 3 | -------------------------------------------------------------------------------- /openapi/task_execution_service.openapi.yaml: -------------------------------------------------------------------------------- 1 | openapi: 3.0.1 2 | info: 3 | title: Task Execution Service 4 | version: 1.1.0 5 | x-logo: 6 | url: 'https://www.ga4gh.org/wp-content/themes/ga4gh/dist/assets/svg/logos/logo-full-color.svg' 7 | license: 8 | name: Apache 2.0 9 | url: 'https://raw.githubusercontent.com/ga4gh/task-execution-schemas/develop/LICENSE' 10 | description: > 11 | ## Executive Summary 12 | 13 | The Task Execution Service (TES) API is a standardized schema and API for 14 | describing and executing batch execution tasks. A task defines a set of 15 | input files, a set of containers and commands to run, a set of 16 | output files and some other logging and metadata. 17 | 18 | 19 | TES servers accept task documents and execute them asynchronously on 20 | available compute resources. A TES server could be built on top of 21 | a traditional HPC queuing system, 22 | such as Grid Engine, Slurm or cloud style compute systems such as AWS Batch 23 | or Kubernetes. 24 | 25 | ## Introduction 26 | 27 | This document describes the TES API and provides details on the specific 28 | endpoints, request formats, and responses. It is intended to provide key 29 | information for developers of TES-compatible services as well as clients 30 | that will call these TES services. Use cases include: 31 | 32 | - Deploying existing workflow engines on new infrastructure. Workflow engines 33 | such as CWL-Tes and Cromwell have extentions for using TES. This will allow 34 | a system engineer to deploy them onto a new infrastructure using a job scheduling 35 | system not previously supported by the engine. 36 | 37 | - Developing a custom workflow management system. This API provides a common 38 | interface to asynchronous batch processing capabilities. A developer can write 39 | new tools against this interface and expect them to work using a variety of 40 | backend solutions that all support the same specification. 41 | 42 | 43 | ## Standards 44 | 45 | The TES API specification is written in OpenAPI and embodies a RESTful service 46 | philosophy. It uses JSON in requests and responses and standard 47 | HTTP/HTTPS for information transport. HTTPS should be used rather than plain HTTP 48 | except for testing or internal-only purposes. 49 | 50 | ### Authentication and Authorization 51 | 52 | Is is envisaged that most TES API instances will require users to authenticate to use the endpoints. 53 | However, the decision if authentication is required should be taken by TES API implementers. 54 | 55 | 56 | If authentication is required, we recommend that TES implementations use an OAuth2 bearer token, although they can choose other mechanisms if appropriate. 57 | 58 | 59 | Checking that a user is authorized to submit TES requests is a responsibility of TES implementations. 60 | 61 | ### CORS 62 | 63 | If TES API implementation is to be used by another website or domain it must implement Cross Origin Resource Sharing (CORS). 64 | Please refer to https://w3id.org/ga4gh/product-approval-support/cors for more information about GA4GH’s recommendations and how to implement CORS. 65 | 66 | security: 67 | # Optionally, this section can be removed or adjusted by implementers based on their needs. 68 | - BearerAuth: [] # Apply globally if authentication is required 69 | - BasicAuth: [] # Alternatively, use Basic Authentication 70 | 71 | servers: 72 | - url: /ga4gh/tes/v1 73 | paths: 74 | /service-info: 75 | get: 76 | tags: 77 | - TaskService 78 | summary: GetServiceInfo 79 | description: |- 80 | Provides information about the service, this structure is based on the 81 | standardized GA4GH service info structure. In addition, this endpoint 82 | will also provide information about customized storage endpoints offered 83 | by the TES server. 84 | operationId: GetServiceInfo 85 | responses: 86 | 200: 87 | description: "" 88 | content: 89 | application/json: 90 | schema: 91 | $ref: '#/components/schemas/tesServiceInfo' 92 | security: 93 | - BearerAuth: [] # Endpoint secured with Bearer token 94 | - BasicAuth: [] # Alternatively, secured with Basic Auth 95 | /tasks: 96 | get: 97 | tags: 98 | - TaskService 99 | summary: ListTasks 100 | description: |- 101 | List tasks tracked by the TES server. This includes queued, active and completed tasks. 102 | How long completed tasks are stored by the system may be dependent on the underlying 103 | implementation. 104 | operationId: ListTasks 105 | parameters: 106 | - name: name_prefix 107 | in: query 108 | description: |- 109 | OPTIONAL. Filter the list to include tasks where the name matches this prefix. 110 | If unspecified, no task name filtering is done. 111 | schema: 112 | type: string 113 | - name: state 114 | description: |- 115 | OPTIONAL. Filter tasks by state. If unspecified, 116 | no task state filtering is done. 117 | in: query 118 | required: false 119 | schema: 120 | $ref: '#/components/schemas/tesState' 121 | - name: tag_key 122 | description: |- 123 | OPTIONAL. Provide key tag to filter. The field tag_key is an array 124 | of key values, and will be zipped with an optional tag_value array. 125 | So the query: 126 | ``` 127 | ?tag_key=foo1&tag_value=bar1&tag_key=foo2&tag_value=bar2 128 | ``` 129 | Should be constructed into the structure { "foo1" : "bar1", "foo2" : "bar2"} 130 | 131 | ``` 132 | ?tag_key=foo1 133 | ``` 134 | Should be constructed into the structure {"foo1" : ""} 135 | 136 | If the tag_value is empty, it will be treated as matching any possible value. 137 | If a tag value is provided, both the tag's key and value must be exact 138 | matches for a task to be returned. 139 | Filter Tags Match? 140 | ---------------------------------------------------------------------- 141 | {"foo": "bar"} {"foo": "bar"} Yes 142 | {"foo": "bar"} {"foo": "bat"} No 143 | {"foo": ""} {"foo": ""} Yes 144 | {"foo": "bar", "baz": "bat"} {"foo": "bar", "baz": "bat"} Yes 145 | {"foo": "bar"} {"foo": "bar", "baz": "bat"} Yes 146 | {"foo": "bar", "baz": "bat"} {"foo": "bar"} No 147 | {"foo": ""} {"foo": "bar"} Yes 148 | {"foo": ""} {} No 149 | in: query 150 | required: false 151 | schema: 152 | type: array 153 | items: 154 | type: string 155 | - name: tag_value 156 | description: |- 157 | OPTIONAL. The companion value field for tag_key 158 | in: query 159 | required: false 160 | schema: 161 | type: array 162 | items: 163 | type: string 164 | - name: page_size 165 | in: query 166 | description: |- 167 | Optional number of tasks to return in one page. 168 | Must be less than 2048. Defaults to 256. 169 | schema: 170 | type: integer 171 | format: int32 172 | - name: page_token 173 | in: query 174 | description: |- 175 | OPTIONAL. Page token is used to retrieve the next page of results. 176 | If unspecified, returns the first page of results. The value can be found 177 | in the `next_page_token` field of the last returned result of ListTasks 178 | schema: 179 | type: string 180 | - $ref: '#/components/parameters/view' 181 | 182 | responses: 183 | 200: 184 | description: "" 185 | content: 186 | application/json: 187 | schema: 188 | $ref: '#/components/schemas/tesListTasksResponse' 189 | 190 | security: 191 | - BearerAuth: [] # Endpoint secured with Bearer token 192 | - BasicAuth: [] # Alternatively, secured with Basic Auth 193 | 194 | post: 195 | tags: 196 | - TaskService 197 | summary: CreateTask 198 | description: |- 199 | Create a new task. The user provides a Task document, which the server 200 | uses as a basis and adds additional fields. 201 | operationId: CreateTask 202 | requestBody: 203 | content: 204 | application/json: 205 | schema: 206 | $ref: '#/components/schemas/tesTask' 207 | required: true 208 | responses: 209 | 200: 210 | description: "" 211 | content: 212 | application/json: 213 | schema: 214 | $ref: '#/components/schemas/tesCreateTaskResponse' 215 | x-codegen-request-body-name: body 216 | security: 217 | - BearerAuth: [] # Endpoint secured with Bearer token 218 | - BasicAuth: [] # Alternatively, secured with Basic Auth 219 | /tasks/{id}: 220 | get: 221 | tags: 222 | - TaskService 223 | summary: GetTask 224 | description: |- 225 | Get a single task, based on providing the exact task ID string. 226 | operationId: GetTask 227 | parameters: 228 | - name: id 229 | in: path 230 | required: true 231 | description: ID of task to retrieve. 232 | schema: 233 | type: string 234 | - $ref: '#/components/parameters/view' 235 | responses: 236 | 200: 237 | description: "" 238 | content: 239 | application/json: 240 | schema: 241 | $ref: '#/components/schemas/tesTask' 242 | security: 243 | - BearerAuth: [] # Endpoint secured with Bearer token 244 | - BasicAuth: [] # Alternatively, secured with Basic Auth 245 | /tasks/{id}:cancel: 246 | post: 247 | tags: 248 | - TaskService 249 | summary: CancelTask 250 | description: Cancel a task based on providing an exact task ID. 251 | operationId: CancelTask 252 | parameters: 253 | - name: id 254 | in: path 255 | description: ID of task to be canceled. 256 | required: true 257 | schema: 258 | type: string 259 | responses: 260 | 200: 261 | description: "" 262 | content: 263 | application/json: 264 | schema: 265 | $ref: '#/components/schemas/tesCancelTaskResponse' 266 | security: 267 | - BearerAuth: [] # Endpoint secured with Bearer token 268 | - BasicAuth: [] # Alternatively, secured with Basic Auth 269 | components: 270 | securitySchemes: 271 | BearerAuth: 272 | type: http 273 | scheme: bearer 274 | bearerFormat: JWT # optional, can specify format if using JWT tokens 275 | 276 | BasicAuth: 277 | type: http 278 | scheme: basic 279 | 280 | parameters: 281 | view: 282 | name: view 283 | in: query 284 | description: |- 285 | OPTIONAL. Affects the fields included in the returned Task messages. 286 | 287 | `MINIMAL`: Task message will include ONLY the fields: 288 | - `tesTask.Id` 289 | - `tesTask.State` 290 | 291 | `BASIC`: Task message will include all fields EXCEPT: 292 | - `tesTask.ExecutorLog.stdout` 293 | - `tesTask.ExecutorLog.stderr` 294 | - `tesInput.content` 295 | - `tesTaskLog.system_logs` 296 | 297 | `FULL`: Task message includes all fields. 298 | schema: 299 | type: string 300 | default: MINIMAL 301 | enum: 302 | - MINIMAL 303 | - BASIC 304 | - FULL 305 | 306 | schemas: 307 | tesCancelTaskResponse: 308 | type: object 309 | description: CancelTaskResponse describes a response from the CancelTask endpoint. 310 | tesCreateTaskResponse: 311 | required: 312 | - id 313 | type: object 314 | properties: 315 | id: 316 | type: string 317 | description: Task identifier assigned by the server. 318 | description: |- 319 | CreateTaskResponse describes a response from the CreateTask endpoint. It 320 | will include the task ID that can be used to look up the status of the job. 321 | tesExecutor: 322 | required: 323 | - command 324 | - image 325 | type: object 326 | properties: 327 | image: 328 | type: string 329 | example: ubuntu:20.04 330 | description: |- 331 | Name of the container image. The string will be passed as the image 332 | argument to the containerization run command. Examples: 333 | - `ubuntu` 334 | - `quay.io/aptible/ubuntu` 335 | - `gcr.io/my-org/my-image` 336 | - `myregistryhost:5000/fedora/httpd:version1.0` 337 | command: 338 | type: array 339 | description: |- 340 | A sequence of program arguments to execute, where the first argument 341 | is the program to execute (i.e. argv). Example: 342 | ``` 343 | { 344 | "command" : ["/bin/md5", "/data/file1"] 345 | } 346 | ``` 347 | items: 348 | type: string 349 | example: ["/bin/md5", "/data/file1"] 350 | workdir: 351 | type: string 352 | description: |- 353 | The working directory that the command will be executed in. 354 | If not defined, the system will default to the directory set by 355 | the container image. 356 | example: /data/ 357 | stdin: 358 | type: string 359 | description: |- 360 | Path inside the container to a file which will be piped 361 | to the executor's stdin. This must be an absolute path. This mechanism 362 | could be used in conjunction with the input declaration to process 363 | a data file using a tool that expects STDIN. 364 | 365 | For example, to get the MD5 sum of a file by reading it into the STDIN 366 | ``` 367 | { 368 | "command" : ["/bin/md5"], 369 | "stdin" : "/data/file1" 370 | } 371 | ``` 372 | example: "/data/file1" 373 | stdout: 374 | type: string 375 | description: |- 376 | Path inside the container to a file where the executor's 377 | stdout will be written to. Must be an absolute path. Example: 378 | ``` 379 | { 380 | "stdout" : "/tmp/stdout.log" 381 | } 382 | ``` 383 | example: "/tmp/stdout.log" 384 | stderr: 385 | type: string 386 | description: |- 387 | Path inside the container to a file where the executor's 388 | stderr will be written to. Must be an absolute path. Example: 389 | ``` 390 | { 391 | "stderr" : "/tmp/stderr.log" 392 | } 393 | ``` 394 | example: "/tmp/stderr.log" 395 | env: 396 | type: object 397 | additionalProperties: 398 | type: string 399 | description: |- 400 | Enviromental variables to set within the container. Example: 401 | ``` 402 | { 403 | "env" : { 404 | "ENV_CONFIG_PATH" : "/data/config.file", 405 | "BLASTDB" : "/data/GRC38", 406 | "HMMERDB" : "/data/hmmer" 407 | } 408 | } 409 | ``` 410 | example: 411 | "BLASTDB" : "/data/GRC38" 412 | "HMMERDB" : "/data/hmmer" 413 | ignore_error: 414 | type: boolean 415 | description: |- 416 | Default behavior of running an array of executors is that execution 417 | stops on the first error. If `ignore_error` is `True`, then the 418 | runner will record error exit codes, but will continue on to the next 419 | tesExecutor. 420 | description: Executor describes a command to be executed, and its environment. 421 | tesExecutorLog: 422 | required: 423 | - exit_code 424 | type: object 425 | properties: 426 | start_time: 427 | type: string 428 | description: Time the executor started, in RFC 3339 format. 429 | example: 2020-10-02T10:00:00-05:00 430 | end_time: 431 | type: string 432 | description: Time the executor ended, in RFC 3339 format. 433 | example: 2020-10-02T11:00:00-05:00 434 | stdout: 435 | type: string 436 | description: |- 437 | Stdout content. 438 | 439 | This is meant for convenience. No guarantees are made about the content. 440 | Implementations may chose different approaches: only the head, only the tail, 441 | a URL reference only, etc. 442 | 443 | In order to capture the full stdout client should set Executor.stdout 444 | to a container file path, and use Task.outputs to upload that file 445 | to permanent storage. 446 | stderr: 447 | type: string 448 | description: |- 449 | Stderr content. 450 | 451 | This is meant for convenience. No guarantees are made about the content. 452 | Implementations may chose different approaches: only the head, only the tail, 453 | a URL reference only, etc. 454 | 455 | In order to capture the full stderr client should set Executor.stderr 456 | to a container file path, and use Task.outputs to upload that file 457 | to permanent storage. 458 | exit_code: 459 | type: integer 460 | description: Exit code. 461 | format: int32 462 | description: ExecutorLog describes logging information related to an Executor. 463 | tesFileType: 464 | type: string 465 | description: |- 466 | Define if input/output element is a file or a directory. It is not required 467 | that the user provide this value, but it is required that the server fill in the 468 | value once the information is avalible at run time. 469 | default: FILE 470 | enum: 471 | - FILE 472 | - DIRECTORY 473 | tesInput: 474 | required: 475 | - path 476 | type: object 477 | properties: 478 | name: 479 | type: string 480 | description: 481 | type: string 482 | url: 483 | type: string 484 | description: |- 485 | REQUIRED, unless "content" is set. 486 | 487 | URL in long term storage, for example: 488 | - s3://my-object-store/file1 489 | - gs://my-bucket/file2 490 | - file:///path/to/my/file 491 | - /path/to/my/file 492 | example: s3://my-object-store/file1 493 | path: 494 | type: string 495 | description: |- 496 | Path of the file inside the container. 497 | Must be an absolute path. 498 | example: /data/file1 499 | type: 500 | $ref: '#/components/schemas/tesFileType' 501 | content: 502 | type: string 503 | description: |- 504 | File content literal. 505 | 506 | Implementations should support a minimum of 128 KiB in this field 507 | and may define their own maximum. 508 | 509 | UTF-8 encoded 510 | 511 | If content is not empty, "url" must be ignored. 512 | streamable: 513 | type: boolean 514 | description: |- 515 | Indicate that a file resource could be accessed using a streaming 516 | interface, ie a FUSE mounted s3 object. This flag indicates that 517 | using a streaming mount, as opposed to downloading the whole file to 518 | the local scratch space, may be faster despite the latency and 519 | overhead. This does not mean that the backend will use a streaming 520 | interface, as it may not be provided by the vendor, but if the 521 | capacity is avalible it can be used without degrading the 522 | performance of the underlying program. 523 | description: Input describes Task input files. 524 | tesListTasksResponse: 525 | required: 526 | - tasks 527 | type: object 528 | properties: 529 | tasks: 530 | type: array 531 | description: |- 532 | List of tasks. These tasks will be based on the original submitted 533 | task document, but with other fields, such as the job state and 534 | logging info, added/changed as the job progresses. 535 | items: 536 | $ref: '#/components/schemas/tesTask' 537 | next_page_token: 538 | type: string 539 | description: |- 540 | Token used to return the next page of results. This value can be used 541 | in the `page_token` field of the next ListTasks request. 542 | description: ListTasksResponse describes a response from the ListTasks endpoint. 543 | tesOutput: 544 | required: 545 | - path 546 | - url 547 | type: object 548 | properties: 549 | name: 550 | type: string 551 | description: User-provided name of output file 552 | description: 553 | type: string 554 | description: Optional users provided description field, can be used for documentation. 555 | url: 556 | type: string 557 | description: |- 558 | URL at which the TES server makes the output accessible after the task is complete. 559 | When tesOutput.path contains wildcards, it must be a directory; see 560 | `tesOutput.path_prefix` for details on how output URLs are constructed in this case. 561 | For Example: 562 | - `s3://my-object-store/file1` 563 | - `gs://my-bucket/file2` 564 | - `file:///path/to/my/file` 565 | path: 566 | type: string 567 | description: |- 568 | Absolute path of the file inside the container. 569 | May contain pattern matching wildcards to select multiple outputs at once, but mind 570 | implications for `tesOutput.url` and `tesOutput.path_prefix`. 571 | Only wildcards defined in IEEE Std 1003.1-2017 (POSIX), 12.3 are supported; see 572 | https://pubs.opengroup.org/onlinepubs/9699919799/utilities/V3_chap02.html#tag_18_13 573 | path_prefix: 574 | type: string 575 | description: |- 576 | Prefix to be removed from matching outputs if `tesOutput.path` contains wildcards; 577 | output URLs are constructed by appending pruned paths to the directory specfied 578 | in `tesOutput.url`. 579 | Required if `tesOutput.path` contains wildcards, ignored otherwise. 580 | type: 581 | $ref: '#/components/schemas/tesFileType' 582 | description: Output describes Task output files. 583 | tesOutputFileLog: 584 | required: 585 | - path 586 | - size_bytes 587 | - url 588 | type: object 589 | properties: 590 | url: 591 | type: string 592 | description: URL of the file in storage, e.g. s3://bucket/file.txt 593 | path: 594 | type: string 595 | description: Path of the file inside the container. Must be an absolute 596 | path. 597 | size_bytes: 598 | type: string 599 | description: |- 600 | Size of the file in bytes. Note, this is currently coded as a string 601 | because official JSON doesn't support int64 numbers. 602 | format: int64 603 | example: 604 | - "1024" 605 | description: |- 606 | OutputFileLog describes a single output file. This describes 607 | file details after the task has completed successfully, 608 | for logging purposes. 609 | tesResources: 610 | type: object 611 | properties: 612 | cpu_cores: 613 | type: integer 614 | description: Requested number of CPUs 615 | format: int32 616 | example: 4 617 | preemptible: 618 | type: boolean 619 | description: |- 620 | Define if the task is allowed to run on preemptible compute instances, 621 | for example, AWS Spot. This option may have no effect when utilized 622 | on some backends that don't have the concept of preemptible jobs. 623 | format: boolean 624 | example: false 625 | ram_gb: 626 | type: number 627 | description: Requested RAM required in gigabytes (GB) 628 | format: double 629 | example: 8 630 | disk_gb: 631 | type: number 632 | description: Requested disk size in gigabytes (GB) 633 | format: double 634 | example: 40 635 | zones: 636 | type: array 637 | description: |- 638 | Request that the task be run in these compute zones. How this string 639 | is utilized will be dependent on the backend system. For example, a 640 | system based on a cluster queueing system may use this string to define 641 | priorty queue to which the job is assigned. 642 | items: 643 | type: string 644 | example: us-west-1 645 | backend_parameters: 646 | type: object 647 | additionalProperties: 648 | type: string 649 | description: |- 650 | Key/value pairs for backend configuration. 651 | ServiceInfo shall return a list of keys that a backend supports. 652 | Keys are case insensitive. 653 | It is expected that clients pass all runtime or hardware requirement key/values 654 | that are not mapped to existing tesResources properties to backend_parameters. 655 | Backends shall log system warnings if a key is passed that is unsupported. 656 | Backends shall not store or return unsupported keys if included in a task. 657 | If backend_parameters_strict equals true, 658 | backends should fail the task if any key/values are unsupported, otherwise, 659 | backends should attempt to run the task 660 | Intended uses include VM size selection, coprocessor configuration, etc. 661 | Example: 662 | ``` 663 | { 664 | "backend_parameters" : { 665 | "VmSize" : "Standard_D64_v3" 666 | } 667 | } 668 | ``` 669 | example: 670 | "VmSize" : "Standard_D64_v3" 671 | backend_parameters_strict: 672 | type: boolean 673 | description: |- 674 | If set to true, backends should fail the task if any backend_parameters 675 | key/values are unsupported, otherwise, backends should attempt to run the task 676 | format: boolean 677 | default: false 678 | example: false 679 | description: Resources describes the resources requested by a task. 680 | tesServiceType: 681 | allOf: 682 | - $ref: 'https://raw.githubusercontent.com/ga4gh-discovery/ga4gh-service-info/v1.0.0/service-info.yaml#/components/schemas/ServiceType' 683 | - type: object 684 | required: 685 | - artifact 686 | properties: 687 | artifact: 688 | type: string 689 | enum: [tes] 690 | example: tes 691 | tesServiceInfo: 692 | allOf: 693 | - $ref: 'https://raw.githubusercontent.com/ga4gh-discovery/ga4gh-service-info/v1.0.0/service-info.yaml#/components/schemas/Service' 694 | - type: object 695 | properties: 696 | storage: 697 | type: array 698 | description: |- 699 | Lists some, but not necessarily all, storage locations supported 700 | by the service. 701 | items: 702 | type: string 703 | example: 704 | - file:///path/to/local/funnel-storage 705 | - s3://ohsu-compbio-funnel/storage 706 | tesResources_backend_parameters: 707 | type: array 708 | description: |- 709 | Lists all tesResources.backend_parameters keys supported 710 | by the service 711 | items: 712 | type: string 713 | example: ["VmSize"] 714 | type: 715 | $ref: '#/components/schemas/tesServiceType' 716 | tesState: 717 | type: string 718 | readOnly: True 719 | description: |- 720 | Task state as defined by the server. 721 | 722 | - `UNKNOWN`: The state of the task is unknown. The cause for this status 723 | message may be dependent on the underlying system. The `UNKNOWN` states 724 | provides a safe default for messages where this field is missing so 725 | that a missing field does not accidentally imply that 726 | the state is QUEUED. 727 | - `QUEUED`: The task is queued and awaiting resources to begin computing. 728 | - `INITIALIZING`: The task has been assigned to a worker and is currently preparing to run. 729 | For example, the worker may be turning on, downloading input files, etc. 730 | - `RUNNING`: The task is running. Input files are downloaded and the first Executor 731 | has been started. 732 | - `PAUSED`: The task is paused. The reasons for this would be tied to 733 | the specific system running the job. An implementation may have the ability 734 | to pause a task, but this is not required. 735 | - `COMPLETE`: The task has completed running. Executors have exited without error 736 | and output files have been successfully uploaded. 737 | - `EXECUTOR_ERROR`: The task encountered an error in one of the Executor processes. Generally, 738 | this means that an Executor exited with a non-zero exit code. 739 | - `SYSTEM_ERROR`: The task was stopped due to a system error, but not from an Executor, 740 | for example an upload failed due to network issues, the worker's ran out 741 | of disk space, etc. 742 | - `CANCELED`: The task was canceled by the user, and downstream resources have been deleted. 743 | - `CANCELING`: The task was canceled by the user, 744 | but the downstream resources are still awaiting deletion. 745 | - `PREEMPTED`: The task is stopped (preempted) by the system. The reasons for this would be tied to 746 | the specific system running the job. Generally, this means that the system reclaimed the compute 747 | capacity for reallocation. 748 | default: UNKNOWN 749 | example: COMPLETE 750 | enum: 751 | - UNKNOWN 752 | - QUEUED 753 | - INITIALIZING 754 | - RUNNING 755 | - PAUSED 756 | - COMPLETE 757 | - EXECUTOR_ERROR 758 | - SYSTEM_ERROR 759 | - CANCELED 760 | - PREEMPTED 761 | - CANCELING 762 | tesTask: 763 | required: 764 | - executors 765 | type: object 766 | properties: 767 | id: 768 | type: string 769 | description: Task identifier assigned by the server. 770 | readOnly: true 771 | example: job-0012345 772 | state: 773 | $ref: '#/components/schemas/tesState' 774 | name: 775 | type: string 776 | description: User-provided task name. 777 | description: 778 | type: string 779 | description: |- 780 | Optional user-provided description of task for documentation purposes. 781 | inputs: 782 | type: array 783 | description: |- 784 | Input files that will be used by the task. Inputs will be downloaded 785 | and mounted into the executor container as defined by the task request 786 | document. 787 | items: 788 | $ref: '#/components/schemas/tesInput' 789 | example: 790 | - { "url" : "s3://my-object-store/file1", "path" : "/data/file1" } 791 | outputs: 792 | type: array 793 | description: |- 794 | Output files. 795 | Outputs will be uploaded from the executor container to long-term storage. 796 | items: 797 | $ref: '#/components/schemas/tesOutput' 798 | example: 799 | - { "path" : "/data/outfile", "url" : "s3://my-object-store/outfile-1", type: "FILE" } 800 | resources: 801 | $ref: '#/components/schemas/tesResources' 802 | executors: 803 | type: array 804 | description: |- 805 | An array of executors to be run. Each of the executors will run one 806 | at a time sequentially. Each executor is a different command that 807 | will be run, and each can utilize a different docker image. But each of 808 | the executors will see the same mapped inputs and volumes that are declared 809 | in the parent CreateTask message. 810 | 811 | Execution stops on the first error. 812 | items: 813 | $ref: '#/components/schemas/tesExecutor' 814 | volumes: 815 | type: array 816 | example: 817 | - "/vol/A/" 818 | description: |- 819 | Volumes are directories which may be used to share data between 820 | Executors. Volumes are initialized as empty directories by the 821 | system when the task starts and are mounted at the same path 822 | in each Executor. 823 | 824 | For example, given a volume defined at `/vol/A`, 825 | executor 1 may write a file to `/vol/A/exec1.out.txt`, then 826 | executor 2 may read from that file. 827 | 828 | (Essentially, this translates to a `docker run -v` flag where 829 | the container path is the same for each executor). 830 | items: 831 | type: string 832 | tags: 833 | type: object 834 | example: 835 | "WORKFLOW_ID" : "cwl-01234" 836 | "PROJECT_GROUP" : "alice-lab" 837 | 838 | additionalProperties: 839 | type: string 840 | description: |- 841 | A key-value map of arbitrary tags. These can be used to store meta-data 842 | and annotations about a task. Example: 843 | ``` 844 | { 845 | "tags" : { 846 | "WORKFLOW_ID" : "cwl-01234", 847 | "PROJECT_GROUP" : "alice-lab" 848 | } 849 | } 850 | ``` 851 | logs: 852 | type: array 853 | description: |- 854 | Task logging information. 855 | Normally, this will contain only one entry, but in the case where 856 | a task fails and is retried, an entry will be appended to this list. 857 | readOnly: true 858 | items: 859 | $ref: '#/components/schemas/tesTaskLog' 860 | creation_time: 861 | type: string 862 | description: |- 863 | Date + time the task was created, in RFC 3339 format. 864 | This is set by the system, not the client. 865 | example: 2020-10-02T10:00:00-05:00 866 | readOnly: true 867 | description: Task describes an instance of a task. 868 | tesTaskLog: 869 | required: 870 | - logs 871 | - outputs 872 | type: object 873 | properties: 874 | logs: 875 | type: array 876 | description: Logs for each executor 877 | items: 878 | $ref: '#/components/schemas/tesExecutorLog' 879 | metadata: 880 | type: object 881 | additionalProperties: 882 | type: string 883 | description: Arbitrary logging metadata included by the implementation. 884 | example: 885 | host: worker-001 886 | slurmm_id: 123456 887 | start_time: 888 | type: string 889 | description: When the task started, in RFC 3339 format. 890 | example: 2020-10-02T10:00:00-05:00 891 | end_time: 892 | type: string 893 | description: When the task ended, in RFC 3339 format. 894 | example: 2020-10-02T11:00:00-05:00 895 | outputs: 896 | type: array 897 | description: |- 898 | Information about all output files. Directory outputs are 899 | flattened into separate items. 900 | items: 901 | $ref: '#/components/schemas/tesOutputFileLog' 902 | system_logs: 903 | type: array 904 | description: |- 905 | System logs are any logs the system decides are relevant, 906 | which are not tied directly to an Executor process. 907 | Content is implementation specific: format, size, etc. 908 | 909 | System logs may be collected here to provide convenient access. 910 | 911 | For example, the system may include the name of the host 912 | where the task is executing, an error message that caused 913 | a SYSTEM_ERROR state (e.g. disk is full), etc. 914 | 915 | System logs are only included in the FULL task view. 916 | items: 917 | type: string 918 | description: TaskLog describes logging information related to a Task. 919 | --------------------------------------------------------------------------------