├── .dockerignore ├── .github └── workflows │ └── build.yaml ├── .gitignore ├── .travis.yml ├── LICENSE ├── README.md ├── charts └── tesk │ ├── .gitignore │ ├── .helmignore │ ├── Chart.yaml │ ├── README.md │ ├── ftp │ └── .netrc-TEMPLATE │ ├── s3-config │ ├── config-TEMPLATE │ └── credentials-TEMPLATE │ ├── service-info │ └── service-info.yaml │ ├── templates │ ├── common │ │ ├── oauth-client-secret.yaml │ │ ├── service-info-cm.yaml │ │ ├── taskmaster-rbac.yaml │ │ ├── tesk-deployment.yaml │ │ └── tesk-svc.yaml │ ├── ftp │ │ ├── ftp-endpoint.yaml │ │ ├── ftp-secret.yaml │ │ ├── ftp-service.yaml │ │ └── netrc-secret.yaml │ ├── ingress │ │ └── ingress-rules.yaml │ ├── openshift │ │ └── oc-route.yaml │ └── storage │ │ ├── aws-secret.yaml │ │ └── openstack.yaml │ ├── tls_secret_name.yml-TEMPLATE │ └── values.yaml ├── documentation ├── deployment.md ├── img │ ├── TESKlogo.png │ ├── TESKlogowfont.png │ ├── architecture.png │ └── project-architecture.png ├── integrated_wes_tes.md ├── local_ftp.md └── tesintro.md ├── examples ├── cancel │ └── counter.json ├── error │ ├── error_stops_execution.json │ ├── executor.json │ ├── input.json │ ├── output_new_dir_file.json │ └── output_new_folder.json ├── localftp │ └── taskWithIO.json ├── resources │ ├── cpu.json │ └── more_cpu_than_nodes.json ├── success │ ├── env.json │ ├── hello.json │ ├── input_content.json │ ├── input_dir_duplicate_file_names.json │ ├── input_dir_file_duplicate_file_names.json │ ├── input_dir_ftp01_tree.json │ ├── input_dir_ftp02_sub.json │ ├── input_dir_ftp03_tree_shadowing.json │ ├── input_dir_ftp04_tree_shadowing.json │ ├── input_dir_ftp05_merge.json │ ├── input_dir_ftp06_file_merge.json │ ├── input_file_dir_merge.json │ ├── input_file_duplicate_names.json │ ├── input_file_http.json │ ├── input_file_http_nested_mounts.json │ ├── output.json │ ├── output_overwrite.json │ ├── output_tree.json │ ├── output_volume.json │ ├── stderr.json │ ├── stdin.json │ ├── stdout.json │ ├── tags.json │ └── workdir.json ├── taskCreate └── taskList └── source └── tesk-core ├── .dockerignore ├── .github └── workflows │ ├── docker-build-publish-filer.yml │ ├── docker-build-publish-taskmaster.yml │ └── tox.yml ├── .gitignore ├── .travis.yml ├── LICENSE ├── MANIFEST.in ├── README.md ├── cloudbuild.yaml ├── cloudbuild_testing.yaml ├── containers ├── filer.Dockerfile └── taskmaster.Dockerfile ├── doc └── taskmaster_architecture.png ├── dockerBuild ├── dockerRun ├── examples ├── inputFile.json ├── inputHelloWorld.json ├── inputHttp.json └── transferPvc │ ├── Readme.md │ ├── clean │ ├── minikubeStart │ ├── pod.yaml │ ├── pv.yaml │ └── pvc.yaml ├── init ├── install ├── pytest.ini ├── scripts ├── dockerBuild ├── dockerRun └── run ├── setup.cfg ├── setup.py ├── src └── tesk_core │ ├── README.md │ ├── Util.py │ ├── __init__.py │ ├── exception.py │ ├── filer.py │ ├── filer_class.py │ ├── filer_s3.py │ ├── job.py │ ├── path.py │ ├── pvc.py │ ├── taskmaster.py │ └── transput.py ├── taskmaster ├── tests ├── FilerClassTest.py ├── TaskMasterTest.py ├── assertThrows.py ├── resources │ ├── copyDirTest │ │ └── src │ │ │ ├── 3.txt │ │ │ └── a │ │ │ ├── 1.txt │ │ │ └── 2.txt │ ├── inputFile.json │ └── test_config ├── test_filer.py ├── test_filer_ftp_pytest.py ├── test_filer_general_pytest.py ├── test_filer_http_pytest.py ├── test_job.py ├── test_s3_filer.py └── test_taskmaster.py └── tox.ini /.dockerignore: -------------------------------------------------------------------------------- 1 | tesk-core/containers/ 2 | -------------------------------------------------------------------------------- /.github/workflows/build.yaml: -------------------------------------------------------------------------------- 1 | name: Helm Chart Testing 2 | 3 | on: 4 | pull_request: 5 | branches: [master] 6 | 7 | jobs: 8 | helm: 9 | runs-on: ubuntu-latest 10 | steps: 11 | - name: Checkout 12 | uses: actions/checkout@v3.3.0 13 | with: 14 | fetch-depth: 0 15 | 16 | - name: Create k3s Cluster 17 | uses: debianmaster/actions-k3s@v1.0.5 18 | with: 19 | version: 'latest' 20 | 21 | - name: Create namespace 22 | run: kubectl create ns tesk 23 | 24 | - name: Helm Deps 25 | run: | 26 | for dir in $(ls -d charts/*); do 27 | helm dependency update $dir; 28 | done 29 | 30 | - name: Helm Lint 31 | run: | 32 | for dir in $(ls -d charts/*); do 33 | helm lint $dir 34 | done 35 | 36 | - name: Apply Helm file 37 | run: helm install -n tesk tesk . -f values.yaml 38 | working-directory: charts/tesk 39 | 40 | - name: Sleep for 30 seconds 41 | run: sleep 30 42 | 43 | - name: Get Helm and k8s 44 | run: helm list -n tesk && kubectl get all -n tesk 45 | 46 | - name: curl URL 47 | run: curl localhost -vL 48 | 49 | - name: Configure Git 50 | run: | 51 | git config user.name "$GITHUB_ACTOR" 52 | git config user.email "$GITHUB_ACTOR@users.noreply.github.com" 53 | 54 | - name: Run chart-releaser 55 | uses: helm/chart-releaser-action@v1.6.0 56 | with: 57 | skip_existing: true 58 | env: 59 | CR_TOKEN: "${{ secrets.GITHUB_TOKEN }}" 60 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.swp 2 | *.pyc 3 | #*.yaml 4 | !/deployment/wes-tes/** 5 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: python 2 | env: 3 | global: 4 | - HELM_URL=https://get.helm.sh 5 | - HELM_TGZ=helm-v3.5.3-linux-amd64.tar.gz 6 | install: 7 | # Install Helm 8 | - wget -q ${HELM_URL}/${HELM_TGZ} 9 | - tar xzf ${HELM_TGZ} 10 | - PATH=`pwd`/linux-amd64/:$PATH 11 | script: 12 | - helm lint ./charts/tesk 13 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright 2018 European Molecular Biology Laboratory 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | An implementation of a task execution engine based on the [TES standard](https://github.com/ga4gh/task-execution-schemas) running on `Kubernetes`. For more details on `TES`, see the (very) brief [introduction to TES](documentation/tesintro.md). 5 | 6 | For organisational reasons, this project is split into 3 repositories: 7 | - This one, which contains documentation and deployment files 8 | - [tesk-api](https://github.com/elixir-cloud-aai/tesk-api): Contains the service that implements the TES API and translates tasks into kubernetes batch calls 9 | - [tesk-core](https://github.com/elixir-cloud-aai/tesk-core): Contains the code that is launched as images into the kubernetes cluster by tesk-api. 10 | 11 | If the API is running on your cluster it will pull the images from our `docker.io` repository automatically. 12 | 13 | `TESK` is designed with the goal to support any `Kubernetes` cluster, for its deployment please refer to the [deployment](documentation/deployment.md) page. 14 | 15 | The technical documentation is available in the [documentation](documentation) folder. 16 | 17 | 18 | ## Architecture 19 | As a diagram: 20 | 21 | ![TESK architecture](documentation/img/architecture.png) 22 | 23 | **Description**: The first pod in the task lifecycle is the API pod, a pod which runs a web server (`Tomcat`) and exposes the `TES` specified endpoints. It consumes `TES` requests, validates them and translates them to `Kubernetes` jobs. The API pod then creates a `task controller` pod, or `taskmaster`. 24 | 25 | The `taskmaster` consumes the executor jobs, inputs and outputs. It first creates `filer` pod, which creates a persistent volume claim (PVC) to mount as scratch space. All mounts are initialized and all files are downloaded into the locations specified in the TES request; the populated PVC can then be used by each executor pod one after the other. After the `filer` has finished, the taskmaster goes through the executors and executes them as pods one by one. **Note**: Each TES task has a separate taskmaster, PVC and executor pods belonging to it; the only 'singleton' pod across tasks is the API pod. 26 | 27 | After the last executor, the `filer` is called once more to process the outputs and push them to remote locations from the PVC. The PVC is the scrubbed, deleted and the taskmaster ends, completing the task. 28 | 29 | ## Requirements 30 | - A working [Kubernetes](https://kubernetes.io/) cluster version 1.9 and later. 31 | - If you want TESK to handle tasks with I/O (and you probably want), you additionally need: 32 | - A default storage class, which TESK will use to create temporary PVCs. It is enough that the storage class supports the RWO mode. 33 | - And, if you want TESK to integrate with workflow managers, you additionally need either an FTP account or a PVC that can be accessed from within or from outside of the cluster by the workflow manager (more in the [deployment](documentation/deployment.md) page). 34 | -------------------------------------------------------------------------------- /charts/tesk/.gitignore: -------------------------------------------------------------------------------- 1 | ftp/.netrc 2 | s3-config/config 3 | s3-config/credentials 4 | secrets.yaml 5 | -------------------------------------------------------------------------------- /charts/tesk/.helmignore: -------------------------------------------------------------------------------- 1 | # Patterns to ignore when building packages. 2 | # This supports shell glob matching, relative path matching, and 3 | # negation (prefixed with !). Only one pattern per line. 4 | .DS_Store 5 | # Common VCS dirs 6 | .git/ 7 | .gitignore 8 | .bzr/ 9 | .bzrignore 10 | .hg/ 11 | .hgignore 12 | .svn/ 13 | # Common backup files 14 | *.swp 15 | *.bak 16 | *.tmp 17 | *~ 18 | # Various IDEs 19 | .project 20 | .idea/ 21 | *.tmproj 22 | .vscode/ 23 | 24 | -------------------------------------------------------------------------------- /charts/tesk/Chart.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v2 2 | name: tesk 3 | description: A TESK-EXILIR Helm chart for Kubernetes 4 | type: application 5 | version: 0.1.1 6 | appVersion: dev 7 | -------------------------------------------------------------------------------- /charts/tesk/README.md: -------------------------------------------------------------------------------- 1 | # Deployment instructions for TESK 2 | 3 | Helm chart and config to deploy the TESK service. Tested with Helm v3.0.0. 4 | 5 | ## Usage 6 | 7 | First you must create a namespace in Kubernetes in which to deploy TESK. In development clusters such as Minikube it is fine to use the `default` namespace. The 8 | command below creates all deployments in the context of this namespace. How 9 | the namespace is created depends on the cluster, so it is not documented here. 10 | 11 | To deploy the application: 12 | * modify [`values.yaml`](values.yaml) 13 | * If you are installing the FTP storage backend (and will not use .netrc file for FTP credentials) and/or OIDC client, create a `secrets.yaml` file. You need to fill up the `username` and `password` of the ftp account that will be potentially used to exchange I/O with a workflow manager such as cwl-tes. If you activated authentication (auth.mode == 'auth' in `values.yaml`), optionally you may activate the OICD client in the Swagger UI as well (you need to register the client by your OIDC provider). To do so, supply the `client_id` and `client_secret` values obtained during the client registration, otherwise the auth section must be removed. 14 | 15 | ``` 16 | ftp: 17 | username: 18 | password: 19 | 20 | auth: 21 | client_id: 22 | client_secret: 23 | ``` 24 | 25 | * If you're using `s3` as your storage option, do not forget to add the necessary `config` and `credentials` files 26 | (see [here](https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-files.html)) under a folder named 27 | **s3-config** (*charts/tesk/s3-config*). 28 | 29 | * If you are installing the FTP storage backend and want to use .netrc file for FTP credentials, place the .netrc in the `ftp` folder. There is a template in the folder. 30 | 31 | * Finally execute: 32 | 33 | ```bash 34 | $ helm upgrade --install tesk-release . -f secrets.yaml -f values.yaml 35 | ``` 36 | 37 | then you can check it all went as expected: 38 | 39 | ```bash 40 | $ helm list -n tesk 41 | NAME NAMESPACE REVISION UPDATED STATUS CHART APP VERSION 42 | tesk-release tesk 16 2020-05-14 11:38:45.521995325 +0300 EEST deployed tesk-0.1.0 dev 43 | ``` 44 | 45 | The first time running this command, the chart will be installed. Afterwards, it will apply any change in the chart. 46 | 47 | *IMPORTANT:* In kubernetes, if you want to install TESK in a namespace different than `default`, it must be specified by using `-n `, so the final command would be something like this: 48 | 49 | ```bash 50 | $ helm upgrade -n tesk --install tesk-release . -f secrets.yaml -f values.yaml 51 | ``` 52 | *Note*: If you're running Helm 3, you might need to also use the `--create-namespace` option, as non-existent namespaces 53 | do not get created by default (see [this](https://github.com/helm/helm/issues/6794)). 54 | 55 | ## Description of values 56 | 57 | See [`values.yaml`](values.yaml) for default values. 58 | 59 | | Key | Type | Description | 60 | | --- | --- | --- | 61 | | host_name | string | FQDN to expose the application | 62 | | storageClass | string | Name of a user preferred storage class (default is empty) | 63 | | storage | string | Can be either 'openstack' or 's3' | 64 | | tesk.image | string | container image (including the version) to be used to run TESK API | 65 | | tesk.port | integer | | 66 | | tesk.taskmaster_image_version | string | the version of the image to be used to run TESK Taskmaster Job | 67 | | tesk.taskmaster_filer_image_version | string | the version of the image to be used to run TESK Filer Job | 68 | | tesk.executor_retries| int | The number of retries on error - actual task compute (executor)| 69 | | tesk.filer_retries| int | The number of retries on error while handling I/O (filer)| 70 | | tesk.debug | boolean | Activates the debugging mode | 71 | | tesk.securityContext.enabled | boolean | Enable securityContext | 72 | | transfer.wes_base_path | string | | 73 | | transfer.tes_base_path | string | | 74 | | transfer.pvc_name | string | | 75 | | auth.mode | string | Can be 'noauth' to disable authentication, or 'auth' to enable it | 76 | | auth.env_subgroup | string | Can be 'EBI' or 'CSC' | 77 | | service.type | string | Can be 'NodePort' or 'ClusterIp' or 'LoadBalancer' | 78 | | service.node_port | integer | Only used if service.type is 'NodePort', specifies the port | 79 | | ftp.classic_ftp_secret | String | The name of a secret to store FTP credentials as keys. If empty, the old-style FTP secret is not created | 80 | | ftp.netrc_secret | String | The name of a secret to store FTP credentials as a netrc file. If empty, the netrc FTP secret is not created | 81 | | ftp.hostip | string | IP of the endpoint of the ftp as seen by containers in K8s (only needed, if in need of a DNS entry for locally installed FTP server) | 82 | | ingress.rules| boolean | Apply or not the ingress rule | 83 | | ingress.ingressClassName | string | Name of the Ingress Class | 84 | | ingress.path | string | | 85 | | ingress.tls_secret_name | string | If no TLS secret name configured, TLS will be switched off. A template can be found at [deployment/tls_secret_name.yml-TEMPLATE](deployment/tls_secret_name.yml-TEMPLATE). If you are using cert-manager the secret will be created automatically.| 86 | | ingress.annotations | string | Annotations for the ingress rules | 87 | -------------------------------------------------------------------------------- /charts/tesk/ftp/.netrc-TEMPLATE: -------------------------------------------------------------------------------- 1 | machine ftp-private.ebi.ac.uk 2 | login ftp-username 3 | password ftp-password 4 | -------------------------------------------------------------------------------- /charts/tesk/s3-config/config-TEMPLATE: -------------------------------------------------------------------------------- 1 | [default] 2 | # Non-standard entry, parsed by TESK, not boto3 3 | endpoint_url=http://localhost:9000 4 | -------------------------------------------------------------------------------- /charts/tesk/s3-config/credentials-TEMPLATE: -------------------------------------------------------------------------------- 1 | [default] 2 | aws_access_key_id=AKIAIOSFODNN7EXAMPLE 3 | aws_secret_access_key=wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY -------------------------------------------------------------------------------- /charts/tesk/service-info/service-info.yaml: -------------------------------------------------------------------------------- 1 | id: 'uk.ac.ebi.tsc.tesk' 2 | name: TESK 3 | type: 4 | group: org.ga4gh 5 | artifact: TES 6 | version: 1.0 7 | description: GA4GH TES Server implementation for Kubernetes 8 | organization: 9 | name: EMBL-EBI 10 | url: https://www.ebi.ac.uk/ 11 | contactUrl: https://github.com/EMBL-EBI-TSI/TESK/issues 12 | documentationUrl: https://github.com/EMBL-EBI-TSI/TESK 13 | createdAt: 2021-04-06 14 | environment: dev 15 | version: 1.0 16 | storage: 17 | - ftp://ftp-private.ebi.ac.uk 18 | -------------------------------------------------------------------------------- /charts/tesk/templates/common/oauth-client-secret.yaml: -------------------------------------------------------------------------------- 1 | {{ if eq .Values.auth.mode "auth" }} 2 | apiVersion: v1 3 | stringData: 4 | id: {{ .Values.auth.client_id }} 5 | secret: {{ .Values.auth.client_secret }} 6 | kind: Secret 7 | metadata: 8 | name: oauth-client-secret 9 | type: Opaque 10 | {{ end }} 11 | -------------------------------------------------------------------------------- /charts/tesk/templates/common/service-info-cm.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: ConfigMap 3 | metadata: 4 | name: service-info-config 5 | data: 6 | {{ (.Files.Glob "service-info/*").AsConfig | indent 2 }} 7 | -------------------------------------------------------------------------------- /charts/tesk/templates/common/taskmaster-rbac.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: v1 3 | kind: ServiceAccount 4 | metadata: 5 | name: taskmaster 6 | --- 7 | apiVersion: rbac.authorization.k8s.io/v1 8 | kind: Role 9 | metadata: 10 | name: taskmaster 11 | rules: 12 | - apiGroups: 13 | - "" 14 | resources: 15 | - pods 16 | verbs: 17 | - list 18 | - patch 19 | - apiGroups: 20 | - "" 21 | resources: 22 | - persistentvolumeclaims 23 | - configmaps 24 | verbs: 25 | - create 26 | - delete 27 | - get 28 | - apiGroups: 29 | - "" 30 | resources: 31 | - pods/log 32 | - pods/status 33 | verbs: 34 | - get 35 | - list 36 | - apiGroups: 37 | - batch 38 | resources: 39 | - jobs 40 | verbs: 41 | - create 42 | - delete 43 | - get 44 | - list 45 | - patch 46 | --- 47 | kind: RoleBinding 48 | apiVersion: rbac.authorization.k8s.io/v1 49 | metadata: 50 | name: taskmaster-role-binding 51 | subjects: 52 | - kind: ServiceAccount 53 | name: taskmaster 54 | roleRef: 55 | kind: Role 56 | name: taskmaster 57 | apiGroup: rbac.authorization.k8s.io 58 | -------------------------------------------------------------------------------- /charts/tesk/templates/common/tesk-deployment.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: apps/v1 2 | kind: Deployment 3 | metadata: 4 | name: tesk-api 5 | spec: 6 | selector: 7 | matchLabels: 8 | app: tesk-api 9 | replicas: 1 10 | template: 11 | metadata: 12 | labels: 13 | app: tesk-api 14 | spec: 15 | serviceAccountName: taskmaster 16 | containers: 17 | - name: tesk-api 18 | image: {{ .Values.tesk.image }} 19 | resources: 20 | limits: 21 | cpu: {{ .Values.tesk.limitsCpu }} 22 | memory: {{ .Values.tesk.limitsMemory }} 23 | requests: 24 | cpu: {{ .Values.tesk.requestsCpu }} 25 | memory: {{ .Values.tesk.requestsMemory }} 26 | env: 27 | - name: TESK_API_TASKMASTER_IMAGE_NAME 28 | value: {{ .Values.tesk.taskmaster_image_name }} 29 | - name: TESK_API_TASKMASTER_IMAGE_VERSION 30 | value: {{ .Values.tesk.taskmaster_image_version }} 31 | - name: TESK_API_TASKMASTER_FILER_IMAGE_NAME 32 | value: {{ .Values.tesk.taskmaster_filer_image_name }} 33 | - name: TESK_API_TASKMASTER_FILER_IMAGE_VERSION 34 | value: {{ .Values.tesk.taskmaster_filer_image_version }} 35 | - name: TESK_API_K8S_NAMESPACE 36 | value: {{.Release.Namespace}} 37 | - name: TESK_API_TASKMASTER_SERVICE_ACCOUNT_NAME 38 | value: taskmaster 39 | - name: TESK_API_TASKMASTER_ENVIRONMENT_EXECUTOR_BACKOFF_LIMIT 40 | value: {{ .Values.tesk.executor_retries | quote }} 41 | - name: TESK_API_TASKMASTER_ENVIRONMENT_FILER_BACKOFF_LIMIT 42 | value: {{ .Values.tesk.filer_retries | quote }} 43 | - name: SERVER_SERVLET_CONTEXT_PATH 44 | value: {{ .Values.ingress.path }} 45 | {{ if .Values.tesk.tes_api_base_path }} 46 | - name: OPENAPI_TASKEXECUTIONSERVICE_BASE_PATH 47 | value: {{ .Values.tesk.tes_api_base_path }} 48 | {{ end }} 49 | - name: TESK_API_SERVICE_INFO_LOCATION 50 | value: file:///etc/tesk/service-info/service-info.yaml 51 | {{ if .Values.ftp.classic_ftp_secret }} 52 | - name: TESK_API_TASKMASTER_FTP_SECRET_NAME 53 | value: {{ .Values.ftp.classic_ftp_secret }} 54 | {{ end }} 55 | {{ if .Values.ftp.netrc_secret }} 56 | - name: TESK_API_TASKMASTER_ENVIRONMENT_NETRC_SECRET_NAME 57 | value: {{ .Values.ftp.netrc_secret }} 58 | {{ end }} 59 | - name: TESK_API_AUTHORISATION_ENV_SUBGROUP 60 | value: {{ .Values.auth.env_subgroup }} 61 | - name: TESK_API_AUTHORISATION_BASE_GROUP 62 | value: {{ .Values.auth.env_basegroup }} 63 | - name: SPRING_PROFILES_ACTIVE 64 | value: {{ .Values.auth.mode }} 65 | 66 | {{ if .Values.transfer.active }} 67 | - name: TESK_API_TASKMASTER_ENVIRONMENT_HOST_BASE_PATH 68 | value: {{ .Values.transfer.wes_base_path }} 69 | - name: TESK_API_TASKMASTER_ENVIRONMENT_CONTAINER_BASE_PATH 70 | value: {{ .Values.transfer.tes_base_path }} 71 | - name: TESK_API_TASKMASTER_ENVIRONMENT_TRANSFER_PVC_NAME 72 | value: {{ .Values.transfer.pvc_name }} 73 | {{ end }} 74 | 75 | {{ if .Values.storageClass }} 76 | - name: TESK_API_TASKMASTER_ENVIRONMENT_STORAGE_CLASS_NAME 77 | value: {{ .Values.storageClass }} 78 | {{ end }} 79 | 80 | - name: TESK_API_TASKMASTER_DEBUG 81 | value: "{{ .Values.tesk.debug }}" 82 | 83 | - name: TESK_API_SWAGGER-OAUTH_SCOPES 84 | value: openid:Standard openid,eduperson_entitlement:Access to groups membership,profile:Identity info about user,email:Email 85 | - name: TESK_API_SWAGGER_OAUTH_CLIENT_ID 86 | valueFrom: 87 | secretKeyRef: 88 | name: oauth-client-secret 89 | key: id 90 | optional: true 91 | - name: TESK_API_SWAGGER_OAUTH_CLIENT_SECRET 92 | valueFrom: 93 | secretKeyRef: 94 | name: oauth-client-secret 95 | key: secret 96 | optional: true 97 | {{- if .Values.tesk.securityContext.enabled }} 98 | securityContext: 99 | allowPrivilegeEscalation: false 100 | capabilities: 101 | drop: 102 | - ALL 103 | {{- end }} 104 | volumeMounts: 105 | - name: service-info-config 106 | mountPath: /etc/tesk/service-info 107 | 108 | ports: 109 | - containerPort: {{ .Values.tesk.port }} 110 | volumes: 111 | - name: service-info-config 112 | configMap: 113 | name: service-info-config 114 | {{- if .Values.tesk.securityContext.enabled }} 115 | securityContext: 116 | runAsUser: 1000 117 | runAsNonRoot: true 118 | seccompProfile: 119 | type: RuntimeDefault 120 | {{- end }} 121 | -------------------------------------------------------------------------------- /charts/tesk/templates/common/tesk-svc.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Service 3 | metadata: 4 | name: tesk-api 5 | labels: 6 | app: tesk-api 7 | spec: 8 | ports: 9 | - port: {{ .Values.tesk.port }} 10 | {{ if eq .Values.service.type "NodePort" }}nodePort: {{ .Values.service.node_port }} # valid only if {{ .Values.service.type }} == NodePort{{ end }} 11 | selector: 12 | app: tesk-api 13 | type: {{ if .Values.service.type }}{{ .Values.service.type }}{{ else }}'ClusterIP'{{ end }} 14 | -------------------------------------------------------------------------------- /charts/tesk/templates/ftp/ftp-endpoint.yaml: -------------------------------------------------------------------------------- 1 | {{ if .Values.ftp.hostip }} 2 | kind: Endpoints 3 | apiVersion: v1 4 | metadata: 5 | name: ftp 6 | subsets: 7 | - addresses: 8 | - ip: {{ .Values.ftp.hostip }} # ipv4 address from vboxnet0 9 | ports: 10 | - port: 21 11 | {{ end }} 12 | -------------------------------------------------------------------------------- /charts/tesk/templates/ftp/ftp-secret.yaml: -------------------------------------------------------------------------------- 1 | {{ if .Values.ftp.classic_ftp_secret }} 2 | apiVersion: v1 3 | kind: Secret 4 | metadata: 5 | name: {{ .Values.ftp.classic_ftp_secret }} 6 | type: Opaque 7 | stringData: 8 | username: {{ .Values.ftp.username }} 9 | password: {{ .Values.ftp.password }} 10 | {{ end }} 11 | -------------------------------------------------------------------------------- /charts/tesk/templates/ftp/ftp-service.yaml: -------------------------------------------------------------------------------- 1 | {{ if .Values.ftp.hostip }} 2 | kind: Service 3 | apiVersion: v1 4 | metadata: 5 | name: ftp 6 | spec: 7 | ports: 8 | - protocol: TCP 9 | port: 21 # internal (inside the cluster) 10 | targetPort: 21 # external (at the host) 11 | {{ end }} 12 | -------------------------------------------------------------------------------- /charts/tesk/templates/ftp/netrc-secret.yaml: -------------------------------------------------------------------------------- 1 | {{ if .Values.ftp.netrc_secret }} 2 | apiVersion: v1 3 | kind: Secret 4 | metadata: 5 | name: {{ .Values.ftp.netrc_secret }} 6 | type: Opaque 7 | stringData: 8 | .netrc: |- 9 | {{ .Files.Get "ftp/.netrc" | indent 4 }} 10 | {{ end }} 11 | -------------------------------------------------------------------------------- /charts/tesk/templates/ingress/ingress-rules.yaml: -------------------------------------------------------------------------------- 1 | {{ if .Values.ingress.rules }} 2 | apiVersion: networking.k8s.io/v1 3 | kind: Ingress 4 | metadata: 5 | name: ingress-rules {{ if .Values.ingress.annotations }} 6 | annotations: 7 | {{- range $key, $value := .Values.ingress.annotations }} 8 | {{ $key }}: {{ $value | quote }} 9 | {{- end }} {{ end }} 10 | spec: 11 | ingressClassName: {{ .Values.ingress.ingressClassName }} 12 | {{ if .Values.ingress.tls_secret_name }} 13 | tls: 14 | - hosts: 15 | - {{ .Values.host_name }} 16 | secretName: {{ .Values.ingress.tls_secret_name }} 17 | {{ end }} 18 | rules: 19 | - host: {{ .Values.host_name }} 20 | http: 21 | paths: 22 | - path: {{ .Values.ingress.path }} 23 | pathType: Prefix 24 | backend: 25 | service: 26 | name: tesk-api 27 | port: 28 | number: {{ .Values.tesk.port }} 29 | {{ end }} 30 | -------------------------------------------------------------------------------- /charts/tesk/templates/openshift/oc-route.yaml: -------------------------------------------------------------------------------- 1 | {{ if and (.Capabilities.APIVersions.Has "route.openshift.io/v1") (eq .Values.ingress.rules false) }} 2 | apiVersion: route.openshift.io/v1 3 | kind: Route 4 | metadata: 5 | name: tesk-svc 6 | spec: 7 | host: {{ .Values.host_name }} 8 | tls: 9 | insecureEdgeTerminationPolicy: Redirect 10 | termination: edge 11 | to: 12 | kind: Service 13 | name: tesk-api 14 | weight: 100 15 | wildcardPolicy: None 16 | status: 17 | ingress: 18 | - conditions: 19 | host: {{ .Values.host_name }} 20 | routerName: router 21 | wildcardPolicy: None 22 | {{ end }} 23 | -------------------------------------------------------------------------------- /charts/tesk/templates/storage/aws-secret.yaml: -------------------------------------------------------------------------------- 1 | {{ if eq .Values.storage "s3" }} 2 | apiVersion: v1 3 | kind: Secret 4 | metadata: 5 | name: aws-secret 6 | data: 7 | config: {{ .Files.Get "s3-config/config" | b64enc }} 8 | credentials: {{ .Files.Get "s3-config/credentials" | b64enc }} 9 | {{ end }} 10 | -------------------------------------------------------------------------------- /charts/tesk/templates/storage/openstack.yaml: -------------------------------------------------------------------------------- 1 | {{ if eq .Values.storage "openstack" }} 2 | apiVersion: storage.k8s.io/v1 3 | kind: StorageClass 4 | metadata: 5 | annotations: 6 | storageclass.kubernetes.io/is-default-class: "true" 7 | name: gold 8 | parameters: 9 | availability: nova 10 | provisioner: kubernetes.io/cinder 11 | {{ end }} 12 | 13 | -------------------------------------------------------------------------------- /charts/tesk/tls_secret_name.yml-TEMPLATE: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | stringData: 3 | tls.crt: |- 4 | -----BEGIN CERTIFICATE----- 5 | 4ocG9QTVDA2VZyTxDtVMXbTNlleVazGhXYNCprvznaFiOKlE4ZPytnHl39zhBR5hO 6 | 4ocG9QTVDA2VZyTxDtVMXbTNlleVazGhXYNCprvznaFiOKlE4ZPytnHl39zhBR5hO 7 | 4ocG9QTVDA2VZyTxDtVMXbTNlleVazGhXYNCprvznaFiOKlE4ZPytnHl39zhBR5hO 8 | 4ocG9QTVDA2VZyTxDtVMXbTNlleVazGhXYNCprvznaFiOKlE4ZPytnHl39zhBR5hO 9 | 4ocG9QTVDA2VZyTxDtVMXbTNlleVazGhXYNCprvznaFiOKlE4ZPytnHl39zhBR5hO 10 | 4ocG9QTVDA2VZyTxDtVMXbTNlleVazGhXYNCprvznaFiOKlE4ZPytnHl39zhBR5hO 11 | 4ocG9QTVDA2VZyTxDtVMXbTNlleVazGhXYNCprvznaFiOKlE4ZPytnHl39zhBR5hO 12 | -----END CERTIFICATE----- 13 | tls.key: |- 14 | -----BEGIN PRIVATE KEY----- 15 | 4ocG9QTVDA2VZyTxDtVMXbTNlleVazGhXYNCprvznaFiOKlE4ZPytnHl39zhBR5hO 16 | 4ocG9QTVDA2VZyTxDtVMXbTNlleVazGhXYNCprvznaFiOKlE4ZPytnHl39zhBR5hO 17 | 4ocG9QTVDA2VZyTxDtVMXbTNlleVazGhXYNCprvznaFiOKlE4ZPytnHl39zhBR5hO 18 | 4ocG9QTVDA2VZyTxDtVMXbTNlleVazGhXYNCprvznaFiOKlE4ZPytnHl39zhBR5hO 19 | 4ocG9QTVDA2VZyTxDtVMXbTNlleVazGhXYNCprvznaFiOKlE4ZPytnHl39zhBR5hO 20 | -----END PRIVATE KEY----- 21 | kind: Secret 22 | metadata: 23 | name: tls_secret 24 | type: kubernetes.io/tls 25 | -------------------------------------------------------------------------------- /charts/tesk/values.yaml: -------------------------------------------------------------------------------- 1 | # Default values for tesk. 2 | # This is a YAML-formatted file. 3 | # Declare variables to be passed into your templates. 4 | 5 | # host_name: tes.tsi.ebi.ac.uk 6 | host_name: "" 7 | # 8 | # 9 | # 10 | 11 | # 'openstack' or 's3' 12 | storage: none 13 | 14 | # Configurable storage class. 15 | storageClass: 16 | 17 | tesk: 18 | # if empty - default base path; currently /ga4gh/tes/v1 19 | #tes_api_base_path: 20 | # For backwards compatibility with TES 0.3 - old base bath 21 | tes_api_base_path: /ga4gh/tes/v1 22 | image: docker.io/elixircloud/tesk-api:1.1.0 23 | port: 8080 24 | taskmaster_image_name: docker.io/elixircloud/tesk-core-taskmaster 25 | taskmaster_image_version: v0.10.2 26 | taskmaster_filer_image_name: docker.io/elixircloud/tesk-core-filer 27 | taskmaster_filer_image_version: v0.10.2 28 | debug: false 29 | executor_retries: 2 30 | filer_retries: 2 31 | 32 | limitsCpu: 1 33 | limitsMemory: 2048Mi 34 | requestsCpu: 1 35 | requestsMemory: 2048Mi 36 | 37 | securityContext: 38 | enabled: true 39 | 40 | transfer: 41 | # If you want local file systems support (i.e. 'file:' urls in inputs and outputs), 42 | # you have to define these 2 properties. 43 | active: false 44 | 45 | # wes_base_path: '/data' # WesElixir via docker-compose 46 | wes_base_path: '/tmp' # WesElixir locally 47 | # Change the value of $wesBasePath in minikubeStart accordingly 48 | tes_base_path: '/transfer' 49 | pvc_name: 'transfer-pvc' 50 | 51 | auth: 52 | # the following variables are specific to each deployment 53 | # mode: auth/noauth 54 | mode: noauth 55 | # EBI/CSC 56 | env_subgroup: CSC 57 | env_basegroup: ECP_CLN 58 | 59 | service: 60 | # the following variables are specific to each deployment 61 | # use: 62 | # - NodePort, if you want to expose API directly 63 | # - LoadBalancer for cloud provider (gcloud) or empty otherwise 64 | # type: NodePort 65 | # node_port: 31567 66 | type: "" 67 | node_port: "" 68 | 69 | ftp: 70 | # If you need FTP configuration, choose one of the 2 methods of providing credentials 71 | classic_ftp_secret: ftp-secret 72 | netrc_secret: 73 | #classic_ftp_secret: 74 | #netrc_secret: netrc-secret 75 | # If you install FTP locally, but outside of k8s and need a DNS entry for it (because your workflow manager might not like the IP address) 76 | # one way of getting a DNS entry for your FTP service is to use a k8s "service without a selector" 77 | # Put the IP under which your pods see see services running on your host (differs depending on the way you installes K8s) 78 | # For virtualBox, it is 192.168.99.1 and your ftp service will be visible under ftp name 79 | # You will be able to use it like this: ftp://ftp/file 80 | hostip: 81 | 82 | ingress: 83 | rules: true 84 | ingressClassName: "" 85 | path: / 86 | # If no TLS secret name configured, TLS will be switched off 87 | tls_secret_name: 88 | # Annotations for Ingress Resource. 89 | annotations: 90 | kubernetes.io/tls-acme: "true" 91 | # Choose one of the following depending on your setup 92 | # cert-manager.io/issuer: letsencrypt-production 93 | cert-manager.io/cluster-issuer: letsencrypt-production 94 | -------------------------------------------------------------------------------- /documentation/deployment.md: -------------------------------------------------------------------------------- 1 | # Deployment instructions for TESK 2 | ## Requirements 3 | * **A Kubernetes cluster version 1.9 and later**. TESK works well in multi-tenancy clusters such as OpenShift and requires access to only one namespace. 4 | * **A default storage class.** To handle tasks with I/O TESK always requires a default storage class (regardless of the chosen storage backend). TESK uses the class to create temporary PVCs. It should be enough that the storage supports the RWO mode. 5 | * **A storage backend** to exchange I/O with the external world. At the moment TESK supports: 6 | * FTP. R/W access to a single FTP account. 7 | * Shared file system. This usually comes in the form of a RWX PVC. 8 | * S3 (WiP). R/W access to one bucket in S3-like storage 9 | 10 | ## Installing TESK 11 | ### Helm 12 | TESK can be installed using Helm 3 (tested with v3.0.0) using [this chart](../charts/tesk). It is best to create a dedicated namespace for TESK, although for test or development clusters it is fine to use the `default` namespace. 13 | The documentation of the chart gives a desciption of all configuration options and below the most common installation scenarios have been described. 14 | TESK installation consists of a single API installed as a K8s deployment and exposed as a K8s service. Additionally, TESK API requires access to the K8s API in order to create K8s Jobs and PVCs. That is why the installation additionally creates objects such as service accounts, roles and role bindings. 15 | The chart does not provide a way to install the default storage class and that needs to be done independently by the cluster administrator. 16 | 17 | ### Exposing TESK API 18 | After executing the chart with the default values, TESK API will be installed, but will be accessible only inside the cluster. There is a number of options of exposing TESK externally and they depend on the type of the cluster. 19 | 20 | #### NodePort and LoadBalancer 21 | The most basic way of exposing TESK on self-managed clusters and a good option for development clusters such as Minikube is to use a NodePort type of service. 22 | In the chart set the values: 23 | ``` 24 | service.type="NodePort" 25 | ## Any values in the range 30000-32767 is fine. 31567 is used as an example 26 | service.node_port: 31567 27 | ``` 28 | After installing the chart TESK API should be accessible under the external IP of any of your cluster nodes and the node port. For minikube you can obtain the IP by running: 29 | ``` 30 | minikube ip 31 | ``` 32 | or open Swagger UI of TESK directly in the browser with: 33 | ``` 34 | minikube service tesk-api 35 | ``` 36 | You should be able to see an empty list of tasks by calling 37 | ``` 38 | http://external_IP_of_a_node:31567/v1/tasks 39 | 40 | { 41 | "tasks" : [ ] 42 | } 43 | 44 | ``` 45 | If your cluster is provided by a Cloud Provider, you may be able to use a LoadBalancer type of service. In the chart set the value: 46 | ``` 47 | service.type="LoadBalancer" 48 | ``` 49 | and consult [K8s documentation](https://kubernetes.io/docs/tasks/access-application-cluster/create-external-load-balancer/) on how to find out the IP of your TESK service. 50 | 51 | #### OpenShift 52 | The chart handles exposing TESK API as OpenShift Route. 53 | 54 | TESK API should be accessible under (Swagger UI): 55 | `https://project-name.openshift-host-name` 56 | and 57 | `https://project-name.openshift-host-name/v1/tasks` 58 | should return an empty list of tasks. 59 | 60 | #### Ingress 61 | Recommended way of exposing any public facing API from K8s cluster such as TESK API is to use Ingress. 62 | You need: 63 | * an **Ingress Controller**. 64 | * a **Hostname** - a DNS entry, where you will expose TESK. TESK can be installed at a subpath as well. 65 | * an **Ingress Resource**, which will instruct the Controller where to expose TESK. 66 | * A **TLS certificate** to serve TESK over https. You can obtain one from a certificate authority or automatically obtain one from [Let's Encrypt](https://letsencrypt.org/). The K8s way to do it is by installing [cert-manager](https://cert-manager.io/) and creating an ACME Issuer. 67 | The example values for TESK Helm chart to create Ingress Resource with annotations for cert-manager, but not to install the controller: 68 | ```yaml 69 | host_name: tes.ebi.ac.uk 70 | ingress: 71 | rules: true 72 | ingressClassName: "" 73 | path: / 74 | # If no TLS secret name configured, TLS will be switched off 75 | tls_secret_name: 76 | # Annotations for Ingress Resource. 77 | annotations: 78 | kubernetes.io/tls-acme: "true" 79 | # Choose one of the following depending on your setup 80 | # cert-manager.io/issuer: letsencrypt-production 81 | cert-manager.io/cluster-issuer: letsencrypt-production 82 | ``` 83 | List of tasks should be reachable under this URL: 84 | ``` 85 | https://tes.ebi.ac.uk/v1/tasks 86 | ``` 87 | 88 | ### Storage backends 89 | 90 | #### Shared file system 91 | TESK can exchange Inputs and Outputs with the external world using the local/shared storage. You need to create a PVC that will be reachable for your workflow manager and for TESK at the same time. 92 | If the workflow manager (or anything else that produces paths to your inputs and outputs) is installed inside the same K8s cluster, you may use a PVC of a storage class providing RWX access and mount it to the pod where the workflow manager is installed in the directory where the manager will be creating/orchestrating inputs/outputs. Depending on the workflow manager, it may be a working directory of your workflow manager process. 93 | If the workflow manager is installed outside of the cluster, you may be able to use a volume mounting storage visible outside of the cluster (hostPath, NFS, etc) and a PVC bound to that volume. We used Minikube with the hostPath type of storage in this secenario successfuly. 94 | Creating the shared PVC is not handled by TESK Helm chart. 95 | Finally you have to setup following values in the chart: 96 | ```yaml 97 | transfer: 98 | # If you want local file systems support (i.e. 'file:' urls in inputs and outputs), 99 | # you have to define these 2 properties. 100 | active: false 101 | 102 | # wes_base_path: '/data' # WesElixir via docker-compose 103 | wes_base_path: '/tmp' # WesElixir locally 104 | # Change the value of $wesBasePath in minikubeStart accordingly 105 | tes_base_path: '/transfer' 106 | pvc_name: 'transfer-pvc' 107 | ``` 108 | 109 | #### FTP 110 | TESK can exchange Inputs and Outputs with the external world using an FTP account. Currently TLS is not supported in TESK, but if you plan to use TESK with cwl-tes and FTP, cwl-tes requires TLS for FTP. The solution is to enable TLS on your FTP server, but not enforce it. 111 | In the Helm chart provide your credentials in one of 2 ways. The old way has been in TESK for a long time, but will be finally superseded by `.netrc`. Provide a name for a secret, which will store credentials (you can keep the default). An empty name switches off the creation of the old-style credentials secret. 112 | ```yaml 113 | ftp: 114 | classic_ftp_secret: ftp-secret 115 | ``` 116 | and additionally provide your username and password in the `secrets.yaml`, as describe [here](../charts/tesk/README.md). 117 | 118 | Alternatively, you can use a `.netrc` file, which will allow storing credentials for more than one FTP server. 119 | Provide a name for a secret, which will store .netrc file: 120 | ```yaml 121 | ftp: 122 | netrc_secret: ftp-secret 123 | ``` 124 | and additionally place a `.netrc` file in the folder `ftp` in the chart (the template of the file is already there). 125 | Keeping both secret names empty switches off FTP support altogether. 126 | 127 | #### S3 128 | TESK can also utilize S3 object storage for exchanging Inputs & Outputs. If you have an S3 endpoint (AWS, minio, etc) that you want to use, simply add the necessary `config` and `credentials` files (see [here](https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-files.html)) under a folder named **s3-config**. You can use the templates provided in *charts/tesk/s3-config* as a point of reference. 129 | 130 | ### Authentication and Authorisation 131 | TESK supports OAuth2/OIDC to authorise API requests. Authentication and authorisation are optional and can be turned off completely. When turned on, TESK API expects an OIDC access token in Authorization Bearer header. TESK can be integrated with any standard OIDC provider, but the solution has been designed to support Elixir AAI in the first place and the authorisation part relies on Elixir's group model. For details, please see [Authentication and Authorisation document](https://github.com/EMBL-EBI-TSI/tesk-api/blob/master/auth.md). 132 | To enable authentication set the following value in the chart: 133 | ``` 134 | auth: 135 | mode: auth 136 | ``` 137 | At the moment enabling authentication also enables authorisation. Consult [this document](https://github.com/EMBL-EBI-TSI/tesk-api/blob/master/auth.md) for details of authorisation. 138 | The support for authorisation configuration in the chart and its documentation is in progress. 139 | ### Additional configuration 140 | The Helm chart has been a fairly recent addition to TESK and TESK owes its to its fantastic contributors. There might be more options that are available for configuration in TESK that have not been reflected in the chart, yet. Have a look [here](https://github.com/EMBL-EBI-TSI/tesk-api) for more configuration options. 141 | -------------------------------------------------------------------------------- /documentation/img/TESKlogo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/elixir-cloud-aai/TESK/90e2e9bc84c7b47dd1fa548b223e72c885bd4ba6/documentation/img/TESKlogo.png -------------------------------------------------------------------------------- /documentation/img/TESKlogowfont.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/elixir-cloud-aai/TESK/90e2e9bc84c7b47dd1fa548b223e72c885bd4ba6/documentation/img/TESKlogowfont.png -------------------------------------------------------------------------------- /documentation/img/architecture.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/elixir-cloud-aai/TESK/90e2e9bc84c7b47dd1fa548b223e72c885bd4ba6/documentation/img/architecture.png -------------------------------------------------------------------------------- /documentation/img/project-architecture.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/elixir-cloud-aai/TESK/90e2e9bc84c7b47dd1fa548b223e72c885bd4ba6/documentation/img/project-architecture.png -------------------------------------------------------------------------------- /documentation/integrated_wes_tes.md: -------------------------------------------------------------------------------- 1 | # Deployment of TESK integrated with WES-ELIXIR 2 | 3 | In this scenario TESK and a workflow engine are deployed together in one K8s cluster and exchange input/output/intermediate files via shared file system and not FTP. In this setting TESK is an internal service and is not exposed at a boundary of the system. 4 | 5 | ## Deployment descriptors 6 | 7 | Example deployment descriptors for the use case reside in [wes-tes directory](/deployment/wes-tes). 8 | 9 | ### RWX PVC 10 | 11 | WES and TESK exchange files over ReadWriteMany PVC. [An example descriptor](/deployment/wes-tes/userstorage/rwx-pvc.yaml) assumes the existence of a default storage class supporting Read Write Many PVCs. 12 | 13 | ### WES 14 | 15 | The workflow engine that was used is [this fork](https://github.com/EMBL-EBI-TSI/WES-ELIXIR/tree/dev) of [WES-ELIXIR](https://github.com/elixir-europe/WES-ELIXIR) adapted for the use case. It consists of 2 services: [API](/deployment/wes-tes/wes/wes-elixir-deployment.yaml) - exposed externally via Ingress definition and [a celery worker](/deployment/wes-tes/wes/wes-celery-deployment.yaml). Both use the same image. WES-ELIXIR has a requirement of MongoDB and RabbitMQ. In the example we assumed both are available, under `mongo` and `rabbitmq` URLs accordingly. We deployed both inside the same cluster, but that is not a requirement. 16 | 17 | ### TES 18 | 19 | TESK is an internal service in this scenario. Newly introduced environment variables describe, how the shared PVC is visible to different components of the system: 20 | * `TESK_API_TASKMASTER_ENVIRONMENT_HOST_BASE_PATH` - the path, where the shared PVC is mounted at WES side 21 | * `TESK_API_TASKMASTER_ENVIRONMENT_CONTAINER_BASE_PATH` - the path, where the shared PVC will be dynamically mounted in filer containers. 22 | * `TESK_API_TASKMASTER_ENVIRONMENT_TRANSFER_PVC_NAME` - the name of the shared PVC. 23 | 24 | In this use case - as usually - TESK requires a dynamic storage class (not necessarily the same that was used for producing shared storage, but we assumed so). 25 | 26 | ### User and Site storage 27 | 28 | In our PoC [Caddy](https://caddyserver.com) server plays a role of both user and site storage. We used [this image](https://github.com/abiosoft/caddy-docker) with an additional [upload plugin](https://github.com/wmark/http.upload). When used as a [User Workspace](/deployment/wes-tes/userstorage/userworkspace.yaml) Caddy is exposed externally via Ingress, serves files from the shared storage and lets users upload files therein. When used as [Site Storage](/deployment/wes-tes/sitestorage/sitestorage.yaml), it has a separate file storage attached and the service is visible only internally. 29 | 30 | -------------------------------------------------------------------------------- /documentation/local_ftp.md: -------------------------------------------------------------------------------- 1 | How to use local FTP 2 | -------------------- 3 | 4 | * Create a new user on the host 5 | 6 | ``` 7 | sudo adduser tesk 8 | ``` 9 | 10 | * Install [vsftpd](https://help.ubuntu.com/lts/serverguide/ftp-server.html.en) 11 | 12 | ``` 13 | sudo apt install vsftpd 14 | ``` 15 | 16 | * Make it writable. In `/etc/vsftpd.conf` set: 17 | 18 | ``` 19 | write_enable=YES 20 | ``` 21 | 22 | * To configure TLS with vsftpd (Optional), 23 | if you are planning to use cwl-tes, then you have to add TLS certificate to vsftpd. 24 | ``` 25 | sudo mkdir /etc/ssl/private 26 | sudo openssl req -x509 -nodes -days 365 -newkey rsa:2048 -keyout /etc/ssl/private/vsftpd.pem -out /etc/ssl/private/vsftpd.pem 27 | ``` 28 | Open the vsftpd configuration file as root: 29 | ``` 30 | sudo nano /etc/vsftpd.conf 31 | ``` 32 | Now, specify the location of our certificate, key files and other configurations to the end of the file. 33 | ``` 34 | rsa_cert_file=/etc/ssl/private/vsftpd.pem 35 | rsa_private_key_file=/etc/ssl/private/vsftpd.pem 36 | ssl_enable=YES 37 | allow_anon_ssl=NO 38 | force_local_data_ssl=NO 39 | force_local_logins_ssl=NO 40 | ssl_tlsv1=YES 41 | ssl_sslv2=NO 42 | ssl_sslv3=NO 43 | require_ssl_reuse=NO 44 | ssl_ciphers=HIGH 45 | ``` 46 | Save and close the file. 47 | Restart vsftpd to enable our changes 48 | ``` 49 | sudo /etc/init.d/vsftpd restart 50 | ``` 51 | If FTP server fails to start, investigate by trying to start it manually: 52 | ``` 53 | sudo vsftpd 54 | ``` 55 | * You may need to configure or disable firewall (such as `ufw`) 56 | * Edit the input and output URLs in the task's JSON (e.g.: `examples/localftp/taskWithIO.json`): 57 | 58 | ``` 59 | "inputs": [ 60 | { 61 | "description": "Downloading a single file from FTP", 62 | "name": "File from FTP", 63 | "path": "/tes/input", 64 | "type": "FILE", 65 | "url": "ftp://ftp/home/tesk/input.txt" <= here 66 | You have to use absolute paths. 67 | Maybe because I'm not using `chroot_local_user` in FTP? 68 | } 69 | 70 | "outputs": [ 71 | { 72 | "description": "Example of uploading a directory to FTP", 73 | "name": "Dir to FTP", 74 | "path": "/tes", 75 | "type": "DIRECTORY", 76 | "url": "ftp://ftp/home/tesk/output" <= and here 77 | } 78 | ``` 79 | 80 | * Configure credentials and hostIP for FTP using the Helm chart according to instructions. 81 | ``` 82 | ftp.hostip: 83 | ``` 84 | `hostIP` needs to be set to the IP, where containers running in your K8s see your services running on localhost. 85 | For `docker` driver, it seems to be `172.30.0.1` (on Linux). For minikube with a VM driver (such as virtualbox), you can check the IP by running: 86 | ``` 87 | minikube ssh "ping host.minikube.internal" 88 | ``` 89 | 90 | 91 | * Create the task: 92 | 93 | ``` 94 | cd examples/ 95 | $ ./taskCreate localftp/taskWithIO.json 96 | ``` 97 | * If you want to run `cwl-tes` locally with the locally installed FTP server, you need a couple more tricks: 98 | 99 | - You need a `hosts` entry with 100 | ``` 101 | 127.0.0.1 ftp 102 | ``` 103 | so that both cwl-tes and TESK see the local FTP at the same address. 104 | - You need a `.netrc` file. 105 | ``` 106 | machine ftp 107 | login tesk 108 | password 109 | ``` 110 | It needs to be readable only to the owner 111 | ``` 112 | chmod 600 ~/.netrc 113 | ``` 114 | - Finally, you can run a workflow: 115 | A good workflow to test your setup is: https://github.com/uniqueg/cwl-example-workflows.git. 116 | which you need to clone 117 | 118 | Then run the following command: 119 | ``` 120 | cwl-tes --tes http://minikube_ip:node_port --remote-storage-url ftp://ftp/home/tesk hashsplitter-workflow.cwl hashsplitter-test.yml 121 | ``` 122 | 123 | - If the workflow uses FTP inputs, adjust them, so they point to existing files on your FTP server. 124 | 125 | For example in the `hashsplitter-test.yml` from the workflow above make the following changes and also 126 | make sure the input.txt file has been uploaded to your FTP. 127 | ``` 128 | input: 129 | class: File 130 | path: ftp://ftp/home/tesk/input.txt 131 | ``` 132 | - Running the above workflow and passing a local file with `--input` parameter (so letting `cwl-tes` upload an input file to FTP) fails for the proposed FTP setup. Most probably this functionality works only with chroot enabled on FTP server. 133 | -------------------------------------------------------------------------------- /documentation/tesintro.md: -------------------------------------------------------------------------------- 1 | # A very brief introduction to TES 2 | 3 | ## Task Execution Service standard 4 | The Global Alliance for Genomics and Health (GA4GH) is an international consortium of academic and industry partners that try to establish standards to promote and facilitate collaboration and data exchange in the life sciences. As part of the 'Cloud Workstream' of this effort 4 standards have been proposed to facilitate running scientific workflows in a cloud environment: the Data Object Service (DOS), Tool Registration Service (TRS), Workflow Execution Service (WES) and the Task Execution Service (TES). These for standards are meant to be independent but complementary to each other in running workflows and handling the associated data needs. TES is a standard that represents the smallest unit of computational work in a workflow that can be independently run in a cloud. TESK is an implementation of this standard by EMBL-EBI running on Google's Kubernetes container orchestration platform. 5 | 6 | ## Technical overview 7 | A minimal TES task is represented as follows: 8 | 9 | ```json 10 | { 11 | "inputs": [ 12 | { 13 | "url": "http://adresss/to/input_file", 14 | "path": "/container/input" 15 | } 16 | ], 17 | "outputs" : [ 18 | { 19 | "url" : "file://path/to/output_file", 20 | "path" : "/container/output" 21 | } 22 | ], 23 | "executors" : [ 24 | { 25 | "image" : "ubuntu", 26 | "command" : ["md5sum", "/container/input"], 27 | "stdout" : "/container/output" 28 | } 29 | ] 30 | } 31 | ``` 32 | 33 | Inputs and outputs are expected to have an URI that can be resolved by the relevant implementation. The executor 'image' entry is ay image that can be reached by the relevant docker instance of the implementation, and would usually refer to a public image on DockerHub or Quay (see also [Dockstore](https://dockstore.org/)). TES tasks are submitted through a RESTful API using JSON. See the [full spec](https://github.com/ga4gh/task-execution-schemas) for a complete list of possible fields and their description. Also take a look at the [examples](../examples/success) directory for a number of basic and more advanced tes tasks. 34 | -------------------------------------------------------------------------------- /examples/cancel/counter.json: -------------------------------------------------------------------------------- 1 | { 2 | "description": "An example task to test job cancellation. Cancel it please, because it runs forever", 3 | "volumes": [ 4 | "/test" 5 | ], 6 | "executors": [ 7 | { 8 | "image": "alpine", 9 | "command": [ 10 | "echo", 11 | "Nothing" 12 | ] 13 | }, 14 | { 15 | "image": "alpine", 16 | "command": [ 17 | "sh", 18 | "-c", 19 | "i=0; while true; do echo \"$i: $(date)\"; i=$((i+1)); sleep 1; done" 20 | ] 21 | } 22 | ] 23 | } 24 | -------------------------------------------------------------------------------- /examples/error/error_stops_execution.json: -------------------------------------------------------------------------------- 1 | { 2 | "description": "Should finish in EXECUTOR_ERROR. Don't run on Kubernetes older than 1.8 (will recreate the executor pod forever). Third executor should not run.", 3 | "executors": [ 4 | { 5 | "image": "alpine", 6 | "command": [ 7 | "echo", 8 | "You will see this in the logs (stdout)." 9 | ] 10 | }, 11 | { 12 | "image": "alpine", 13 | "command": [ 14 | "sh", 15 | "-c", 16 | "exit 1" 17 | ] 18 | }, 19 | { 20 | "image": "alpine", 21 | "command": [ 22 | "echo", 23 | "This shouldn't appear in the logs (stdout)." 24 | ] 25 | } 26 | ] 27 | } 28 | -------------------------------------------------------------------------------- /examples/error/executor.json: -------------------------------------------------------------------------------- 1 | { 2 | "description": "Should finish in EXECUTOR_ERROR. Don't run on Kubernetes older than 1.8 (will recreate the executor pod forever)", 3 | "executors": [ 4 | { 5 | "image": "alpine", 6 | "command": [ 7 | "sh", 8 | "-c", 9 | "exit 1" 10 | ] 11 | } 12 | ] 13 | } 14 | -------------------------------------------------------------------------------- /examples/error/input.json: -------------------------------------------------------------------------------- 1 | { 2 | "description": "Should finish in SYSTEM_ERROR. No executors should run.", 3 | "inputs": [ 4 | { 5 | "url": "http://nonexistent.ebi.ac.uk", 6 | "path": "/somewhere/file", 7 | "type": "FILE" 8 | } 9 | ], 10 | "executors": [ 11 | { 12 | "image": "alpine", 13 | "command": [ 14 | "echo", 15 | "This shouldn't appear in the logs." 16 | ] 17 | }, 18 | { 19 | "image": "alpine", 20 | "command": [ 21 | "cat", 22 | "/somewhere/file" 23 | ] 24 | } 25 | ] 26 | } 27 | -------------------------------------------------------------------------------- /examples/error/output_new_dir_file.json: -------------------------------------------------------------------------------- 1 | { 2 | "description": "FTP output - directory can be created only in DIRECTORY use case, so will not work (will try to create new and will fail)", 3 | "outputs": [ 4 | { 5 | "description": "ftp://ftp-private.ebi.ac.uk/upload/new does not exist", 6 | "path": "/tes/file1.txt", 7 | "type": "FILE", 8 | "url": "ftp://ftp-private.ebi.ac.uk/upload/examples/new/file1.txt" 9 | } 10 | ], 11 | "executors": [ 12 | { 13 | "image": "alpine", 14 | "command": [ 15 | "echo", 16 | "This goes to file1" 17 | ], 18 | "stdout": "/tes/file1.txt" 19 | } 20 | ] 21 | } 22 | -------------------------------------------------------------------------------- /examples/error/output_new_folder.json: -------------------------------------------------------------------------------- 1 | { 2 | "description": "FTP output - only one directory can be created, so will not work (will try to create new/tes - because of scp-like behaviour if creating a source dir under destination folder)", 3 | "outputs": [ 4 | { 5 | "description": "ftp://ftp-private.ebi.ac.uk/upload/examples/new does not exist", 6 | "path": "/tes", 7 | "type": "DIRECTORY", 8 | "url": "ftp://ftp-private.ebi.ac.uk/upload/examples/new" 9 | } 10 | ], 11 | "executors": [ 12 | { 13 | "image": "alpine", 14 | "command": [ 15 | "echo", 16 | "This goes to file1" 17 | ], 18 | "stdout": "/tes/file1.txt" 19 | }, 20 | { 21 | "image": "alpine", 22 | "command": [ 23 | "sh", 24 | "-c", 25 | "echo 'This goes to file2' > /tes/file2.txt" 26 | ] 27 | } 28 | ] 29 | } 30 | -------------------------------------------------------------------------------- /examples/localftp/taskWithIO.json: -------------------------------------------------------------------------------- 1 | { 2 | "description": "An example presenting: Inputs, Outputs, Resources, Stdout, Workdir", 3 | "executors": [ 4 | { 5 | "command": [ 6 | "sh", 7 | "-c", 8 | "cat ./input ; echo Hello $SECRET_PROJECT_NAME" 9 | ], 10 | "env": { 11 | "SECRET_PROJECT_NAME": "TESK" 12 | }, 13 | "image": "alpine", 14 | "stderr": "/tes/err", 15 | "stdout": "/tes/output", 16 | "workdir": "/tes" 17 | } 18 | ], 19 | "inputs": [ 20 | { 21 | "description": "Downloading a single file from FTP", 22 | "name": "File from FTP", 23 | "path": "/tes/input", 24 | "type": "FILE", 25 | "url": "ftp://ftp/home/tesk/input.txt" 26 | } 27 | ], 28 | "name": "hello tesk", 29 | "outputs": [ 30 | { 31 | "description": "Example of uploading a directory to FTP", 32 | "name": "Dir to FTP", 33 | "path": "/tes", 34 | "type": "DIRECTORY", 35 | "url": "ftp://ftp/home/tesk/output" 36 | } 37 | ], 38 | "resources": { 39 | "cpu_cores": 1, 40 | "disk_gb": 0.1, 41 | "ram_gb": 1 42 | }, 43 | "tags": { 44 | "Version": "5.15" 45 | }, 46 | "volumes": [ 47 | "/tes/temp" 48 | ] 49 | } -------------------------------------------------------------------------------- /examples/resources/cpu.json: -------------------------------------------------------------------------------- 1 | { 2 | "description": "Run a couple of those and see, how they are scheduled. (In our case, only 3 will run, because of our 3 worker nodes with 3 or more cpus)", 3 | "executors": [ 4 | { 5 | "image": "alpine", 6 | "command": [ 7 | "sh", 8 | "-c", 9 | "i=0; while true; do echo \"$i: $(date)\"; i=$((i+1)); sleep 1; done" 10 | ] 11 | } 12 | ], 13 | "resources": { 14 | "cpu_cores": 3 15 | } 16 | } 17 | -------------------------------------------------------------------------------- /examples/resources/more_cpu_than_nodes.json: -------------------------------------------------------------------------------- 1 | { 2 | "description": "Request for more cpu than any nodes - will be Pending forever. TODO - what shall we do in such case?", 3 | "executors": [ 4 | { 5 | "image": "alpine", 6 | "command": [ 7 | "echo", 8 | "Nothing" 9 | ] 10 | } 11 | ], 12 | "resources": { 13 | "cpu_cores": 5 14 | } 15 | } 16 | -------------------------------------------------------------------------------- /examples/success/env.json: -------------------------------------------------------------------------------- 1 | { 2 | "description": "Testing environment variables. We don't run in shell by default.", 3 | "executors": [ 4 | { 5 | "image": "alpine", 6 | "command": [ 7 | "echo", 8 | "$SECRET_PROJECT_NAME", 9 | "$PROJECT_STATUS" 10 | ], 11 | "env": { 12 | "SECRET_PROJECT_NAME": "TESK", 13 | "PROJECT_STATUS": "rocks!" 14 | } 15 | }, 16 | { 17 | "image": "alpine", 18 | "command": [ 19 | "sh", 20 | "-c", 21 | "echo $SECRET_PROJECT_NAME $PROJECT_STATUS" 22 | ], 23 | "env": { 24 | "SECRET_PROJECT_NAME": "TESK", 25 | "PROJECT_STATUS": "rocks!" 26 | } 27 | } 28 | ] 29 | } 30 | -------------------------------------------------------------------------------- /examples/success/hello.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "Hello World", 3 | "description": "Hello World, inspired by Funnel's most basic example", 4 | "executors": [ 5 | { 6 | "image": "alpine", 7 | "command": [ 8 | "echo", 9 | "TESK says: Hello World" 10 | ] 11 | } 12 | ] 13 | } 14 | -------------------------------------------------------------------------------- /examples/success/input_content.json: -------------------------------------------------------------------------------- 1 | { 2 | "description": "Demonstrates inlined input", 3 | "inputs": [ 4 | { 5 | "content": "ABC TESK and some more text.", 6 | "path": "/tes/volumes/input", 7 | "type": "FILE" 8 | } 9 | ], 10 | "executors": [ 11 | { 12 | "image": "alpine", 13 | "command": [ 14 | "cat", 15 | "/tes/volumes/input" 16 | ] 17 | } 18 | ] 19 | } 20 | -------------------------------------------------------------------------------- /examples/success/input_dir_duplicate_file_names.json: -------------------------------------------------------------------------------- 1 | { 2 | "description": "2 different folders with filename collision, mounted to the same place. The latest one wins (das in our case)", 3 | "inputs": [ 4 | { 5 | "url": "ftp://ftp.ebi.ac.uk/pub/databases/16S_RNA/README", 6 | "path": "/tes/files/rna_README", 7 | "type": "FILE" 8 | }, 9 | { 10 | "url": "ftp://ftp.ebi.ac.uk/pub/software/das/README", 11 | "path": "/tes/files/das_README", 12 | "type": "FILE" 13 | }, 14 | { 15 | "url": "ftp://ftp.ebi.ac.uk/pub/databases/16S_RNA", 16 | "path": "/tes", 17 | "type": "DIRECTORY" 18 | }, 19 | { 20 | "url": "ftp://ftp.ebi.ac.uk/pub/software/das", 21 | "path": "/tes", 22 | "type": "DIRECTORY" 23 | } 24 | ], 25 | "executors": [ 26 | { 27 | "image": "ubuntu", 28 | "command": [ 29 | "sh", 30 | "-c", 31 | "cd /tes; find * -name \"*README\" -print0 | du --files0-from=- -b" 32 | ] 33 | } 34 | ] 35 | } 36 | -------------------------------------------------------------------------------- /examples/success/input_dir_file_duplicate_file_names.json: -------------------------------------------------------------------------------- 1 | { 2 | "description": "Placing a file at a location (/tes/README), that is colliding with a file from directory input. The latest one wins (das in our case)", 3 | "inputs": [ 4 | { 5 | "url": "ftp://ftp.ebi.ac.uk/pub/databases/16S_RNA/README", 6 | "path": "/tes/files/rna_README", 7 | "type": "FILE" 8 | }, 9 | { 10 | "url": "ftp://ftp.ebi.ac.uk/pub/software/das/README", 11 | "path": "/tes/files/das_README", 12 | "type": "FILE" 13 | }, 14 | { 15 | "url": "ftp://ftp.ebi.ac.uk/pub/databases/16S_RNA/README", 16 | "path": "/tes/README", 17 | "type": "FILE" 18 | }, 19 | { 20 | "url": "ftp://ftp.ebi.ac.uk/pub/software/das", 21 | "path": "/tes", 22 | "type": "DIRECTORY" 23 | } 24 | ], 25 | "executors": [ 26 | { 27 | "image": "ubuntu", 28 | "command": [ 29 | "sh", 30 | "-c", 31 | "cd /tes; find * -name \"*README\" -print0 | du --files0-from=- -b" 32 | ] 33 | } 34 | ] 35 | } 36 | -------------------------------------------------------------------------------- /examples/success/input_dir_ftp01_tree.json: -------------------------------------------------------------------------------- 1 | { 2 | "description": "Demonstrates retrieving 2 independent FTP directory trees (with subfolders). If the FTP is stable, 1. executor should output dos (!!) and mac subfolders with number of files in them (mac 7, dos 3). We will also list the files in both dirs, to have sth to compare with for next examples", 3 | "inputs": [ 4 | { 5 | "url": "ftp://ftp.ebi.ac.uk/pub/software/tools", 6 | "path": "/tes/old-tools", 7 | "type": "DIRECTORY" 8 | }, 9 | { 10 | "url": "ftp://ftp.ebi.ac.uk/pub/software/das", 11 | "path": "/elsewhere", 12 | "type": "DIRECTORY" 13 | } 14 | ], 15 | "executors": [ 16 | { 17 | "image": "alpine", 18 | "command": [ 19 | "sh", 20 | "-c", 21 | "find . -type d -exec sh -c 'echo \"$(find \"{}\" -type f | wc -l)\" {}' \\; | sort -nr" 22 | ], 23 | "workdir": "/tes/old-tools" 24 | }, 25 | { 26 | "image": "alpine", 27 | "command": [ 28 | "sh", 29 | "-c", 30 | "find . -type d -exec sh -c 'echo \"$(find \"{}\" -type f | wc -l)\" {}' \\;" 31 | ], 32 | "workdir": "/elsewhere" 33 | }, 34 | { 35 | "image": "alpine", 36 | "command": [ 37 | "sh", 38 | "-c", 39 | "find . -type f" 40 | ], 41 | "workdir": "/tes/old-tools" 42 | }, 43 | { 44 | "image": "alpine", 45 | "command": [ 46 | "sh", 47 | "-c", 48 | "find . -type f" 49 | ], 50 | "workdir": "/elsewhere" 51 | } 52 | ] 53 | } 54 | -------------------------------------------------------------------------------- /examples/success/input_dir_ftp02_sub.json: -------------------------------------------------------------------------------- 1 | { 2 | "description": "Uses FTP directories from previous example (remember dos and mac?). This time 2. directory is mounted below mountpoint for the 1. directory (nested). If the FTP is stable, should output dos (3), mac (7) and windows (3) subfolders", 3 | "inputs": [ 4 | { 5 | "url": "ftp://ftp.ebi.ac.uk/pub/software/tools", 6 | "path": "/tes/old-tools", 7 | "type": "DIRECTORY" 8 | }, 9 | { 10 | "url": "ftp://ftp.ebi.ac.uk/pub/software/das", 11 | "path": "/tes/old-tools/windows", 12 | "type": "DIRECTORY" 13 | } 14 | ], 15 | "executors": [ 16 | { 17 | "image": "alpine", 18 | "command": [ 19 | "sh", 20 | "-c", 21 | "find . -type d -exec sh -c 'echo \"$(find \"{}\" -type f | wc -l)\" {}' \\; | sort -nr" 22 | ], 23 | "workdir": "/tes/old-tools" 24 | }, 25 | { 26 | "image": "alpine", 27 | "command": [ 28 | "sh", 29 | "-c", 30 | "find . -type f" 31 | ], 32 | "workdir": "/tes/old-tools" 33 | } 34 | ] 35 | } 36 | -------------------------------------------------------------------------------- /examples/success/input_dir_ftp03_tree_shadowing.json: -------------------------------------------------------------------------------- 1 | { 2 | "description": "Uses FTP directories from previous examples (dos and mac). This time, we mount second source directory in place of existing directory (dos, that is). The content coming from 'ftp://ftp.ebi.ac.uk/pub/software/tools/dos' and 'ftp://ftp.ebi.ac.uk/pub/software/das' will be merged. Expect 6 files in dos directory.", 3 | "inputs": [ 4 | { 5 | "url": "ftp://ftp.ebi.ac.uk/pub/software/tools", 6 | "path": "/tes/old-tools", 7 | "type": "DIRECTORY" 8 | }, 9 | { 10 | "url": "ftp://ftp.ebi.ac.uk/pub/software/das", 11 | "path": "/tes/old-tools/dos", 12 | "type": "DIRECTORY" 13 | } 14 | ], 15 | "executors": [ 16 | { 17 | "image": "alpine", 18 | "command": [ 19 | "sh", 20 | "-c", 21 | "find . -type d -exec sh -c 'echo \"$(find \"{}\" -type f | wc -l)\" {}' \\; | sort -nr" 22 | ], 23 | "workdir": "/tes/old-tools" 24 | }, 25 | { 26 | "image": "alpine", 27 | "command": [ 28 | "sh", 29 | "-c", 30 | "find . -type f" 31 | ], 32 | "workdir": "/tes/old-tools" 33 | } 34 | ] 35 | } 36 | -------------------------------------------------------------------------------- /examples/success/input_dir_ftp04_tree_shadowing.json: -------------------------------------------------------------------------------- 1 | { 2 | "description": "The same as previous (03), but proves order does not matter.", 3 | "inputs": [ 4 | { 5 | "url": "ftp://ftp.ebi.ac.uk/pub/software/das", 6 | "path": "/tes/old-tools/dos", 7 | "type": "DIRECTORY" 8 | }, 9 | { 10 | "url": "ftp://ftp.ebi.ac.uk/pub/software/tools", 11 | "path": "/tes/old-tools", 12 | "type": "DIRECTORY" 13 | } 14 | ], 15 | "executors": [ 16 | { 17 | "image": "alpine", 18 | "command": [ 19 | "sh", 20 | "-c", 21 | "find . -type d -exec sh -c 'echo \"$(find \"{}\" -type f | wc -l)\" {}' \\; | sort -nr" 22 | ], 23 | "workdir": "/tes/old-tools" 24 | }, 25 | { 26 | "image": "alpine", 27 | "command": [ 28 | "sh", 29 | "-c", 30 | "find . -type f" 31 | ], 32 | "workdir": "/tes/old-tools" 33 | } 34 | ] 35 | } 36 | -------------------------------------------------------------------------------- /examples/success/input_dir_ftp05_merge.json: -------------------------------------------------------------------------------- 1 | { 2 | "description": "And now two source directories merged to a common place. There should be 6 files altogether in /tes/old-tools", 3 | "inputs": [ 4 | { 5 | "url": "ftp://ftp.ebi.ac.uk/pub/software/das", 6 | "path": "/tes/old-tools", 7 | "type": "DIRECTORY" 8 | }, 9 | { 10 | "url": "ftp://ftp.ebi.ac.uk/pub/software/tools/dos", 11 | "path": "/tes/old-tools", 12 | "type": "DIRECTORY" 13 | } 14 | ], 15 | "executors": [ 16 | { 17 | "image": "alpine", 18 | "command": [ 19 | "sh", 20 | "-c", 21 | "find . -type d -exec sh -c 'echo \"$(find \"{}\" -type f | wc -l)\" {}' \\; | sort -nr" 22 | ], 23 | "workdir": "/tes/old-tools" 24 | }, 25 | { 26 | "image": "alpine", 27 | "command": [ 28 | "sh", 29 | "-c", 30 | "find . -type f" 31 | ], 32 | "workdir": "/tes/old-tools" 33 | } 34 | ] 35 | } 36 | -------------------------------------------------------------------------------- /examples/success/input_dir_ftp06_file_merge.json: -------------------------------------------------------------------------------- 1 | { 2 | "description": "ERROR - FIXME: And now add files from different locations to directory tree retrieved from FTP. There should be 4 files in dos and 8 in mac.", 3 | "inputs": [ 4 | { 5 | "url": "https://raw.githubusercontent.com/EMBL-EBI-TSI/TESK/master/README.md", 6 | "path": "/tes/old-tools/mac/readme.md", 7 | "type": "FILE" 8 | }, 9 | { 10 | "url": "ftp://ftp.ebi.ac.uk/pub/software/tools", 11 | "path": "/tes/old-tools", 12 | "type": "DIRECTORY" 13 | }, 14 | { 15 | "url": "ftp://ftp.ebi.ac.uk/pub/software/das/README", 16 | "path": "/tes/old-tools/dos/readme.txt", 17 | "type": "FILE" 18 | } 19 | ], 20 | "executors": [ 21 | { 22 | "image": "alpine", 23 | "command": [ 24 | "sh", 25 | "-c", 26 | "find . -type d -exec sh -c 'echo \"$(find \"{}\" -type f | wc -l)\" {}' \\; | sort -nr" 27 | ], 28 | "workdir": "/tes/old-tools" 29 | }, 30 | { 31 | "image": "alpine", 32 | "command": [ 33 | "sh", 34 | "-c", 35 | "find . -type f" 36 | ], 37 | "workdir": "/tes/old-tools" 38 | } 39 | ] 40 | } 41 | -------------------------------------------------------------------------------- /examples/success/input_file_dir_merge.json: -------------------------------------------------------------------------------- 1 | { 2 | "description": "ERROR - FIXME: Demonstrates handling 2 http file inputs, that need to go to single directory.", 3 | "inputs": [ 4 | { 5 | "url": "https://raw.githubusercontent.com/EMBL-EBI-TSI/TESK/master/examples/success/hello.json", 6 | "path": "/tes/json", 7 | "type": "FILE" 8 | }, 9 | { 10 | "url": "https://raw.githubusercontent.com/EMBL-EBI-TSI/TESK/v0.1.9/scripts/taskmaster.py", 11 | "path": "/tes/python", 12 | "type": "FILE" 13 | } 14 | ], 15 | "executors": [ 16 | { 17 | "image": "alpine", 18 | "command": [ 19 | "sh", 20 | "-c", 21 | "find /tes -type f" 22 | ] 23 | } 24 | ] 25 | } 26 | -------------------------------------------------------------------------------- /examples/success/input_file_duplicate_names.json: -------------------------------------------------------------------------------- 1 | { 2 | "description": "ERROR- FIXME: Demonstrates an attempt of placing 2 different files in the same place. At the moment, teh last one wins (will overwrite previous occurrences)", 3 | "inputs": [ 4 | { 5 | "url": "https://raw.githubusercontent.com/EMBL-EBI-TSI/TESK/master/examples/success/hello.json", 6 | "path": "/tes/volumes/input", 7 | "type": "FILE" 8 | }, 9 | { 10 | "url": "https://raw.githubusercontent.com/EMBL-EBI-TSI/TESK/master/examples/success/stdin.json", 11 | "path": "/tes/volumes/input", 12 | "type": "FILE" 13 | } 14 | ], 15 | "executors": [ 16 | { 17 | "image": "alpine", 18 | "command": [ 19 | "cat", 20 | "/tes/volumes/input" 21 | ] 22 | } 23 | ] 24 | } 25 | -------------------------------------------------------------------------------- /examples/success/input_file_http.json: -------------------------------------------------------------------------------- 1 | { 2 | "description": "Demonstrates handling single http file input. Will output some nice looking JSON to stdout.", 3 | "inputs": [ 4 | { 5 | "url": "https://raw.githubusercontent.com/EMBL-EBI-TSI/TESK/master/examples/success/hello.json", 6 | "path": "/tes/volumes/input", 7 | "type": "FILE" 8 | } 9 | ], 10 | "executors": [ 11 | { 12 | "image": "alpine", 13 | "command": [ 14 | "cat", 15 | "/tes/volumes/input" 16 | ] 17 | } 18 | ] 19 | } 20 | -------------------------------------------------------------------------------- /examples/success/input_file_http_nested_mounts.json: -------------------------------------------------------------------------------- 1 | { 2 | "description": "Demonstrates handling 2 http file inputs with nested mountpoints. Should find both files in their respective locations.", 3 | "inputs": [ 4 | { 5 | "url": "https://raw.githubusercontent.com/EMBL-EBI-TSI/TESK/master/examples/success/hello.json", 6 | "path": "/tes/volumes/input.json", 7 | "type": "FILE" 8 | }, 9 | { 10 | "url": "https://raw.githubusercontent.com/EMBL-EBI-TSI/TESK/master/examples/success/env.json", 11 | "path": "/tes/file.json", 12 | "type": "FILE" 13 | } 14 | ], 15 | "executors": [ 16 | { 17 | "image": "alpine", 18 | "command": [ 19 | "sh", 20 | "-c", 21 | "find . -name '*.json'" 22 | ] 23 | } 24 | ] 25 | } 26 | -------------------------------------------------------------------------------- /examples/success/output.json: -------------------------------------------------------------------------------- 1 | { 2 | "description": "FTP output - Caution - for DIRECTORY creates and not only copies directory. Works in a different way that inputs.", 3 | "outputs": [ 4 | { 5 | "description": "ftp://ftp-private.ebi.ac.uk/upload/examples/existing1 exists and is initially empty - expect file.txt created and having contents of file1.txt", 6 | "path": "/tes/file1.txt", 7 | "type": "FILE", 8 | "url": "ftp://ftp-private.ebi.ac.uk/upload/examples/existing1/file.txt" 9 | }, 10 | { 11 | "description": "ftp://ftp-private.ebi.ac.uk/upload/examples/existing2 exists and contains file.txt - expect file.txt overwriten and having contents of file1.txt", 12 | "path": "/tes/file1.txt", 13 | "type": "FILE", 14 | "url": "ftp://ftp-private.ebi.ac.uk/upload/examples/existing2/file.txt" 15 | }, 16 | { 17 | "description": "ftp://ftp-private.ebi.ac.uk/upload/examples/tes does not exist - expect tes directory will be created and will contain file1.txt and file2.txt. Different behaviour than by inputs!!! It is not directory contents that is copied (inputs for directories work as merge), but a new directory is created with the source name (no name change possible).", 18 | "path": "/tes", 19 | "type": "DIRECTORY", 20 | "url": "ftp://ftp-private.ebi.ac.uk/upload/examples" 21 | } 22 | ], 23 | "executors": [ 24 | { 25 | "image": "alpine", 26 | "command": [ 27 | "echo", 28 | "This goes to file1" 29 | ], 30 | "stdout": "/tes/file1.txt" 31 | }, 32 | { 33 | "image": "alpine", 34 | "command": [ 35 | "sh", 36 | "-c", 37 | "echo 'This goes to file2' > /tes/file2.txt" 38 | ] 39 | } 40 | ] 41 | } 42 | -------------------------------------------------------------------------------- /examples/success/output_overwrite.json: -------------------------------------------------------------------------------- 1 | { 2 | "description": "FTP output - overwrite DIR", 3 | "outputs": [ 4 | { 5 | "description": "ftp://ftp-private.ebi.ac.uk/upload/examples/tes already exists and containes file1.txt and remaining.txt - expect tes will contain file1.txt (changed), file2.txt and remaining.txt.", 6 | "path": "/tes", 7 | "type": "DIRECTORY", 8 | "url": "ftp://ftp-private.ebi.ac.uk/upload/examples" 9 | } 10 | ], 11 | "executors": [ 12 | { 13 | "image": "alpine", 14 | "command": [ 15 | "echo", 16 | "This goes to file1" 17 | ], 18 | "stdout": "/tes/file1.txt" 19 | }, 20 | { 21 | "image": "alpine", 22 | "command": [ 23 | "sh", 24 | "-c", 25 | "echo 'This goes to file2' > /tes/file2.txt" 26 | ] 27 | } 28 | ] 29 | } 30 | -------------------------------------------------------------------------------- /examples/success/output_tree.json: -------------------------------------------------------------------------------- 1 | { 2 | "description": "FTP output - Copies directory with subdirs and files (recursively)", 3 | "outputs": [ 4 | { 5 | "description": "ftp://ftp-private.ebi.ac.uk/upload/examples/tes does not exist - expect tes directory will be created and will contain subfolder sub).", 6 | "path": "/tes", 7 | "type": "DIRECTORY", 8 | "url": "ftp://ftp-private.ebi.ac.uk/upload/examples" 9 | } 10 | ], 11 | "executors": [ 12 | { 13 | "image": "alpine", 14 | "command": [ 15 | "echo", 16 | "This goes to file1" 17 | ], 18 | "stdout": "/tes/file1.txt" 19 | }, 20 | { 21 | "image": "alpine", 22 | "command": [ 23 | "sh", 24 | "-c", 25 | "mkdir /tes/sub; echo 'This goes to file2' > /tes/sub/file2.txt;" 26 | ] 27 | } 28 | ] 29 | } 30 | -------------------------------------------------------------------------------- /examples/success/output_volume.json: -------------------------------------------------------------------------------- 1 | { 2 | "description": "FTP outputting file and directory from volume location", 3 | "volumes": [ 4 | "/tes" 5 | ], 6 | "outputs": [ 7 | { 8 | "description": "ftp://ftp-private.ebi.ac.uk/upload/examples/existing1 exists and is initially empty - expect file1.txt created", 9 | "path": "/tes/file1.txt", 10 | "type": "FILE", 11 | "url": "ftp://ftp-private.ebi.ac.uk/upload/examples/existing1/file1.txt" 12 | }, 13 | { 14 | "description": "ftp://ftp-private.ebi.ac.uk/upload/examples/tes does not exist - expect will be created and will contain file1.txt and file2.txt", 15 | "path": "/tes", 16 | "type": "DIRECTORY", 17 | "url": "ftp://ftp-private.ebi.ac.uk/upload/examples" 18 | } 19 | ], 20 | "executors": [ 21 | { 22 | "image": "alpine", 23 | "command": [ 24 | "echo", 25 | "This goes to file1" 26 | ], 27 | "stdout": "/tes/file1.txt" 28 | }, 29 | { 30 | "image": "alpine", 31 | "command": [ 32 | "sh", 33 | "-c", 34 | "echo 'This goes to file2' > /tes/file2.txt" 35 | ] 36 | } 37 | ] 38 | } 39 | -------------------------------------------------------------------------------- /examples/success/stderr.json: -------------------------------------------------------------------------------- 1 | { 2 | "description": "Demonstrates capturing stderr to file (and volumes). In stdout of 1. executor there should be only items containing bash; Second executor should output errors from grep.", 3 | "volumes": [ 4 | "/outputs" 5 | ], 6 | "executors": [ 7 | { 8 | "image": "ubuntu", 9 | "command": [ 10 | "sh", 11 | "-c", 12 | "grep bash /etc/s* || exit 0" 13 | ], 14 | "stderr": "/outputs/stderr" 15 | }, 16 | { 17 | "image": "alpine", 18 | "command": [ 19 | "cat", 20 | "/outputs/stderr" 21 | ] 22 | } 23 | ] 24 | } 25 | -------------------------------------------------------------------------------- /examples/success/stdin.json: -------------------------------------------------------------------------------- 1 | { 2 | "description": "Testing stdin (in its simplest form, without mounted inputs)", 3 | "executors": [ 4 | { 5 | "image": "alpine", 6 | "command": [ 7 | "cat" 8 | ], 9 | "stdin": "/etc/fstab" 10 | } 11 | ] 12 | } 13 | -------------------------------------------------------------------------------- /examples/success/stdout.json: -------------------------------------------------------------------------------- 1 | { 2 | "description": "Demonstrates capturing stdout to file (and volumes).", 3 | "volumes": [ 4 | "/outputs" 5 | ], 6 | "executors": [ 7 | { 8 | "image": "ubuntu", 9 | "command": [ 10 | "echo", 11 | "This will appear in stdout, but of the 2. executor." 12 | ], 13 | "stdout": "/outputs/stdout" 14 | }, 15 | { 16 | "image": "alpine", 17 | "command": [ 18 | "cat", 19 | "/outputs/stdout" 20 | ] 21 | } 22 | ] 23 | } 24 | -------------------------------------------------------------------------------- /examples/success/tags.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "I do nothing", 3 | "description": "Well, nothing happens here. Just observe that executor spec, name, description and tags appear in the output", 4 | "executors": [ 5 | { 6 | "image": "alpine", 7 | "command": [ 8 | "sh", 9 | "-c", 10 | "exit 0" 11 | ] 12 | } 13 | ], 14 | "tags": { 15 | "project": "Important one", 16 | "author": "Jane Doe" 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /examples/success/workdir.json: -------------------------------------------------------------------------------- 1 | { 2 | "description": "Testing workdir", 3 | "executors": [ 4 | { 5 | "image": "alpine", 6 | "command": [ 7 | "ls" 8 | ], 9 | "workdir": "/etc" 10 | } 11 | ] 12 | } 13 | -------------------------------------------------------------------------------- /examples/taskCreate: -------------------------------------------------------------------------------- 1 | 2 | 3 | # 4 | # Usage: 5 | # 6 | # 1. Update 'baseUrl' (below) 7 | # 8 | # 2. Call this script 9 | # 10 | # taskCreate 11 | # 12 | # e.g. 13 | # 14 | # taskCreate localftp/taskWithIO.json 15 | # 16 | 17 | # $ minikube service list 18 | # |-------------|----------------------|-----------------------------| 19 | # | NAMESPACE | NAME | URL | 20 | # |-------------|----------------------|-----------------------------| 21 | # | default | kubernetes | No node port | 22 | # | default | tesk-api | http://192.168.99.100:31934 | <= baseUrl 23 | # | kube-system | kube-dns | No node port | 24 | # | kube-system | kubernetes-dashboard | No node port | 25 | # |-------------|----------------------|-----------------------------| 26 | 27 | baseUrl='http://192.168.99.100:31209' 28 | 29 | jsonFile="$1" 30 | 31 | curl -iv \ 32 | -X POST \ 33 | -s \ 34 | --header 'Content-Type: application/json' \ 35 | --header 'Accept: application/json' \ 36 | -d "@${jsonFile}" \ 37 | "$baseUrl/v1/tasks" 38 | 39 | -------------------------------------------------------------------------------- /examples/taskList: -------------------------------------------------------------------------------- 1 | 2 | curl -i http://192.168.99.100:31934/v1/tasks 3 | 4 | -------------------------------------------------------------------------------- /source/tesk-core/.dockerignore: -------------------------------------------------------------------------------- 1 | .coverage 2 | .pytest_cache/ 3 | .tox/ 4 | .eggs/ 5 | tesk-core/containers/ 6 | -------------------------------------------------------------------------------- /source/tesk-core/.github/workflows/docker-build-publish-filer.yml: -------------------------------------------------------------------------------- 1 | name: tesk-core-filer 2 | 3 | on: 4 | push: 5 | branches: [ 'testing-gh-action' ] 6 | tags: [ '*' ] 7 | 8 | workflow_dispatch: 9 | inputs: 10 | profile: 11 | description: Profile name 12 | required: false 13 | default: tesk-core-filler 14 | 15 | jobs: 16 | build-from-source: 17 | runs-on: ubuntu-latest 18 | steps: 19 | - name: Checkout elixir-cloud-aai/tesk-core 20 | uses: actions/checkout@v3 21 | 22 | - name: Set up QEMU 23 | uses: docker/setup-qemu-action@v2 24 | 25 | - name: Set up Docker Buildx 26 | uses: docker/setup-buildx-action@v2 27 | 28 | - name: Login to DockerHub 29 | uses: docker/login-action@v2 30 | with: 31 | username: ${{ secrets.DOCKERHUB_USERNAME }} 32 | password: ${{ secrets.DOCKERHUB_TOKEN }} 33 | 34 | - name: Extract metadata (tags, labels) for Docker 35 | id: meta 36 | uses: docker/metadata-action@v4 37 | with: 38 | images: | 39 | elixircloud/${{ github.workflow }} 40 | 41 | - name: Build and push Docker images 42 | uses: docker/build-push-action@v3 43 | with: 44 | context: . 45 | push: true 46 | file: ./containers/filer.Dockerfile 47 | tags: ${{ steps.meta.outputs.tags }} 48 | labels: ${{ steps.meta.outputs.labels }} 49 | 50 | -------------------------------------------------------------------------------- /source/tesk-core/.github/workflows/docker-build-publish-taskmaster.yml: -------------------------------------------------------------------------------- 1 | name: tesk-core-taskmaster 2 | 3 | on: 4 | push: 5 | branches: [ 'testing-gh-action' ] 6 | tags: [ '*' ] 7 | 8 | workflow_dispatch: 9 | inputs: 10 | profile: 11 | description: Profile name 12 | required: false 13 | default: tesk-core-taskmaster 14 | 15 | jobs: 16 | build-from-source: 17 | runs-on: ubuntu-latest 18 | steps: 19 | - name: Checkout elixir-cloud-aai/tesk-core 20 | uses: actions/checkout@v3 21 | 22 | - name: Set up QEMU 23 | uses: docker/setup-qemu-action@v2 24 | 25 | - name: Set up Docker Buildx 26 | uses: docker/setup-buildx-action@v2 27 | 28 | - name: Login to DockerHub 29 | uses: docker/login-action@v2 30 | with: 31 | username: ${{ secrets.DOCKERHUB_USERNAME }} 32 | password: ${{ secrets.DOCKERHUB_TOKEN }} 33 | 34 | - name: Extract metadata (tags, labels) for Docker 35 | id: meta 36 | uses: docker/metadata-action@v4 37 | with: 38 | images: | 39 | elixircloud/${{ github.workflow }} 40 | 41 | - name: Build and push Docker images 42 | uses: docker/build-push-action@v3 43 | with: 44 | context: . 45 | push: true 46 | file: ./containers/taskmaster.Dockerfile 47 | tags: ${{ steps.meta.outputs.tags }} 48 | labels: ${{ steps.meta.outputs.labels }} 49 | 50 | -------------------------------------------------------------------------------- /source/tesk-core/.github/workflows/tox.yml: -------------------------------------------------------------------------------- 1 | name: Python package 2 | 3 | on: 4 | - push 5 | - pull_request 6 | 7 | jobs: 8 | build: 9 | runs-on: ubuntu-latest 10 | strategy: 11 | matrix: 12 | python-version: ['3.8', '3.9', '3.10', '3.11', '3.12'] 13 | 14 | steps: 15 | - uses: actions/checkout@v3 16 | - name: Set up Python ${{ matrix.python-version }} 17 | uses: actions/setup-python@v4 18 | with: 19 | python-version: ${{ matrix.python-version }} 20 | - name: Install dependencies 21 | run: | 22 | python -m pip install --upgrade pip 23 | python -m pip install tox tox-gh-actions 24 | - name: Test with tox 25 | run: tox 26 | -------------------------------------------------------------------------------- /source/tesk-core/.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | .eggs/ 17 | lib/ 18 | lib64/ 19 | parts/ 20 | sdist/ 21 | var/ 22 | wheels/ 23 | *.egg-info/ 24 | .installed.cfg 25 | *.egg 26 | .idea 27 | MANIFEST 28 | 29 | # PyInstaller 30 | # Usually these files are written by a python script from a template 31 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 32 | *.manifest 33 | *.spec 34 | 35 | # Installer logs 36 | pip-log.txt 37 | pip-delete-this-directory.txt 38 | 39 | # Unit test / coverage reports 40 | htmlcov/ 41 | .tox/ 42 | .coverage 43 | .coverage.* 44 | .cache 45 | nosetests.xml 46 | coverage.xml 47 | *.cover 48 | .hypothesis/ 49 | .pytest_cache/ 50 | 51 | # Sphinx documentation 52 | docs/_build/ 53 | 54 | # PyBuilder 55 | target/ 56 | 57 | # pyenv 58 | .python-version 59 | 60 | # Environments 61 | .env 62 | .venv 63 | env/ 64 | venv/ 65 | ENV/ 66 | env.bak/ 67 | venv.bak/ 68 | 69 | # mkdocs documentation 70 | /site 71 | 72 | # mypy 73 | .mypy_cache/ 74 | 75 | # vim 76 | .swp 77 | -------------------------------------------------------------------------------- /source/tesk-core/.travis.yml: -------------------------------------------------------------------------------- 1 | sudo: false 2 | language: python 3 | dist: focal 4 | cache: pip 5 | python: 6 | - '3.8' 7 | - '3.9' 8 | - '3.10' 9 | - '3.11' 10 | - '3.12' 11 | install: 12 | - sudo apt update 13 | - sudo apt upgrade 14 | - sudo apt install rustc cargo 15 | - pip install tox-travis 16 | script: tox 17 | 18 | -------------------------------------------------------------------------------- /source/tesk-core/LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright 2017 EMBL - European Bioinformatics Intitute 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /source/tesk-core/MANIFEST.in: -------------------------------------------------------------------------------- 1 | include README.md 2 | -------------------------------------------------------------------------------- /source/tesk-core/README.md: -------------------------------------------------------------------------------- 1 | [![Build Status](https://travis-ci.org/EMBL-EBI-TSI/tesk-core.svg?branch=master)](https://travis-ci.org/EMBL-EBI-TSI/tesk-core) 2 | [![codecov](https://codecov.io/gh/EMBL-EBI-TSI/tesk-core/branch/master/graph/badge.svg)](https://codecov.io/gh/EMBL-EBI-TSI/tesk-core) 3 | 4 | ## Introduction 5 | 6 | This project is part of the [TESK](https://github.com/EMBL-EBI-TSI/TESK) initiative. 7 | It contains the code needed to generate 2 types of agents that reside in kubernetes: 8 | * The taskmaster, which spins up the containers needed to complete tasks as defined by TESK 9 | * The filer, which populates volumes and input files and uploads output files 10 | 11 | ## How to use 12 | 13 | Since the code is meant to be in kubernetes pods, the code needs to be packaged into containers. 14 | Their descriptions can be found in `containers/`. 15 | The root folder assumed to build the containers is the root of this package. 16 | 17 | ## Unit testing 18 | 19 | Unit testing needs the `tox` package. 20 | This software will take care of creating virtual environments and installing dependencies in them before running the actual tests and generating the coverage reports. 21 | 22 | ``` 23 | $ tox 24 | ``` 25 | -------------------------------------------------------------------------------- /source/tesk-core/cloudbuild.yaml: -------------------------------------------------------------------------------- 1 | steps: 2 | - name: 'gcr.io/cloud-builders/docker' 3 | args: ['build', '-t', 'eu.gcr.io/tes-wes/taskmaster:$TAG_NAME', '-f', 'containers/taskmaster.Dockerfile', '.'] 4 | - name: 'gcr.io/cloud-builders/docker' 5 | args: ['build', '-t', 'eu.gcr.io/tes-wes/filer:$TAG_NAME', '-f', 'containers/filer.Dockerfile', '.'] 6 | images: ['eu.gcr.io/tes-wes/taskmaster:$TAG_NAME', 'eu.gcr.io/tes-wes/filer:$TAG_NAME'] 7 | -------------------------------------------------------------------------------- /source/tesk-core/cloudbuild_testing.yaml: -------------------------------------------------------------------------------- 1 | steps: 2 | - name: 'gcr.io/cloud-builders/docker' 3 | args: ['build', '-t', 'eu.gcr.io/tes-wes/taskmaster:testing', '-f', 'containers/taskmaster.Dockerfile', '.'] 4 | - name: 'gcr.io/cloud-builders/docker' 5 | args: ['build', '-t', 'eu.gcr.io/tes-wes/filer:testing', '-f', 'containers/filer.Dockerfile', '.'] 6 | images: ['eu.gcr.io/tes-wes/taskmaster:testing', 'eu.gcr.io/tes-wes/filer:testing'] 7 | -------------------------------------------------------------------------------- /source/tesk-core/containers/filer.Dockerfile: -------------------------------------------------------------------------------- 1 | # Builder: produce wheels 2 | 3 | FROM alpine:3.10 as builder 4 | 5 | RUN apk add --no-cache python3 6 | RUN apk add --no-cache git 7 | RUN python3 -m pip install --upgrade setuptools pip wheel 8 | 9 | WORKDIR /app/ 10 | COPY . . 11 | 12 | RUN python3 setup.py bdist_wheel 13 | 14 | # Install: copy tesk-core*.whl and install it with dependencies 15 | 16 | FROM alpine:3.10 17 | 18 | RUN apk add --no-cache python3 19 | 20 | COPY --from=builder /app/dist/tesk*.whl /root/ 21 | RUN python3 -m pip install --disable-pip-version-check --no-cache-dir /root/tesk*.whl 22 | 23 | USER 100 24 | 25 | ENTRYPOINT ["filer"] 26 | -------------------------------------------------------------------------------- /source/tesk-core/containers/taskmaster.Dockerfile: -------------------------------------------------------------------------------- 1 | # Builder: produce wheels 2 | 3 | FROM alpine:3.10 as builder 4 | 5 | RUN apk add --no-cache python3 6 | RUN apk add --no-cache git 7 | RUN python3 -m pip install --upgrade setuptools pip wheel 8 | 9 | WORKDIR /app/ 10 | COPY . . 11 | 12 | RUN python3 setup.py bdist_wheel 13 | 14 | # Install: copy tesk-core*.whl and install it with dependencies 15 | 16 | FROM alpine:3.10 17 | 18 | RUN apk add --no-cache python3 19 | 20 | COPY --from=builder /app/dist/tesk*.whl /root/ 21 | RUN python3 -m pip install --disable-pip-version-check --no-cache-dir /root/tesk*.whl 22 | 23 | RUN adduser --uid 100 -S taskmaster 24 | USER 100 25 | 26 | ENTRYPOINT ["taskmaster"] 27 | -------------------------------------------------------------------------------- /source/tesk-core/doc/taskmaster_architecture.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/elixir-cloud-aai/TESK/90e2e9bc84c7b47dd1fa548b223e72c885bd4ba6/source/tesk-core/doc/taskmaster_architecture.png -------------------------------------------------------------------------------- /source/tesk-core/dockerBuild: -------------------------------------------------------------------------------- 1 | scripts//dockerBuild -------------------------------------------------------------------------------- /source/tesk-core/dockerRun: -------------------------------------------------------------------------------- 1 | scripts//dockerRun -------------------------------------------------------------------------------- /source/tesk-core/examples/inputFile.json: -------------------------------------------------------------------------------- 1 | { 2 | "outputs": [ 3 | ], 4 | "inputs": [ 5 | 6 | { 7 | "name": "input1", 8 | "description": "Example", 9 | "url": "file:///home/tfga/workspace/cwl-tes/README.md", 10 | "path": "/some/volume/input.txt", 11 | "type": "FILE" 12 | } 13 | ], 14 | "volumes": [ 15 | ], 16 | "executors": [ 17 | { 18 | "apiVersion": "batch/v1", 19 | "kind": "Job", 20 | "metadata": { 21 | "annotations": { 22 | }, 23 | "labels": { 24 | "job-type": "executor", 25 | "taskmaster-name": "task-1000", 26 | "executor-no": "0", 27 | "creator-user-id": "test-user-id" 28 | }, 29 | "name": "task-1000-ex-00" 30 | }, 31 | "spec": { 32 | "template": { 33 | "metadata": { 34 | "name": "task-1000-ex-00" 35 | }, 36 | "spec": { 37 | "containers": [ 38 | { 39 | "command": [ 40 | "cat", 41 | "/some/volume/input.txt" 42 | ], 43 | "image": "ubuntu", 44 | "name": "task-1000-ex-00", 45 | "resources": { 46 | } 47 | } 48 | ], 49 | "restartPolicy": "Never" 50 | } 51 | } 52 | } 53 | } 54 | ], 55 | "resources": { 56 | "disk_gb": 0.1 57 | } 58 | } -------------------------------------------------------------------------------- /source/tesk-core/examples/inputHelloWorld.json: -------------------------------------------------------------------------------- 1 | { 2 | "outputs": [ 3 | ], 4 | "inputs": [ 5 | 6 | { 7 | "name": "input1", 8 | "description": "Example", 9 | "url": "ftp://example.org/resource.txt", 10 | "path": "/some/volume/input.txt", 11 | "type": "FILE" 12 | } 13 | ], 14 | "volumes": [ 15 | ], 16 | "executors": [ 17 | { 18 | "apiVersion": "batch/v1", 19 | "kind": "Job", 20 | "metadata": { 21 | "annotations": { 22 | }, 23 | "labels": { 24 | "job-type": "executor", 25 | "taskmaster-name": "task-98605447", 26 | "executor-no": "0", 27 | "creator-user-id": "test-user-id" 28 | }, 29 | "name": "task-98605447-ex-00" 30 | }, 31 | "spec": { 32 | "template": { 33 | "metadata": { 34 | "name": "task-98605447-ex-00" 35 | }, 36 | "spec": { 37 | "containers": [ 38 | { 39 | "command": [ 40 | "echo", 41 | "hello world" 42 | ], 43 | "image": "ubuntu", 44 | "name": "task-98605447-ex-00", 45 | "resources": { 46 | } 47 | } 48 | ], 49 | "restartPolicy": "Never" 50 | } 51 | } 52 | } 53 | } 54 | ], 55 | "resources": { 56 | "disk_gb": 0.1 57 | } 58 | } -------------------------------------------------------------------------------- /source/tesk-core/examples/inputHttp.json: -------------------------------------------------------------------------------- 1 | { 2 | "outputs": [ 3 | ], 4 | "inputs": [ 5 | 6 | { 7 | "name": "input1", 8 | "description": "Example", 9 | "url": "https://github.com/EMBL-EBI-TSI/tesk-core/raw/master/README.md", 10 | "path": "/some/volume/input.txt", 11 | "type": "FILE" 12 | } 13 | ], 14 | "volumes": [ 15 | ], 16 | "executors": [ 17 | { 18 | "apiVersion": "batch/v1", 19 | "kind": "Job", 20 | "metadata": { 21 | "annotations": { 22 | }, 23 | "labels": { 24 | "job-type": "executor", 25 | "taskmaster-name": "task-1000", 26 | "executor-no": "0", 27 | "creator-user-id": "test-user-id" 28 | }, 29 | "name": "task-1000-ex-00" 30 | }, 31 | "spec": { 32 | "template": { 33 | "metadata": { 34 | "name": "task-1000-ex-00" 35 | }, 36 | "spec": { 37 | "containers": [ 38 | { 39 | "command": [ 40 | "cat", 41 | "/some/volume/input.txt" 42 | ], 43 | "image": "ubuntu", 44 | "name": "task-1000-ex-00", 45 | "resources": { 46 | } 47 | } 48 | ], 49 | "restartPolicy": "Never" 50 | } 51 | } 52 | } 53 | } 54 | ], 55 | "resources": { 56 | "disk_gb": 0.1 57 | } 58 | } -------------------------------------------------------------------------------- /source/tesk-core/examples/transferPvc/Readme.md: -------------------------------------------------------------------------------- 1 | #### PVC 2 | 3 | 4 | 1. `$ ./minikubeStart` 5 | 6 | Creates `/transferAtNode` mount at the node. 7 | 8 | 2. `$ kuCreate .` 9 | 10 | This will create: 11 | 12 | * a `pv` 13 | * a `pvc` that uses the `pv` 14 | * a `pod` that mounts the `pvc` 15 | 16 | This pod will have the mount `/transfer` => `/home/tfga/workspace/cwl-tes`: 17 | 18 | $ ./clean 19 | -- Deleting all pods ------------------------------------ 20 | pod "nginx-pod" deleted 21 | 22 | -- Deleting all pvc ------------------------------------ 23 | persistentvolumeclaim "transfer-pvc" deleted 24 | 25 | -- Deleting all pv ------------------------------------ 26 | persistentvolume "transfer-pv" deleted 27 | 28 | tfga@tfga ~/workspace/KubernetesTest/pvc 29 | $ kuCreate . 30 | pod/nginx-pod created 31 | persistentvolume/transfer-pv created 32 | persistentvolumeclaim/transfer-pvc created 33 | 34 | tfga@tfga ~/workspace/KubernetesTest/pvc 35 | $ kuSshPod nginx-pod 36 | root@nginx-pod:/# ls /transfer/ 37 | LICENSE old tmp0k0657m1 tmp4a18kf4u tmp9h0p6cg3 tmpbjm_c9lm tmpf68dn9n8 tmpig_ryne9 tmpmmn0ia6i tmpp9vkcipm tmpt89n8hz8 tmpw61zd_kd tmpxrbhu5mj tmpyyjhvh8q 38 | README.md requirements.txt tmp0pzg_4xc tmp4jsl4elr tmp9zt9vkub tmpbn9f3s3t tmpfcxj9a_y tmpiz83h1fm tmpn47zb3m8 tmppi05vm5l tmptms28mke tmpwea3jpp6 tmpxrnnz9bj tmpz0fg89z_ 39 | build runFileSystem tmp1b46r818 tmp4qfxl_6k tmp_f9xuo8k tmpc5y3_h7w tmpfuup87gk tmpj7f96z1r tmpn9f2w3lj tmpplw4y2nn tmpu25kek04 tmpwjeajmv6 tmpxuhnrw8n tmpzgouou8q 40 | cwl-tes runFunnel tmp2m8non62 tmp57zvcdsl tmp_ngu1f6g tmpcxpn_zmu tmpg0m9rffx tmpjty8x2dl tmpnb2d4ibx tmppqgpkjsi tmpu27kpb2e tmpwq5ni_9v tmpxzi574au tmpzgx87pcz 41 | cwl_tes runTesDev+Ftp tmp2mck1kr9 tmp6llnimfl tmp_zio3bq9 tmpd25x5s55 tmpg2iwsv7s tmpk1aa5z63 tmpnf4uazrx tmpquo3o00v tmpupko0xrx tmpws_ce7sy tmpy1883fph tox.ini 42 | cwl_tes.egg-info setup.py tmp2pnkovzz tmp77zykmun tmpab8tvnq_ tmpda_r6p8_ tmpgpcozofq tmpk6gju6xl tmpnq_qbh6w tmpr1ciykyp tmpusj6yl5b tmpx1m3dmmy tmpy1vzo358 unify 43 | debug.log tests tmp2th8wyn_ tmp78nw56aj tmpah7u70t8 tmpdcv4pwg3 tmpgq80j8u5 tmpkpe3ipse tmpoefw5ujx tmprhfwrhy7 tmpve0cce76 tmpx31o4ylu tmpy6p_gwpm verbose.log 44 | dist tmp049iugiv tmp36ryoi5x tmp835xzwq_ tmpazcfpcik tmpdk2aq8tc tmph875ges2 tmpktik6m0v tmpolb1fklc tmpsfzx43fd tmpvxt19l8r tmpx5wnwpoe tmpy6tmx0rg 45 | funnel-work-dir tmp06oyrxvw tmp3yxlshpt tmp8onbqh8f tmpbe78f6el tmpdutg8ng_ tmphrtip1o8 tmpld1j21ye tmpoxvg9xpl tmpslx5l0rl tmpvyj9cppw tmpx9opul51 tmpystuxi10 46 | 47 | You can only do this with containers like nginx, because it _keeps running_ (it's a server). If you do it with `filer`, it will start and finish before you can `docker exec` into it. 48 | You can only `docker exec` into running containers. 49 | -------------------------------------------------------------------------------- /source/tesk-core/examples/transferPvc/clean: -------------------------------------------------------------------------------- 1 | 2 | 3 | deleteAll() { 4 | 5 | objType="$1" 6 | 7 | echo "-- Deleting all $objType ------------------------------------" 8 | kubectl delete "$objType" --all 9 | echo 10 | } 11 | 12 | deleteAll pods 13 | deleteAll pvc 14 | deleteAll pv 15 | -------------------------------------------------------------------------------- /source/tesk-core/examples/transferPvc/minikubeStart: -------------------------------------------------------------------------------- 1 | 2 | 3 | wesBasePath='/home/tfga/workspace/cwl-tes' 4 | 5 | minikube start --mount --mount-string "$wesBasePath:/transferAtNode" 6 | 7 | -------------------------------------------------------------------------------- /source/tesk-core/examples/transferPvc/pod.yaml: -------------------------------------------------------------------------------- 1 | kind: Pod 2 | apiVersion: v1 3 | metadata: 4 | name: nginx-pod 5 | spec: 6 | volumes: 7 | - name: transfer-volume 8 | persistentVolumeClaim: 9 | claimName: transfer-pvc 10 | containers: 11 | - name: nginx-container 12 | image: nginx 13 | volumeMounts: 14 | - mountPath: /transfer 15 | name: transfer-volume 16 | -------------------------------------------------------------------------------- /source/tesk-core/examples/transferPvc/pv.yaml: -------------------------------------------------------------------------------- 1 | kind: PersistentVolume 2 | apiVersion: v1 3 | metadata: 4 | name: transfer-pv 5 | labels: 6 | type: local 7 | spec: 8 | storageClassName: manual 9 | capacity: 10 | storage: 10Gi 11 | accessModes: 12 | - ReadWriteOnce 13 | hostPath: 14 | path: /transferAtNode 15 | -------------------------------------------------------------------------------- /source/tesk-core/examples/transferPvc/pvc.yaml: -------------------------------------------------------------------------------- 1 | kind: PersistentVolumeClaim 2 | apiVersion: v1 3 | metadata: 4 | name: transfer-pvc 5 | spec: 6 | storageClassName: manual 7 | accessModes: 8 | - ReadWriteOnce 9 | resources: 10 | requests: 11 | storage: 3Gi 12 | -------------------------------------------------------------------------------- /source/tesk-core/init: -------------------------------------------------------------------------------- 1 | 2 | # Creating virtualenv 3 | virtualenv --clear -p python2.7 .venv 4 | 5 | # Activating virtualenv 6 | . .venv/bin/activate 7 | 8 | ./install 9 | -------------------------------------------------------------------------------- /source/tesk-core/install: -------------------------------------------------------------------------------- 1 | 2 | 3 | pip install . 4 | 5 | -------------------------------------------------------------------------------- /source/tesk-core/pytest.ini: -------------------------------------------------------------------------------- 1 | [pytest] 2 | env = 3 | FTP_USER=user 4 | FTP_PASS=pass 5 | TESK_FTP_USERNAME=user 6 | TESK_FTP_PASSWORD=pass 7 | FTP_HOME =/tmp 8 | FTP_FIXTURE_SCOPE=function 9 | FTP_PORT = 2111 -------------------------------------------------------------------------------- /source/tesk-core/scripts/dockerBuild: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # 3 | # Usage: 4 | # 5 | # buildDockerImage filer or 6 | # buildDockerImage taskmaster 7 | # 8 | 9 | IMAGE=$1 10 | 11 | if [ -z "$IMAGE" ]; 12 | then 13 | echo "Use: $0 [tag]" 14 | exit 12 15 | fi 16 | 17 | TAG=$2 18 | 19 | if [ -z "$TAG" ]; 20 | then 21 | TAG=testing 22 | fi 23 | 24 | if command -V buildah; 25 | then 26 | buildah bud -t "docker.io/elixircloud/tesk-core-$IMAGE:$TAG" \ 27 | --format=docker --no-cache \ 28 | -f "containers/$IMAGE.Dockerfile" 29 | else 30 | docker build -t "docker.io/elixircloud/tesk-core-$1:$TAG" -f "containers/$1.Dockerfile" . 31 | fi 32 | -------------------------------------------------------------------------------- /source/tesk-core/scripts/dockerRun: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # 3 | # Usage: 4 | # 5 | # buildRun filer or 6 | # buildRun taskmaster 7 | # 8 | 9 | imageName="$1" 10 | shift 11 | 12 | docker run "docker.io/elixircloud/tesk-core-$imageName:testing" "$@" 13 | -------------------------------------------------------------------------------- /source/tesk-core/scripts/run: -------------------------------------------------------------------------------- 1 | 2 | # 3 | # Usage: 4 | # 5 | # ./run examples/inputFile.json 6 | # 7 | 8 | jsonFile="$1" 9 | 10 | 11 | require map 12 | 13 | export HOST_BASE_PATH='/home/tfga/workspace/cwl-tes/' 14 | export CONTAINER_BASE_PATH='/transfer' 15 | 16 | echo "HOST_BASE_PATH='$HOST_BASE_PATH'" 17 | echo "CONTAINER_BASE_PATH='$CONTAINER_BASE_PATH'" 18 | echo 19 | 20 | 21 | deleteAll() { 22 | 23 | objType="$1" 24 | 25 | echo "-- Deleting all $objType -----------------------------------" 26 | kubectl delete "$objType" --all 27 | echo 28 | } 29 | 30 | deleteAll pods 31 | deleteAll jobs 32 | deleteAll pvc 33 | deleteAll pv 34 | 35 | echo "-- Creating transfer PV and PVC --------------------------------" 36 | echo 37 | 38 | map kuCreate examples/transferPvc/pv.yaml \ 39 | examples/transferPvc/pvc.yaml 40 | 41 | echo "-- Running -----------------------------------------------------" 42 | echo 43 | 44 | ./taskmaster -f "$jsonFile" -fv 'testing' -d --localKubeConfig 45 | 46 | echo "-- Result ------------------------------------------------------" 47 | echo 48 | 49 | kubectl get pods 50 | 51 | -------------------------------------------------------------------------------- /source/tesk-core/setup.cfg: -------------------------------------------------------------------------------- 1 | [metadata] 2 | description-file=README.md 3 | [aliases] 4 | test=pytest 5 | -------------------------------------------------------------------------------- /source/tesk-core/setup.py: -------------------------------------------------------------------------------- 1 | import codecs 2 | from os import path 3 | from setuptools import setup, find_packages 4 | 5 | HERE = path.abspath(path.dirname(__file__)) 6 | 7 | # Get the long description from the README file 8 | with codecs.open(path.join(HERE, 'README.md'), encoding='utf-8') as f: 9 | LONG_DESC = f.read() 10 | 11 | INSTALL_DEPS = ['kubernetes==9.0.0', 12 | 'requests>=2.20.0', 13 | 'urllib3==1.26.19', 14 | 'boto3==1.16.18', 15 | ] 16 | TEST_DEPS = [ 'pytest', 17 | 'pyfakefs', 18 | 'pytest-mock' 19 | , 'fs', 20 | 'moto', 21 | 'pytest-localftpserver' 22 | ] 23 | 24 | DEV_DEPS = [] 25 | 26 | setup( 27 | name='teskcore', 28 | 29 | # https://pypi.python.org/pypi/setuptools_scm 30 | use_scm_version=True, 31 | 32 | description='TES on Kubernetes', 33 | long_description=LONG_DESC, 34 | long_description_content_type="text/markdown", 35 | 36 | url='https://github.com/EMBL-EBI-TSI/TESK', 37 | 38 | author='Erik van der Bergh', 39 | author_email='evdbergh@ebi.ac.uk', 40 | 41 | license='Apache License 2.0', 42 | 43 | # See https://pypi.python.org/pypi?%3Aaction=list_classifiers 44 | classifiers=[ 45 | # How mature is this project? Common values are 46 | # 3 - Alpha 47 | # 4 - Beta 48 | # 5 - Production/Stable 49 | 'Development Status :: 4 - Beta', 50 | 51 | 'Intended Audience :: System Administrators', 52 | 53 | 'License :: OSI Approved :: Apache Software License', 54 | 55 | 'Programming Language :: Python :: 3', 56 | 'Programming Language :: Python :: 3.7' 57 | ], 58 | 59 | # What does your project relate to? 60 | keywords='tes kubernetes ebi', 61 | 62 | packages = find_packages('src'), 63 | package_dir = {'': 'src'}, 64 | 65 | entry_points={ 66 | 'console_scripts' : [ 67 | 'filer = tesk_core.filer:main', 68 | 'taskmaster = tesk_core.taskmaster:main' 69 | ] 70 | }, 71 | test_suite='tests', 72 | 73 | # List run-time dependencies here. These will be installed by pip when 74 | # your project is installed. For an analysis of "install_requires" vs pip's 75 | # requirements files see: 76 | # https://packaging.python.org/en/latest/requirements.html 77 | install_requires=INSTALL_DEPS, 78 | 79 | setup_requires=['setuptools_scm'], 80 | 81 | tests_require=TEST_DEPS, 82 | 83 | python_requires='>=3.5, <4.0', 84 | 85 | # List additional groups of dependencies here (e.g. development 86 | # dependencies). You can install these using the following syntax, 87 | # for example: 88 | # $ pip install -e .[dev,test] 89 | extras_require={ 90 | 'dev': DEV_DEPS, 91 | 'test': TEST_DEPS 92 | }, 93 | ) 94 | -------------------------------------------------------------------------------- /source/tesk-core/src/tesk_core/README.md: -------------------------------------------------------------------------------- 1 | # taskmaster architecture 2 | 3 | The core flow of the taskmaster is creating a series of Job objects (representation of Kubernetes job) that are run, and polled until they are done. Architecture flow starting from main: 4 | 5 | ![taskmaster architecture](../doc/taskmaster_architecture.png) 6 | 7 | For more details see source comments. 8 | -------------------------------------------------------------------------------- /source/tesk-core/src/tesk_core/Util.py: -------------------------------------------------------------------------------- 1 | import json 2 | 3 | 4 | def pprint(data): 5 | 6 | return json.dumps(data, indent=4) 7 | 8 | -------------------------------------------------------------------------------- /source/tesk-core/src/tesk_core/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/elixir-cloud-aai/TESK/90e2e9bc84c7b47dd1fa548b223e72c885bd4ba6/source/tesk-core/src/tesk_core/__init__.py -------------------------------------------------------------------------------- /source/tesk-core/src/tesk_core/exception.py: -------------------------------------------------------------------------------- 1 | 2 | 3 | class UnknownProtocol(Exception): 4 | pass 5 | 6 | class FileProtocolDisabled(Exception): 7 | pass 8 | 9 | class InvalidHostPath(Exception): 10 | pass 11 | -------------------------------------------------------------------------------- /source/tesk-core/src/tesk_core/filer_class.py: -------------------------------------------------------------------------------- 1 | import json 2 | from tesk_core import path 3 | from tesk_core.path import fileEnabled 4 | 5 | 6 | class Filer: 7 | 8 | def getVolumes(self): return self.spec['spec']['template']['spec']['volumes'] 9 | 10 | def getContainer(self, i): return self.spec['spec']['template']['spec']['containers'][i] 11 | 12 | def getVolumeMounts(self): return self.getContainer(0)['volumeMounts'] 13 | def getEnv(self): return self.getContainer(0)['env'] 14 | def getImagePullPolicy(self): return self.getContainer(0)['imagePullPolicy'] 15 | 16 | 17 | def __init__(self, name, data, filer_name='eu.gcr.io/tes-wes/filer', filer_version='v0.5', pullPolicyAlways = False, json_pvc=None): 18 | self.name = name 19 | self.json_pvc = json_pvc 20 | self.spec = { 21 | "kind": "Job", 22 | "apiVersion": "batch/v1", 23 | "metadata": {"name": name}, 24 | "spec": { 25 | "template": { 26 | "metadata": {"name": "tesk-filer"}, 27 | "spec": { 28 | "containers": [{ 29 | "name": "filer", 30 | "image": "%s:%s" % (filer_name, filer_version), 31 | "args": [], 32 | "env": [], 33 | "volumeMounts": [], 34 | "imagePullPolicy": 'Always' if pullPolicyAlways else 'IfNotPresent' 35 | } 36 | ], 37 | "volumes": [], 38 | "restartPolicy": "Never" 39 | } 40 | } 41 | } 42 | } 43 | 44 | env = self.getEnv() 45 | if json_pvc is None: 46 | env.append({"name": "JSON_INPUT", "value": json.dumps(data)}) 47 | env.append({"name": "HOST_BASE_PATH", "value": path.HOST_BASE_PATH}) 48 | env.append( 49 | {"name": "CONTAINER_BASE_PATH", "value": path.CONTAINER_BASE_PATH}) 50 | 51 | if json_pvc: 52 | self.getVolumeMounts().append({ 53 | "name" : 'jsoninput' 54 | , 'mountPath' : '/jsoninput' 55 | }) 56 | self.getVolumes().append({ 57 | "name" : 'jsoninput' 58 | , "configMap" : { 'name' : json_pvc } 59 | }) 60 | 61 | if fileEnabled(): 62 | self.getVolumeMounts().append({ 63 | 64 | "name" : 'transfer-volume' 65 | , 'mountPath' : path.CONTAINER_BASE_PATH 66 | }) 67 | 68 | self.getVolumes().append({ 69 | 70 | "name" : 'transfer-volume' 71 | , 'persistentVolumeClaim' : { 'claimName' : path.TRANSFER_PVC_NAME } 72 | 73 | }) 74 | 75 | self.add_s3_mount() 76 | 77 | def add_s3_mount(self): 78 | """ Mounts the s3 configuration file. The secret name is hardcoded and 79 | set to 'aws-secret'. 80 | """ 81 | 82 | env = self.getEnv() 83 | env.append({"name": "AWS_CONFIG_FILE", "value": "/aws/config"}) 84 | env.append( 85 | { 86 | "name": "AWS_SHARED_CREDENTIALS_FILE", 87 | "value": "/aws/credentials" 88 | } 89 | ) 90 | 91 | self.getVolumeMounts().append( 92 | { 93 | "name": "s3-conf", 94 | "mountPath": "/aws", 95 | "readOnly": True, 96 | } 97 | ) 98 | self.getVolumes().append( 99 | { 100 | "name": "s3-conf", 101 | "secret": { 102 | "secretName": "aws-secret", 103 | "items": [ 104 | { 105 | "key": "credentials", 106 | "path": "credentials" 107 | }, 108 | { 109 | "key": "config", 110 | "path": "config" 111 | } 112 | ], 113 | "optional": True, 114 | } 115 | } 116 | ) 117 | 118 | def set_ftp(self, user, pw): 119 | env = self.getEnv() 120 | env.append({"name": "TESK_FTP_USERNAME", "value": user}) 121 | env.append({"name": "TESK_FTP_PASSWORD", "value": pw}) 122 | 123 | def set_backoffLimit(self, limit): 124 | """Set a number of retries of a job execution (default value is 6). Use the environment variable 125 | TESK_API_TASKMASTER_ENVIRONMENT_FILER_BACKOFF_LIMIT to explicitly set this value. 126 | 127 | Args: 128 | limit: The number of retries before considering a Job as failed. 129 | """ 130 | self.spec['spec'].update({"backoffLimit": limit}) 131 | 132 | def add_volume_mount(self, pvc): 133 | self.getVolumeMounts().extend(pvc.volume_mounts) 134 | self.getVolumes().append({"name": "task-volume", 135 | "persistentVolumeClaim": { 136 | "claimName": pvc.name}}) 137 | 138 | 139 | def add_netrc_mount(self, netrc_name='netrc'): 140 | ''' 141 | Sets $HOME to an arbitrary location (to prevent its change as a result of runAsUser), currently hardcoded to `/opt/home` 142 | Mounts the secret netrc into that location: $HOME/.netrc. 143 | ''' 144 | 145 | self.getVolumeMounts().append({"name" : 'netrc', 146 | "mountPath" : '/opt/home/.netrc', 147 | "subPath" : ".netrc" 148 | }) 149 | self.getVolumes().append({"name" : "netrc", 150 | "secret" : { 151 | "secretName" : netrc_name, 152 | "defaultMode" : 420, 153 | "items" : [ 154 | { 155 | "key": ".netrc", 156 | "path": ".netrc" 157 | } 158 | ] 159 | } 160 | }) 161 | self.getEnv().append({"name": "HOME", 162 | "value": "/opt/home" 163 | }) 164 | 165 | 166 | def get_spec(self, mode, debug=False): 167 | if self.json_pvc is None: 168 | self.spec['spec']['template']['spec']['containers'][0]['args'] = [ 169 | mode, "$(JSON_INPUT)"] 170 | else: 171 | self.spec['spec']['template']['spec']['containers'][0]['args'] = [ 172 | mode, "/jsoninput/JSON_INPUT.gz"] 173 | 174 | if debug: 175 | self.spec['spec']['template']['spec']['containers'][0][ 176 | 'args'].append( 177 | '-d') 178 | 179 | self.spec['spec']['template']['metadata']['name'] = self.name 180 | return self.spec 181 | -------------------------------------------------------------------------------- /source/tesk-core/src/tesk_core/filer_s3.py: -------------------------------------------------------------------------------- 1 | import sys 2 | import os 3 | import logging 4 | import re 5 | import botocore 6 | import boto3 7 | from tesk_core.transput import Transput, Type 8 | 9 | class S3Transput(Transput): 10 | def __init__(self, path, url, ftype): 11 | Transput.__init__(self, path, url, ftype) 12 | self.bucket, self.file_path = self.get_bucket_name_and_file_path() 13 | self.bucket_obj = None 14 | 15 | def __enter__(self): 16 | client = boto3.resource('s3', endpoint_url=self.extract_endpoint()) 17 | if self.check_if_bucket_exists(client): 18 | sys.exit(1) 19 | self.bucket_obj = client.Bucket(self.bucket) 20 | return self 21 | 22 | def extract_endpoint(self): 23 | return boto3.client('s3').meta.endpoint_url 24 | 25 | def check_if_bucket_exists(self, client): 26 | try: 27 | client.meta.client.head_bucket(Bucket=self.bucket) 28 | except botocore.exceptions.ClientError as e: 29 | # If a client error is thrown, then check that it was a 404 error. 30 | # If it was a 404 error, then the bucket does not exist. 31 | logging.error('Got status code: %s', e.response['Error']['Code']) 32 | if e.response['Error']['Code'] == "404": 33 | logging.error("Failed to fetch Bucket, reason: %s", e.response['Error']['Message']) 34 | return 1 35 | return 0 36 | 37 | def get_bucket_name_and_file_path(self): 38 | """ 39 | If the S3 url is similar to s3://idr-bucket-1/README.txt format 40 | """ 41 | 42 | bucket = self.netloc 43 | file_path = self.url_path[1:] 44 | 45 | return bucket, file_path 46 | 47 | def download_file(self): 48 | logging.debug('Downloading s3 object: "%s" Target: %s', self.bucket + "/" + self.file_path, self.path) 49 | basedir = os.path.dirname(self.path) 50 | os.makedirs(basedir, exist_ok=True) 51 | return self.get_s3_file(self.path, self.file_path) 52 | 53 | def upload_file(self): 54 | logging.debug('Uploading s3 object: "%s" Target: %s', self.path, self.bucket + "/" + self.file_path) 55 | try: 56 | self.bucket_obj.upload_file(Filename=self.path, Key=self.file_path) 57 | except (botocore.exceptions.ClientError, OSError) as err: 58 | logging.error("File upload failed for '%s'", self.bucket + "/" + self.file_path) 59 | logging.error(err) 60 | return 1 61 | return 0 62 | 63 | def upload_dir(self): 64 | logging.debug('Uploading s3 object: "%s" Target: %s', self.path, self.bucket + "/" + self.file_path) 65 | try: 66 | for item in os.listdir(self.path): 67 | path = os.path.join(self.path,item) 68 | if os.path.isdir(path): 69 | file_type = Type.Directory 70 | elif os.path.isfile(path): 71 | file_type = Type.File 72 | else: 73 | # An exception is raised, if the object type is neither file or directory 74 | logging.error("Object is neither file or directory : '%s' ",path) 75 | raise IOError 76 | file_path = os.path.join(self.url, item) 77 | with S3Transput(path, file_path, file_type) as transfer: 78 | if transfer.upload(): 79 | return 1 80 | except OSError as err: 81 | logging.error("File upload failed for '%s'", self.bucket + "/" + self.file_path) 82 | logging.error(err) 83 | return 1 84 | return 0 85 | 86 | def download_dir(self): 87 | logging.debug('Downloading s3 object: "%s" Target: %s', self.bucket + "/" + self.file_path, self.path) 88 | client = boto3.client('s3', endpoint_url=self.extract_endpoint()) 89 | if not self.file_path.endswith('/'): 90 | self.file_path += '/' 91 | objects = client.list_objects_v2(Bucket=self.bucket, Prefix=self.file_path) 92 | 93 | # If the file path does not exists in s3 bucket, 'Contents' key will not be present in objects 94 | if "Contents" not in objects: 95 | logging.error('Got status code: %s', 404) 96 | logging.error("Invalid file path!.") 97 | return 1 98 | 99 | # Looping through the list of objects and downloading them 100 | for obj in objects["Contents"]: 101 | file_name = os.path.basename(obj["Key"]) 102 | dir_name = os.path.dirname(obj["Key"]) 103 | path_to_create = re.sub(r'^' + self.file_path.strip('/').replace('/', '\/') + '', "", dir_name).strip('/') 104 | path_to_create = os.path.join(self.path, path_to_create) 105 | os.makedirs(path_to_create, exist_ok=True) 106 | if self.get_s3_file(os.path.join(path_to_create, file_name), obj["Key"]): 107 | return 1 108 | return 0 109 | 110 | def get_s3_file(self, file_name, key): 111 | try: 112 | self.bucket_obj.download_file(Filename=file_name, Key=key) 113 | except botocore.exceptions.ClientError as err: 114 | logging.error('Got status code: %s', err.response['Error']['Code']) 115 | logging.error(err.response['Error']['Message']) 116 | return 1 117 | return 0 118 | -------------------------------------------------------------------------------- /source/tesk-core/src/tesk_core/job.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import time 3 | from datetime import datetime, timezone 4 | from kubernetes import client, config 5 | from kubernetes.client.rest import ApiException 6 | from tesk_core.Util import pprint 7 | 8 | 9 | logging.basicConfig(format='%(message)s', level=logging.INFO) 10 | class Job: 11 | def __init__(self, body, name='task-job', namespace='default'): 12 | self.name = name 13 | self.namespace = namespace 14 | self.status = 'Initialized' 15 | self.bv1 = client.BatchV1Api() 16 | self.cv1 = client.CoreV1Api() 17 | self.timeout = 240 18 | self.body = body 19 | self.body['metadata']['name'] = self.name 20 | 21 | def run_to_completion(self, poll_interval, check_cancelled, pod_timeout): 22 | 23 | logging.debug("Creating job '{}'...".format(self.name)) 24 | logging.debug(pprint(self.body)) 25 | self.timeout = pod_timeout 26 | try: 27 | self.bv1.create_namespaced_job(self.namespace, self.body) 28 | except ApiException as ex: 29 | if ex.status == 409: 30 | logging.debug(f"Reading existing job: {self.name} ") 31 | self.bv1.read_namespaced_job(self.name, self.namespace) 32 | else: 33 | logging.debug(ex.body) 34 | raise ApiException(ex.status, ex.reason) 35 | is_all_pods_running = False 36 | status, is_all_pods_running = self.get_status(is_all_pods_running) 37 | while status == 'Running': 38 | if check_cancelled(): 39 | self.delete() 40 | return 'Cancelled' 41 | time.sleep(poll_interval) 42 | status, is_all_pods_running = self.get_status(is_all_pods_running) 43 | return status 44 | 45 | 46 | def get_status(self, is_all_pods_runnning): 47 | job = self.bv1.read_namespaced_job(self.name, self.namespace) 48 | try: 49 | if job.status.conditions[0].type == 'Complete' and job.status.conditions[0].status: 50 | self.status = 'Complete' 51 | elif job.status.conditions[0].type == 'Failed' and job.status.conditions[0].status: 52 | self.status = 'Failed' 53 | else: 54 | self.status = 'Error' 55 | except TypeError: # The condition is not initialized, so it is not complete yet, wait for it 56 | self.status = 'Running' 57 | job_duration = 0 58 | if job.status.active and job.status.start_time: 59 | job_duration = (datetime.now(timezone.utc) - job.status.start_time).total_seconds() 60 | if job_duration > self.timeout and not is_all_pods_runnning: 61 | pods = (self.cv1.list_namespaced_pod(self.namespace 62 | , label_selector='job-name={}'.format(self.name))).items 63 | is_all_pods_runnning = True 64 | for pod in pods: 65 | if pod.status.phase == "Pending" and pod.status.start_time: 66 | is_all_pods_runnning = False 67 | delta = (datetime.now(timezone.utc) - pod.status.start_time).total_seconds() 68 | if delta > self.timeout and \ 69 | pod.status.container_statuses[0].state.waiting.reason == "ImagePullBackOff": 70 | logging.info(pod.status.container_statuses[0].state.waiting) 71 | return 'Error', is_all_pods_runnning 72 | 73 | return self.status, is_all_pods_runnning 74 | 75 | def delete(self): 76 | logging.info("Removing failed jobs") 77 | self.bv1.delete_namespaced_job( 78 | self.name, self.namespace, body=client.V1DeleteOptions(propagation_policy="Background")) -------------------------------------------------------------------------------- /source/tesk-core/src/tesk_core/path.py: -------------------------------------------------------------------------------- 1 | import os 2 | from os.path import relpath 3 | from tesk_core.exception import InvalidHostPath 4 | try: 5 | from urllib.parse import urlparse 6 | except ImportError: 7 | from urlparse import urlparse 8 | 9 | 10 | 11 | 12 | def getEnv(varName): 13 | 14 | return os.environ.get(varName) 15 | 16 | 17 | def getPathEnv(varName): 18 | ''' 19 | Gets a path from env var 'varName' and normalizes it 20 | 21 | e.g. removes trailing slashes. 22 | This removes some cases from the rest of the code. 23 | ''' 24 | 25 | varContent = getEnv(varName) 26 | 27 | return os.path.normpath(varContent) if varContent else None 28 | 29 | 30 | HOST_BASE_PATH = getPathEnv('HOST_BASE_PATH') 31 | CONTAINER_BASE_PATH = getPathEnv('CONTAINER_BASE_PATH') 32 | TRANSFER_PVC_NAME = getEnv('TRANSFER_PVC_NAME') 33 | 34 | def fileEnabled(): 35 | 36 | return HOST_BASE_PATH is not None \ 37 | and CONTAINER_BASE_PATH is not None 38 | 39 | 40 | def getPath(url): 41 | 42 | parsed_url = urlparse(url) 43 | 44 | return parsed_url.path 45 | 46 | 47 | def isDescendant(base, path): 48 | ''' 49 | Is 'path' is a descendant of 'base'? 50 | ''' 51 | 52 | return os.path.commonprefix([base, path]) == base 53 | 54 | 55 | def validatePath(path): 56 | 57 | if not isDescendant(HOST_BASE_PATH, path): 58 | 59 | raise InvalidHostPath("'{path}' is not a descendant of 'HOST_BASE_PATH' ({HOST_BASE_PATH})".format( path = path 60 | , HOST_BASE_PATH = HOST_BASE_PATH 61 | )) 62 | 63 | 64 | def containerPath(path): 65 | 66 | validatePath(path) 67 | 68 | relPath = relpath(path, HOST_BASE_PATH) 69 | 70 | return os.path.join(CONTAINER_BASE_PATH, relPath) 71 | 72 | 73 | 74 | -------------------------------------------------------------------------------- /source/tesk-core/src/tesk_core/pvc.py: -------------------------------------------------------------------------------- 1 | from kubernetes import client, config 2 | from kubernetes.client.rest import ApiException 3 | from tesk_core.Util import pprint 4 | import os 5 | import logging 6 | 7 | 8 | class PVC(): 9 | 10 | def __init__(self, name='task-pvc', size_gb=1, namespace='default'): 11 | self.name = name 12 | self.spec = {'apiVersion': 'v1', 13 | 'kind': 'PersistentVolumeClaim', 14 | 'metadata': {'name': name}, 15 | 'spec': { 16 | 'accessModes': ['ReadWriteOnce'], 17 | 'resources': {'requests': {'storage': str(size_gb) + 'Gi'}} 18 | } 19 | } 20 | 21 | self.subpath_idx = 0 22 | self.namespace = namespace 23 | self.cv1 = client.CoreV1Api() 24 | 25 | # The environment variable 'TESK_API_TASKMASTER_ENVIRONMENT_STORAGE_CLASS_NAME' 26 | # can be set to the preferred, non-default, user-defined storageClass 27 | if os.environ.get('STORAGE_CLASS_NAME') is not None: 28 | self.spec['spec'].update({'storageClassName': os.environ.get('STORAGE_CLASS_NAME')}) 29 | 30 | def set_volume_mounts(self, mounts): 31 | self.volume_mounts = mounts 32 | 33 | def get_subpath(self): 34 | subpath = 'dir' + str(self.subpath_idx) 35 | self.subpath_idx += 1 36 | return subpath 37 | 38 | def create(self): 39 | 40 | logging.debug('Creating PVC...') 41 | logging.debug(pprint(self.spec)) 42 | try: 43 | return self.cv1.create_namespaced_persistent_volume_claim(self.namespace, self.spec) 44 | except ApiException as ex: 45 | if ex.status == 409: 46 | logging.debug(f"Reading existing PVC: {self.name}") 47 | return self.cv1.read_namespaced_persistent_volume_claim(self.name, self.namespace) 48 | else: 49 | logging.debug(ex.body) 50 | raise ApiException(ex.status, ex.reason) 51 | 52 | 53 | def delete(self): 54 | cv1 = client.CoreV1Api() 55 | cv1.delete_namespaced_persistent_volume_claim( 56 | self.name, self.namespace, body=client.V1DeleteOptions()) 57 | -------------------------------------------------------------------------------- /source/tesk-core/src/tesk_core/taskmaster.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | import argparse 4 | import json 5 | import os 6 | import re 7 | import sys 8 | import logging 9 | import gzip 10 | from kubernetes import client, config 11 | from tesk_core.job import Job 12 | from tesk_core.pvc import PVC 13 | from tesk_core.filer_class import Filer 14 | 15 | created_jobs = [] 16 | poll_interval = 5 17 | task_volume_basename = 'task-volume' 18 | args = None 19 | logger = None 20 | 21 | def run_executor(executor, namespace, pvc=None): 22 | jobname = executor['metadata']['name'] 23 | spec = executor['spec']['template']['spec'] 24 | 25 | if os.environ.get('EXECUTOR_BACKOFF_LIMIT') is not None: 26 | executor['spec'].update({'backoffLimit': int(os.environ['EXECUTOR_BACKOFF_LIMIT'])}) 27 | 28 | if pvc is not None: 29 | mounts = spec['containers'][0].setdefault('volumeMounts', []) 30 | mounts.extend(pvc.volume_mounts) 31 | volumes = spec.setdefault('volumes', []) 32 | volumes.extend([{'name': task_volume_basename, 'persistentVolumeClaim': { 33 | 'readonly': False, 'claimName': pvc.name}}]) 34 | logger.debug('Created job: ' + jobname) 35 | job = Job(executor, jobname, namespace) 36 | logger.debug('Job spec: ' + str(job.body)) 37 | 38 | global created_jobs 39 | created_jobs.append(job) 40 | 41 | status = job.run_to_completion(poll_interval, check_cancelled,args.pod_timeout) 42 | if status != 'Complete': 43 | if status == 'Error': 44 | job.delete() 45 | exit_cancelled('Got status ' + status) 46 | 47 | # TODO move this code to PVC class 48 | 49 | 50 | def append_mount(volume_mounts, name, path, pvc): 51 | 52 | # Checks all mount paths in volume_mounts if the path given is already in 53 | # there 54 | duplicate = next( 55 | (mount for mount in volume_mounts if mount['mountPath'] == path), 56 | None) 57 | # If not, add mount path 58 | if duplicate is None: 59 | subpath = pvc.get_subpath() 60 | logger.debug(' '.join( 61 | ['appending' + name + 62 | 'at path' + path + 63 | 'with subPath:' + subpath])) 64 | volume_mounts.append( 65 | {'name': name, 'mountPath': path, 'subPath': subpath}) 66 | 67 | 68 | def dirname(iodata): 69 | if iodata['type'] == 'FILE': 70 | # strip filename from path 71 | r = '(.*)/' 72 | dirname = re.match(r, iodata['path']).group(1) 73 | logger.debug('dirname of ' + iodata['path'] + 'is: ' + dirname) 74 | elif iodata['type'] == 'DIRECTORY': 75 | dirname = iodata['path'] 76 | 77 | return dirname 78 | 79 | 80 | def generate_mounts(data, pvc): 81 | volume_mounts = [] 82 | 83 | # gather volumes that need to be mounted, without duplicates 84 | volume_name = task_volume_basename 85 | for volume in data['volumes']: 86 | append_mount(volume_mounts, volume_name, volume, pvc) 87 | 88 | # gather other paths that need to be mounted from inputs/outputs FILE and 89 | # DIRECTORY entries 90 | for aninput in data['inputs']: 91 | dirnm = dirname(aninput) 92 | append_mount(volume_mounts, volume_name, dirnm, pvc) 93 | 94 | for anoutput in data['outputs']: 95 | dirnm = dirname(anoutput) 96 | append_mount(volume_mounts, volume_name, dirnm, pvc) 97 | 98 | return volume_mounts 99 | 100 | 101 | def init_pvc(data, filer): 102 | task_name = data['executors'][0]['metadata']['labels']['taskmaster-name'] 103 | pvc_name = task_name + '-pvc' 104 | pvc_size = data['resources']['disk_gb'] 105 | pvc = PVC(pvc_name, pvc_size, args.namespace) 106 | 107 | mounts = generate_mounts(data, pvc) 108 | logging.debug(mounts) 109 | logging.debug(type(mounts)) 110 | pvc.set_volume_mounts(mounts) 111 | filer.add_volume_mount(pvc) 112 | 113 | pvc.create() 114 | # to global var for cleanup purposes 115 | global created_pvc 116 | created_pvc = pvc 117 | 118 | if os.environ.get('NETRC_SECRET_NAME') is not None: 119 | filer.add_netrc_mount(os.environ.get('NETRC_SECRET_NAME')) 120 | 121 | filerjob = Job( 122 | filer.get_spec('inputs', args.debug), 123 | task_name + '-inputs-filer', 124 | args.namespace) 125 | 126 | global created_jobs 127 | created_jobs.append(filerjob) 128 | # filerjob.run_to_completion(poll_interval) 129 | status = filerjob.run_to_completion(poll_interval, check_cancelled, args.pod_timeout) 130 | if status != 'Complete': 131 | exit_cancelled('Got status ' + status) 132 | 133 | return pvc 134 | 135 | 136 | def run_task(data, filer_name, filer_version, have_json_pvc=False): 137 | task_name = data['executors'][0]['metadata']['labels']['taskmaster-name'] 138 | pvc = None 139 | 140 | if have_json_pvc: 141 | json_pvc = task_name 142 | else: 143 | json_pvc = None 144 | 145 | if data['volumes'] or data['inputs'] or data['outputs']: 146 | 147 | filer = Filer(task_name + '-filer', data, filer_name, filer_version, args.pull_policy_always, json_pvc) 148 | 149 | if os.environ.get('TESK_FTP_USERNAME') is not None: 150 | filer.set_ftp( 151 | os.environ['TESK_FTP_USERNAME'], 152 | os.environ['TESK_FTP_PASSWORD']) 153 | 154 | if os.environ.get('FILER_BACKOFF_LIMIT') is not None: 155 | filer.set_backoffLimit(int(os.environ['FILER_BACKOFF_LIMIT'])) 156 | 157 | pvc = init_pvc(data, filer) 158 | 159 | for executor in data['executors']: 160 | run_executor(executor, args.namespace, pvc) 161 | 162 | # run executors 163 | logging.debug("Finished running executors") 164 | 165 | # upload files and delete pvc 166 | if data['volumes'] or data['inputs'] or data['outputs']: 167 | filerjob = Job( 168 | filer.get_spec('outputs', args.debug), 169 | task_name + '-outputs-filer', 170 | args.namespace) 171 | 172 | global created_jobs 173 | created_jobs.append(filerjob) 174 | 175 | # filerjob.run_to_completion(poll_interval) 176 | status = filerjob.run_to_completion(poll_interval, check_cancelled, args.pod_timeout) 177 | if status != 'Complete': 178 | exit_cancelled('Got status ' + status) 179 | else: 180 | pvc.delete() 181 | 182 | 183 | def newParser(): 184 | 185 | parser = argparse.ArgumentParser(description='TaskMaster main module') 186 | group = parser.add_mutually_exclusive_group(required=True) 187 | group.add_argument( 188 | 'json', 189 | help='string containing json TES request, required if -f is not given', 190 | nargs='?') 191 | group.add_argument( 192 | '-f', 193 | '--file', 194 | help='TES request as a file or \'-\' for stdin, required if json is not given') 195 | 196 | parser.add_argument( 197 | '-p', 198 | '--poll-interval', 199 | help='Job polling interval', 200 | default=5) 201 | parser.add_argument( 202 | '-pt', 203 | '--pod-timeout', 204 | type=int, 205 | help='Pod creation timeout', 206 | default=240) 207 | parser.add_argument( 208 | '-fn', 209 | '--filer-name', 210 | help='Filer image version', 211 | default='eu.gcr.io/tes-wes/filer') 212 | parser.add_argument( 213 | '-fv', 214 | '--filer-version', 215 | help='Filer image version', 216 | default='v0.1.9') 217 | parser.add_argument( 218 | '-n', 219 | '--namespace', 220 | help='Kubernetes namespace to run in', 221 | default='default') 222 | parser.add_argument( 223 | '-s', 224 | '--state-file', 225 | help='State file for state.py script', 226 | default='/tmp/.teskstate') 227 | parser.add_argument( 228 | '-d', 229 | '--debug', 230 | help='Set debug mode', 231 | action='store_true') 232 | parser.add_argument( 233 | '--localKubeConfig', 234 | help='Read k8s configuration from localhost', 235 | action='store_true') 236 | parser.add_argument( 237 | '--pull-policy-always', 238 | help="set imagePullPolicy = 'Always'", 239 | action='store_true') 240 | 241 | 242 | return parser 243 | 244 | 245 | def newLogger(loglevel): 246 | logging.basicConfig( 247 | format='%(asctime)s %(levelname)s: %(message)s', 248 | datefmt='%m/%d/%Y %I:%M:%S', 249 | level=loglevel) 250 | logging.getLogger('kubernetes.client').setLevel(logging.CRITICAL) 251 | logger = logging.getLogger(__name__) 252 | 253 | return logger 254 | 255 | 256 | 257 | def main(): 258 | have_json_pvc = False 259 | 260 | parser = newParser() 261 | global args 262 | 263 | args = parser.parse_args() 264 | 265 | poll_interval = args.poll_interval 266 | 267 | loglevel = logging.ERROR 268 | if args.debug: 269 | loglevel = logging.DEBUG 270 | 271 | global logger 272 | logger = newLogger(loglevel) 273 | logger.debug('Starting taskmaster') 274 | 275 | # Get input JSON 276 | if args.file is None: 277 | data = json.loads(args.json) 278 | elif args.file == '-': 279 | data = json.load(sys.stdin) 280 | else: 281 | if args.file.endswith('.gz'): 282 | with gzip.open(args.file, 'rb') as fh: 283 | data = json.loads(fh.read()) 284 | have_json_pvc = True 285 | else: 286 | with open(args.file) as fh: 287 | data = json.load(fh) 288 | 289 | # Load kubernetes config file 290 | if args.localKubeConfig: 291 | config.load_kube_config() 292 | else: 293 | config.load_incluster_config() 294 | 295 | global created_pvc 296 | created_pvc = None 297 | 298 | # Check if we're cancelled during init 299 | if check_cancelled(): 300 | exit_cancelled('Cancelled during init') 301 | 302 | run_task(data, args.filer_name, args.filer_version, have_json_pvc) 303 | 304 | 305 | def clean_on_interrupt(): 306 | logger.debug('Caught interrupt signal, deleting jobs and pvc') 307 | 308 | for job in created_jobs: 309 | job.delete() 310 | 311 | 312 | 313 | def exit_cancelled(reason='Unknown reason'): 314 | logger.error('Cancelling taskmaster: ' + reason) 315 | sys.exit(0) 316 | 317 | 318 | def check_cancelled(): 319 | 320 | labelInfoFile = '/podinfo/labels' 321 | 322 | if not os.path.exists(labelInfoFile): 323 | return False 324 | 325 | with open(labelInfoFile) as fh: 326 | for line in fh.readlines(): 327 | name, label = line.split('=') 328 | logging.debug('Got label: ' + label) 329 | if label == '"Cancelled"': 330 | return True 331 | 332 | return False 333 | 334 | 335 | if __name__ == "__main__": 336 | try: 337 | main() 338 | except KeyboardInterrupt: 339 | clean_on_interrupt() 340 | -------------------------------------------------------------------------------- /source/tesk-core/src/tesk_core/transput.py: -------------------------------------------------------------------------------- 1 | import enum 2 | import os 3 | import netrc 4 | import logging 5 | try: 6 | from urllib.parse import urlparse 7 | except ImportError: 8 | from urlparse import urlparse 9 | 10 | 11 | @enum.unique 12 | class Type(enum.Enum): 13 | File = 'FILE' 14 | Directory = 'DIRECTORY' 15 | 16 | 17 | class Transput: 18 | def __init__(self, path, url, ftype): 19 | self.path = path 20 | self.url = url 21 | self.ftype = ftype 22 | 23 | parsed_url = urlparse(url) 24 | self.netloc = parsed_url.netloc 25 | self.url_path = parsed_url.path 26 | self.netrc_file = None 27 | try: 28 | netrc_path = os.path.join(os.environ['HOME'], '.netrc') 29 | except KeyError: 30 | netrc_path = '/.netrc' 31 | try: 32 | self.netrc_file = netrc.netrc(netrc_path) 33 | except IOError as fnfe: 34 | logging.error(fnfe) 35 | except netrc.NetrcParseError as err: 36 | logging.error('netrc.NetrcParseError') 37 | logging.error(err) 38 | except Exception as er: 39 | logging.error(er) 40 | 41 | def upload(self): 42 | logging.debug('%s uploading %s %s', self.__class__.__name__, 43 | self.ftype, self.url) 44 | if self.ftype == Type.File: 45 | return self.upload_file() 46 | if self.ftype == Type.Directory: 47 | return self.upload_dir() 48 | return 1 49 | 50 | def download(self): 51 | logging.debug('%s downloading %s %s', self.__class__.__name__, 52 | self.ftype, self.url) 53 | if self.ftype == Type.File: 54 | return self.download_file() 55 | if self.ftype == Type.Directory: 56 | return self.download_dir() 57 | return 1 58 | 59 | def delete(self): 60 | pass 61 | 62 | def download_file(self): 63 | raise NotImplementedError() 64 | 65 | def download_dir(self): 66 | raise NotImplementedError() 67 | 68 | def upload_file(self): 69 | raise NotImplementedError() 70 | 71 | def upload_dir(self): 72 | raise NotImplementedError() 73 | 74 | # make it compatible with contexts (with keyword) 75 | def __enter__(self): 76 | return self 77 | 78 | def __exit__(self, error_type, error_value, traceback): 79 | self.delete() 80 | # Swallow all exceptions since the filer mostly works with error codes 81 | return False 82 | -------------------------------------------------------------------------------- /source/tesk-core/taskmaster: -------------------------------------------------------------------------------- 1 | 2 | 3 | PYTHONPATH="src" python src/tesk_core/taskmaster.py "$@" 4 | -------------------------------------------------------------------------------- /source/tesk-core/tests/FilerClassTest.py: -------------------------------------------------------------------------------- 1 | # encoding: utf-8 2 | 3 | import unittest 4 | import os 5 | from tesk_core.filer_class import Filer 6 | from tesk_core import path 7 | from tesk_core.Util import pprint 8 | 9 | try: 10 | from unittest.mock import patch # Python 3 @UnresolvedImport 11 | except: 12 | from mock import patch 13 | 14 | 15 | 16 | 17 | @patch('tesk_core.path.HOST_BASE_PATH' , '/home/tfga/workspace/cwl-tes') 18 | @patch('tesk_core.path.CONTAINER_BASE_PATH' , '/transfer') 19 | @patch('tesk_core.path.TRANSFER_PVC_NAME' , 'transfer-pvc') 20 | @patch.dict(os.environ, 21 | { 22 | "AWS_SHARED_CREDENTIALS_FILE": "/aws/credentials", 23 | "AWS_CONFIG_FILE": "/aws/config", 24 | }) 25 | class FilerClassTest_env(unittest.TestCase): 26 | 27 | def test_env_vars(self): 28 | 29 | f = Filer('name', {'a': 1}) 30 | f.set_backoffLimit(10) 31 | 32 | pprint(f.spec) 33 | 34 | self.assertEquals(f.getEnv(), [ 35 | 36 | { 'name': 'JSON_INPUT' , 'value': '{"a": 1}' } 37 | ,{ 'name': 'HOST_BASE_PATH' , 'value': '/home/tfga/workspace/cwl-tes' } 38 | ,{ 'name': 'CONTAINER_BASE_PATH' , 'value': '/transfer' } 39 | ,{"name": "AWS_CONFIG_FILE", "value": "/aws/config"} 40 | ,{"name": "AWS_SHARED_CREDENTIALS_FILE", "value": "/aws/credentials"}, 41 | ]) 42 | self.assertEquals(f.spec['spec']['backoffLimit'], 10) 43 | 44 | 45 | def test_mounts(self): 46 | ''' 47 | kind: Pod 48 | apiVersion: v1 49 | metadata: 50 | name: tfga-pod 51 | spec: 52 | containers: 53 | - name: tfga-container 54 | image: eu.gcr.io/tes-wes/filer:testing 55 | volumeMounts: 56 | - mountPath: /transfer 57 | name: transfer-volume 58 | volumes: 59 | - name: transfer-volume 60 | hostPath: 61 | path: /transferAtNode 62 | # persistentVolumeClaim: 63 | # claimName: task-pv-claim 64 | ''' 65 | 66 | f = Filer('name', {'a': 1}) 67 | 68 | pprint(f.spec) 69 | 70 | pprint(f.getVolumeMounts()) 71 | 72 | self.assertEquals(f.getVolumeMounts(), [ 73 | 74 | { "name" : 'transfer-volume' 75 | , 'mountPath' : path.CONTAINER_BASE_PATH, 76 | }, 77 | {'mountPath': '/aws', 'name': 's3-conf', 'readOnly': True} 78 | ]) 79 | 80 | self.assertEquals(f.getVolumes(), [ 81 | 82 | { "name" : 'transfer-volume' 83 | , 'persistentVolumeClaim' : { 'claimName' : 'transfer-pvc' } 84 | }, 85 | { 86 | "name": "s3-conf", 87 | "secret": { 88 | "secretName": "aws-secret", 89 | "items": [ 90 | { 91 | "key": "credentials", 92 | "path": "credentials" 93 | }, 94 | { 95 | "key": "config", 96 | "path": "config" 97 | } 98 | ], 99 | "optional": True, 100 | } 101 | } 102 | ]) 103 | 104 | 105 | class FilerClassTest_no_env(unittest.TestCase): 106 | 107 | def test_mounts_file_disabled(self): 108 | 109 | f = Filer('name', {'a': 1}) 110 | 111 | pprint(f.spec) 112 | 113 | pprint(f.getVolumeMounts()) 114 | 115 | self.assertEquals(f.getVolumeMounts() , [ 116 | {'mountPath': '/aws', 'name': 's3-conf', 'readOnly': True} 117 | ]) 118 | self.assertEquals(f.getVolumes() , [ 119 | { 120 | "name": "s3-conf", 121 | "secret": { 122 | "secretName": "aws-secret", 123 | "items": [ 124 | { 125 | "key": "credentials", 126 | "path": "credentials" 127 | }, 128 | { 129 | "key": "config", 130 | "path": "config" 131 | } 132 | ], 133 | "optional": True, 134 | } 135 | } 136 | ]) 137 | 138 | 139 | def test_image_pull_policy(self): 140 | 141 | f = Filer('name', {'a': 1}) 142 | self.assertEquals(f.getImagePullPolicy() , 'IfNotPresent') 143 | 144 | f = Filer('name', {'a': 1}, pullPolicyAlways = True) 145 | self.assertEquals(f.getImagePullPolicy() , 'Always') 146 | 147 | 148 | 149 | 150 | 151 | 152 | 153 | if __name__ == "__main__": 154 | #import sys;sys.argv = ['', 'Test.testName'] 155 | unittest.main() 156 | -------------------------------------------------------------------------------- /source/tesk-core/tests/TaskMasterTest.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | from tesk_core.taskmaster import newParser, run_task, newLogger 3 | from argparse import Namespace 4 | import json 5 | import logging 6 | try: 7 | from unittest.mock import patch # Python 3 @UnresolvedImport 8 | except: 9 | from mock import patch 10 | 11 | 12 | 13 | def pvcCreateMock(self): print '[mock] Creating PVC...' 14 | def pvcDeleteMock(self): print '[mock] Deleting PVC...' 15 | 16 | def jobRunToCompletionMock(job, b, c): 17 | 18 | print "[mock] Creating job '{}'...".format(job.name) 19 | 20 | return 'Complete' 21 | 22 | 23 | class ParserTest(unittest.TestCase): 24 | 25 | 26 | def test_defaults(self): 27 | 28 | parser = newParser() 29 | 30 | args = parser.parse_args(["json"]) 31 | 32 | print(args) 33 | 34 | self.assertEquals( args 35 | , Namespace( debug=False, file=None, filer_version='v0.1.9', json='json', namespace='default', poll_interval=5, state_file='/tmp/.teskstate' 36 | , localKubeConfig=False 37 | , pull_policy_always=False 38 | ) 39 | ) 40 | 41 | 42 | def test_localKubeConfig(self): 43 | 44 | parser = newParser() 45 | 46 | args = parser.parse_args(['json', '--localKubeConfig']) 47 | 48 | print(args) 49 | 50 | self.assertEquals( args 51 | , Namespace( debug=False, file=None, filer_version='v0.1.9', json='json', namespace='default', poll_interval=5, state_file='/tmp/.teskstate' 52 | , localKubeConfig=True 53 | , pull_policy_always=False 54 | ) 55 | ) 56 | 57 | 58 | def test_pullPolicyAlways(self): 59 | 60 | parser = newParser() 61 | 62 | self.assertEquals( parser.parse_args(['json' ]).pull_policy_always, False ) 63 | self.assertEquals( parser.parse_args(['json', '--pull-policy-always']).pull_policy_always, True ) 64 | 65 | 66 | 67 | @patch('tesk_core.taskmaster.args' , Namespace(debug=True, namespace='default', pull_policy_always=True)) 68 | @patch('tesk_core.taskmaster.logger' , newLogger(logging.DEBUG)) 69 | @patch('tesk_core.taskmaster.PVC.create' , pvcCreateMock) 70 | @patch('tesk_core.taskmaster.PVC.delete' , pvcDeleteMock) 71 | @patch('tesk_core.taskmaster.Job.run_to_completion' , jobRunToCompletionMock) 72 | def test_run_task(self): 73 | 74 | with open('tests/resources/inputFile.json') as fh: 75 | data = json.load(fh) 76 | 77 | run_task(data, 'filer_version') 78 | 79 | 80 | 81 | if __name__ == "__main__": 82 | #import sys;sys.argv = ['', 'Test.testName'] 83 | unittest.main() -------------------------------------------------------------------------------- /source/tesk-core/tests/assertThrows.py: -------------------------------------------------------------------------------- 1 | 2 | 3 | class AssertThrowsMixin(object): 4 | 5 | def assertThrows(self, func, exceptionClass, errorMessage = None): 6 | 7 | with self.assertRaises(exceptionClass) as cm: 8 | 9 | func() 10 | 11 | if errorMessage: 12 | 13 | self.assertEqual(str(cm.exception), errorMessage) 14 | 15 | 16 | -------------------------------------------------------------------------------- /source/tesk-core/tests/resources/copyDirTest/src/3.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/elixir-cloud-aai/TESK/90e2e9bc84c7b47dd1fa548b223e72c885bd4ba6/source/tesk-core/tests/resources/copyDirTest/src/3.txt -------------------------------------------------------------------------------- /source/tesk-core/tests/resources/copyDirTest/src/a/1.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/elixir-cloud-aai/TESK/90e2e9bc84c7b47dd1fa548b223e72c885bd4ba6/source/tesk-core/tests/resources/copyDirTest/src/a/1.txt -------------------------------------------------------------------------------- /source/tesk-core/tests/resources/copyDirTest/src/a/2.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/elixir-cloud-aai/TESK/90e2e9bc84c7b47dd1fa548b223e72c885bd4ba6/source/tesk-core/tests/resources/copyDirTest/src/a/2.txt -------------------------------------------------------------------------------- /source/tesk-core/tests/resources/inputFile.json: -------------------------------------------------------------------------------- 1 | { 2 | "outputs": [ 3 | ], 4 | "inputs": [ 5 | 6 | { 7 | "name": "input1", 8 | "description": "Example", 9 | "url": "file:///home/tfga/workspace/cwl-tes/README.md", 10 | "path": "/some/volume/input.txt", 11 | "type": "FILE" 12 | } 13 | ], 14 | "volumes": [ 15 | ], 16 | "executors": [ 17 | { 18 | "apiVersion": "batch/v1", 19 | "kind": "Job", 20 | "metadata": { 21 | "annotations": { 22 | }, 23 | "labels": { 24 | "job-type": "executor", 25 | "taskmaster-name": "task-1000", 26 | "executor-no": "0", 27 | "creator-user-id": "test-user-id" 28 | }, 29 | "name": "task-1000-ex-00" 30 | }, 31 | "spec": { 32 | "template": { 33 | "metadata": { 34 | "name": "task-1000-ex-00" 35 | }, 36 | "spec": { 37 | "containers": [ 38 | { 39 | "command": [ 40 | "cat", 41 | "/some/volume/input.txt" 42 | ], 43 | "image": "ubuntu", 44 | "name": "task-1000-ex-00", 45 | "resources": { 46 | } 47 | } 48 | ], 49 | "restartPolicy": "Never" 50 | } 51 | } 52 | } 53 | } 54 | ], 55 | "resources": { 56 | "disk_gb": 0.1 57 | } 58 | } -------------------------------------------------------------------------------- /source/tesk-core/tests/resources/test_config: -------------------------------------------------------------------------------- 1 | [default] 2 | endpoint_url=http://foo.bar 3 | 4 | [other_profile] 5 | endpoint_url=http://other.endpoint 6 | -------------------------------------------------------------------------------- /source/tesk-core/tests/test_filer.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | import logging 3 | import os 4 | from tesk_core.filer import newTransput, FTPTransput, HTTPTransput, FileTransput,\ 5 | process_file, logConfig, getPath, copyDir, copyFile, ftp_check_directory,\ 6 | subfolders_in 7 | from tesk_core.exception import UnknownProtocol, InvalidHostPath,\ 8 | FileProtocolDisabled 9 | from tesk_core.path import containerPath 10 | from tesk_core.filer_s3 import S3Transput 11 | from assertThrows import AssertThrowsMixin 12 | from fs.opener import open_fs 13 | from io import StringIO 14 | from unittest.mock import patch 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | def getTree(rootDir): 23 | strio = StringIO() 24 | with open_fs(rootDir) as dst1_fs: 25 | dst1_fs.tree(file=strio) 26 | treeTxt = strio.getvalue() 27 | strio.close() 28 | return treeTxt 29 | 30 | 31 | def stripLines(txt): 32 | return '\n'.join([line.strip() for line in txt.splitlines()[1:]]) 33 | 34 | 35 | @patch('tesk_core.path.HOST_BASE_PATH', '/home/tfga/workspace/cwl-tes') 36 | @patch('tesk_core.path.CONTAINER_BASE_PATH', '/transfer') 37 | class FilerTest(unittest.TestCase, AssertThrowsMixin): 38 | 39 | @classmethod 40 | def setUpClass(cls): 41 | logConfig(logging.DEBUG) # Doesn't work... 42 | 43 | @patch('tesk_core.filer.copyDir') 44 | @patch('tesk_core.filer.shutil.copy') 45 | def test_download_file(self, copyMock, copyDirMock): 46 | filedata = { 47 | "url": "file:///home/tfga/workspace/cwl-tes/tmphrtip1o8/md5", 48 | "path": "/var/lib/cwl/stgda974802-fa81-4f0b-8fe4-341d5655af4b/md5", 49 | 50 | "type": "FILE", # File = 'FILE' 51 | # Directory = 'DIRECTORY' 52 | 53 | "name": "md5", 54 | "description": "cwl_input:md5" 55 | } 56 | 57 | process_file('inputs', filedata) 58 | 59 | copyDirMock.assert_not_called() 60 | 61 | copyMock.assert_called_once_with('/transfer/tmphrtip1o8/md5', 62 | '/var/lib/cwl/stgda974802-fa81-4f0b-' 63 | '8fe4-341d5655af4b/md5') 64 | 65 | @patch('tesk_core.filer.copyDir') 66 | @patch('tesk_core.filer.shutil.copy') 67 | def test_download_dir(self, copyMock, copyDirMock): 68 | filedata = { 69 | "url": "file:///home/tfga/workspace/cwl-tes/tmphrtip1o8/", 70 | "path": "/TclSZU", 71 | "type": "DIRECTORY", 72 | "name": "workdir" 73 | } 74 | 75 | process_file('inputs', filedata) 76 | 77 | copyMock.assert_not_called() 78 | 79 | copyDirMock.assert_called_once_with('/transfer/tmphrtip1o8', '/TclSZU') 80 | 81 | @patch('tesk_core.filer.copyDir') 82 | @patch('tesk_core.filer.shutil.copy') 83 | def test_upload_dir(self, copyMock, copyDirMock): 84 | filedata = { 85 | "url": "file:///home/tfga/workspace/cwl-tes/tmphrtip1o8/", 86 | "path": "/TclSZU", 87 | "type": "DIRECTORY", 88 | "name": "workdir" 89 | } 90 | 91 | process_file('outputs', filedata) 92 | 93 | copyMock.assert_not_called() 94 | 95 | copyDirMock.assert_called_once_with('/TclSZU', '/transfer/tmphrtip1o8') 96 | 97 | @patch('tesk_core.filer.copyDir') 98 | @patch('tesk_core.filer.copyFile') 99 | def test_upload_file(self, copyFileMock, copyDirMock): 100 | 101 | filedata = { 102 | "url": "file:///home/tfga/workspace/cwl-tes/tmphrtip1o8/md5", 103 | "path": "/TclSZU/md5", 104 | "type": "FILE", 105 | "name": "stdout" 106 | } 107 | 108 | process_file('outputs', filedata) 109 | 110 | copyDirMock.assert_not_called() 111 | 112 | copyFileMock.assert_called_once_with( '/TclSZU/md5' 113 | , '/transfer/tmphrtip1o8/md5') 114 | 115 | 116 | @patch('tesk_core.filer.copyDir') 117 | @patch('tesk_core.filer.copyFile') 118 | def test_upload_file_glob(self, copyFileMock, copyDirMock): 119 | 120 | filedata = { 121 | "url": "file:///home/tfga/workspace/cwl-tes/tmphrtip1o8/md5*", 122 | "path": "/TclSZU/md5*", 123 | "type": "FILE", 124 | "name": "stdout" 125 | } 126 | 127 | process_file('outputs', filedata) 128 | 129 | copyDirMock.assert_not_called() 130 | 131 | copyFileMock.assert_called_once_with( '/TclSZU/md5*' 132 | , '/transfer/tmphrtip1o8/md5*') 133 | 134 | 135 | def test_copyDir(self): 136 | def rmDir(d): 137 | os.system('rm -r {}'.format(d)) 138 | 139 | baseDir = 'tests/resources/copyDirTest/' 140 | src = os.path.join(baseDir, 'src') 141 | dst1 = os.path.join(baseDir, 'dst1') 142 | dst2 = os.path.join(baseDir, 'dst2') 143 | 144 | rmDir(dst1) 145 | rmDir(dst2) 146 | 147 | self.assertTrue(os.path.exists(src)) # src should exist 148 | self.assertFalse(os.path.exists(dst1)) # dst1 shouldn't 149 | self.assertFalse(os.path.exists(dst2)) # dst2 shouldn't 150 | 151 | # Copying to existing dst --------------------------------------------- 152 | # Let's create dst1 153 | os.mkdir(dst1) 154 | self.assertTrue(os.path.exists(dst1)) # Now dst1 should exist 155 | 156 | # Let's try to copy 157 | copyDir(src, dst1) 158 | 159 | 160 | self.assertEqual(getTree(dst1), 161 | stripLines(''' 162 | |-- a 163 | | |-- 1.txt 164 | | `-- 2.txt 165 | `-- 3.txt 166 | ''' 167 | ) 168 | ) 169 | 170 | # Copying to non-existing dst ----------------------------------------- 171 | self.assertFalse(os.path.exists(dst2)) # dst2 should not exist 172 | 173 | # Let's try to copy 174 | copyDir(src, dst2) 175 | 176 | self.assertEqual(getTree(dst2), 177 | stripLines(''' 178 | |-- a 179 | | |-- 1.txt 180 | | `-- 2.txt 181 | `-- 3.txt 182 | ''' 183 | ) 184 | ) 185 | 186 | def test_getPath(self): 187 | 188 | self.assertEqual( getPath('file:///home/tfga/workspace/cwl-tes/tmphrtip1o8/md5') 189 | , '/home/tfga/workspace/cwl-tes/tmphrtip1o8/md5') 190 | 191 | def test_getPathNoScheme(self): 192 | 193 | self.assertEquals( getPath('/home/tfga/workspace/cwl-tes/tmphrtip1o8/md5') 194 | , '/home/tfga/workspace/cwl-tes/tmphrtip1o8/md5') 195 | 196 | self.assertEqual( containerPath('/home/tfga/workspace/cwl-tes/tmphrtip1o8/md5') 197 | , '/transfer/tmphrtip1o8/md5') 198 | 199 | def test_containerPath(self): 200 | self.assertEqual( 201 | containerPath('/home/tfga/workspace/cwl-tes/tmphrtip1o8/md5'), 202 | '/transfer/tmphrtip1o8/md5') 203 | 204 | # What happens if 'path' is not a descendant of HOST_BASE_PATH? 205 | self.assertThrows(lambda: containerPath('/someOtherFolder'), 206 | InvalidHostPath, 207 | "'/someOtherFolder' is not a descendant of " 208 | "'HOST_BASE_PATH' (/home/tfga/workspace/cwl-tes)" 209 | ) 210 | 211 | def test_newTransput(self): 212 | self.assertEqual(newTransput('ftp', 'test.com'), FTPTransput) 213 | self.assertEqual(newTransput('http', 'test.com'), HTTPTransput) 214 | self.assertEqual(newTransput('https', 'test.com'), HTTPTransput) 215 | self.assertEqual(newTransput('file', '/home/tfga/workspace/'), FileTransput) 216 | self.assertEqual(newTransput('s3', '/home/tfga/workspace/'), S3Transput) 217 | self.assertEqual(newTransput('http', 's3.aws.com'), HTTPTransput) 218 | 219 | self.assertThrows(lambda: newTransput('svn', 'example.com') 220 | , UnknownProtocol 221 | , "Unknown protocol: 'svn'" 222 | ) 223 | 224 | @patch('ftplib.FTP') 225 | def test_ftp_check_directory(self, conn): 226 | """ Ensure that when the path provided is an existing directory, the 227 | return value is 0.""" 228 | path = os.path.curdir 229 | self.assertEqual(ftp_check_directory(conn, path), 0) 230 | 231 | def test_subfolders_in(self): 232 | """ Ensure the all the subfolders of a path are properly returned.""" 233 | path = "/this/is/a/path" 234 | subfldrs = ['/this', '/this/is', '/this/is/a', '/this/is/a/path'] 235 | self.assertEqual(subfolders_in(path), subfldrs) 236 | 237 | 238 | 239 | class FilerTest_no_env(unittest.TestCase, AssertThrowsMixin): 240 | 241 | def test_newTransput_file_disabled(self): 242 | self.assertThrows( lambda: newTransput('file','/home/user/test') 243 | , FileProtocolDisabled 244 | , "'file:' protocol disabled\n" 245 | "To enable it, both 'HOST_BASE_PATH' and 'CONTAINER_BASE_PATH' environment variables must be defined." 246 | ) 247 | 248 | 249 | if __name__ == "__main__": 250 | # import sys;sys.argv = ['', 'Test.testName'] 251 | unittest.main() 252 | -------------------------------------------------------------------------------- /source/tesk-core/tests/test_filer_ftp_pytest.py: -------------------------------------------------------------------------------- 1 | """ Tests for 'filer.py' FTP functionalities using 'pytest'.""" 2 | 3 | from unittest import mock 4 | import ftplib 5 | import os 6 | 7 | from tesk_core.filer import ( 8 | FTPTransput, 9 | Type, 10 | ftp_login, 11 | ftp_upload_file, 12 | ftp_download_file, 13 | ftp_check_directory, 14 | ftp_make_dirs 15 | ) 16 | 17 | 18 | def test_ftp_login(mocker): 19 | """ Ensure ftp_login detects ftp credentials and properly calls 20 | ftplib.FTP.login.""" 21 | 22 | conn = mocker.patch('ftplib.FTP') 23 | mock_login = mocker.patch('ftplib.FTP.login') 24 | with mock.patch.dict( 25 | 'os.environ', 26 | { 27 | 'TESK_FTP_USERNAME': 'test', 28 | 'TESK_FTP_PASSWORD': 'test_pass', 29 | } 30 | ): 31 | ftp_login(conn, None, None) 32 | mock_login.assert_called_with('test', 'test_pass') 33 | 34 | 35 | def test_ftp_upload_file_error(mocker, caplog): 36 | """ Ensure that upon upload error, ftp_upload_file behaves correctly.""" 37 | 38 | conn = mocker.patch('ftplib.FTP') 39 | mocker.patch('ftplib.FTP.storbinary', side_effect=ftplib.error_reply) 40 | assert 1 == ftp_upload_file(conn, 41 | 'tests/test_filer.py', 42 | '/home/tesk/test_copy.py') 43 | assert 'Unable to upload file' in caplog.text 44 | 45 | 46 | def test_ftp_download_file_error(mocker, caplog): 47 | """ Ensure that upon download error, ftp_download_file behaves correctly. 48 | """ 49 | 50 | conn = mocker.patch('ftplib.FTP') 51 | mocker.patch('ftplib.FTP.retrbinary', side_effect=ftplib.error_perm) 52 | with mock.patch('builtins.open', mock.mock_open(), create=False) as m: 53 | assert 1 == ftp_download_file(conn, 54 | 'test_filer_ftp_pytest.py', 55 | 'test_copy.py') 56 | assert 'Unable to download file' in caplog.text 57 | 58 | 59 | def test_ftp_download_file_success(mocker, caplog): 60 | """ Ensure that upon successful download, the local destination file has 61 | been created.""" 62 | 63 | conn = mocker.patch('ftplib.FTP') 64 | mock_retrbin = mocker.patch('ftplib.FTP.retrbinary') 65 | with mock.patch('builtins.open', mock.mock_open(), create=False) as m: 66 | assert 0 == ftp_download_file(conn, 67 | 'test_filer_ftp_pytest.py', 68 | 'test_copy.py') 69 | 70 | mock_retrbin.assert_called_with( 71 | "RETR " + "test_filer_ftp_pytest.py", 72 | mock.ANY 73 | ) 74 | 75 | m.assert_called_with('test_copy.py', 'w+b') 76 | 77 | # Since we want to avoid file creation in testing and we're using 78 | # 'create=False', we cannot check whether a file exists or not (but 79 | # it's not really necessary since we can assert that the necessary 80 | # functions have been invoked. 81 | # assert os.path.exists('test_copy.py') 82 | 83 | 84 | def test_ftp_upload_dir(mocker, fs, ftpserver): 85 | """ Check whether the upload of a directory through FTP completes 86 | successfully. """ 87 | 88 | # Fake local nested directories with files 89 | fs.create_dir('dir1') 90 | fs.create_dir('dir1/dir2') 91 | fs.create_file('dir1/file1', contents="this is random") 92 | fs.create_file('dir1/dir2/file2', contents="not really") 93 | fs.create_file('dir1/dir2/file4.txt', contents="took me a while") 94 | 95 | login_dict = ftpserver.get_login_data() 96 | 97 | conn = ftplib.FTP() 98 | 99 | mocker.patch('ftplib.FTP.connect', 100 | side_effect=conn.connect( 101 | host=login_dict['host'], 102 | port=login_dict['port'] 103 | ) 104 | ) 105 | mocker.patch( 106 | 'ftplib.FTP.login', 107 | side_effect=conn.login(login_dict['user'], login_dict['passwd']) 108 | ) 109 | mocker.patch('ftplib.FTP.pwd', side_effect=conn.pwd) 110 | mocker.patch('ftplib.FTP.cwd', side_effect=conn.cwd) 111 | mocker.patch('ftplib.FTP.mkd', side_effect=conn.mkd) 112 | mock_storbinary = mocker.patch('ftplib.FTP.storbinary') 113 | 114 | ftp_obj = FTPTransput( 115 | "dir1", 116 | "ftp://" + login_dict['host'] + "/dir1", 117 | Type.Directory, 118 | ftp_conn=conn 119 | ) 120 | 121 | ftp_obj.upload_dir() 122 | 123 | # We use mock.ANY since the 2nd argument of the 'ftplib.FTP.storbinary' is 124 | # a file object and we can't have the same between the original and the 125 | # mock calls 126 | assert sorted(mock_storbinary.mock_calls) == sorted([ 127 | mock.call('STOR /' + '/dir1/file1', mock.ANY), 128 | mock.call('STOR /' + '/dir1/dir2/file2', mock.ANY), 129 | mock.call('STOR /' + '/dir1/dir2/file4.txt', mock.ANY) 130 | ]) 131 | 132 | 133 | def test_ftp_download_dir(mocker, tmpdir, tmp_path, ftpserver): 134 | """ Check whether the download of a directory through FTP completes 135 | successfully. """ 136 | 137 | # Temporary nested directories with files 138 | file1 = tmpdir.mkdir("dir1").join("file1") 139 | file1.write("this is random") 140 | file2 = tmpdir.mkdir("dir1/dir2").join("file2") 141 | file2.write('not really') 142 | file3 = tmpdir.join('dir1/dir2/file3') 143 | file3.write('took me a while') 144 | 145 | # Temporary folder for download 146 | tmpdir.mkdir('downloads') 147 | 148 | # Populate the server with the above files to later download 149 | ftpserver.put_files({ 150 | 'src': str(tmp_path) + '/dir1/file1', 151 | 'dest': 'remote1/file1' 152 | }) 153 | ftpserver.put_files({ 154 | 'src': str(tmp_path) + '/dir1/dir2/file2', 155 | 'dest': 'remote1/remote2/file2' 156 | }) 157 | ftpserver.put_files({ 158 | 'src': str(tmp_path) + '/dir1/dir2/file3', 159 | 'dest': 'remote1/remote2/file3' 160 | }) 161 | 162 | login_dict = ftpserver.get_login_data() 163 | 164 | conn = ftplib.FTP() 165 | conn.connect(host=login_dict['host'], port=login_dict['port']) 166 | conn.login(login_dict['user'], login_dict['passwd']) 167 | 168 | mock_retrbinary = mocker.patch( 169 | 'ftplib.FTP.retrbinary', 170 | side_effect=conn.retrbinary 171 | ) 172 | 173 | ftp_obj = FTPTransput( 174 | str(tmp_path) + "downloads", 175 | "ftp://" + login_dict['host'], 176 | Type.Directory, 177 | ftp_conn=conn 178 | ) 179 | 180 | ftp_obj.download_dir() 181 | 182 | # We use mock.ANY since the 2nd argument of the 'ftplib.FTP.storbinary' is 183 | # a file object and we can't have the same between the original and the 184 | # mock calls 185 | assert sorted(mock_retrbinary.mock_calls) == sorted([ 186 | mock.call('RETR ' + '/remote1/file1', mock.ANY), 187 | mock.call('RETR ' + '/remote1/remote2/file2', mock.ANY), 188 | mock.call('RETR ' + '/remote1/remote2/file3', mock.ANY) 189 | ]) 190 | 191 | assert os.path.exists(str(tmp_path) + 'downloads/remote1/file1') 192 | assert os.path.exists(str(tmp_path) + 'downloads/remote1/remote2/file2') 193 | assert os.path.exists(str(tmp_path) + 'downloads/remote1/remote2/file3') 194 | 195 | 196 | def test_ftp_check_directory_error(mocker, caplog): 197 | """Ensure ftp_check_directory_error creates the proper error log 198 | message in case of error.""" 199 | 200 | conn = mocker.patch('ftplib.FTP') 201 | mocker.patch('ftplib.FTP.cwd', side_effect=ftplib.error_reply) 202 | assert 1 == ftp_check_directory(conn, '/folder/file') 203 | assert 'Could not check if path' in caplog.text 204 | 205 | 206 | def test_ftp_make_dirs(mocker): 207 | """ In case of existing directory, exit with 0. """ 208 | 209 | conn = mocker.patch('ftplib.FTP') 210 | assert ftp_make_dirs(conn, os.curdir) == 0 211 | 212 | 213 | def test_ftp_make_dirs_error(mocker, ftpserver, caplog): 214 | """ Ensure in case of 'ftplib.error_reply', both the return value 215 | and the error message are correct. """ 216 | 217 | login_dict = ftpserver.get_login_data() 218 | 219 | conn = ftplib.FTP() 220 | conn.connect(host=login_dict['host'], port=login_dict['port']) 221 | conn.login(login_dict['user'], login_dict['passwd']) 222 | 223 | mocker.patch('ftplib.FTP.cwd', side_effect=ftplib.error_reply) 224 | 225 | assert ftp_make_dirs(conn, 'dir1') == 1 226 | assert 'Unable to create directory' in caplog.text 227 | -------------------------------------------------------------------------------- /source/tesk-core/tests/test_filer_general_pytest.py: -------------------------------------------------------------------------------- 1 | """Tests for 'filer.py' general purpose functionalities using 'pytest'.""" 2 | 3 | # Note: In tests such as 'test_process_file_with_scheme' or 4 | # 'test_copyContent_dir', only the outer function of each unit under testing is 5 | # checked, since mocking a function apparently affects its output. Maybe 6 | # there's a way to bypass that issue and test deeper down the call tree. 7 | 8 | import pytest 9 | 10 | from tesk_core.filer import ( 11 | process_file, 12 | copyContent, 13 | FileProtocolDisabled 14 | ) 15 | 16 | 17 | def test_process_file_no_scheme(caplog): 18 | """ Ensure that when process_file is called without a scheme and no 19 | 'HOST_BASE_PATH', 'CONTAINER_BASE_PATH' environment variables 20 | set, the appropriate error is raised.""" 21 | 22 | filedata = {'url': 'www.foo.bar'} 23 | 24 | with pytest.raises(FileProtocolDisabled): 25 | process_file('upload', filedata) 26 | 27 | 28 | def test_process_file_with_scheme(mocker): 29 | """ Ensure expected behaviour when 'process_file' is called with scheme. 30 | In this test example, scheme is 'http', filedata:type is 'FILE' and 31 | ttype is 'inputs'.""" 32 | 33 | filedata = { 34 | 'url': 'http://www.foo.bar', 35 | 'path': '.', 36 | 'type': 'FILE', 37 | } 38 | mock_new_Trans = mocker.patch('tesk_core.filer.newTransput') 39 | process_file('inputs', filedata) 40 | 41 | mock_new_Trans.assert_called_once_with('http','www.foo.bar') 42 | 43 | 44 | def test_process_file_from_content(tmpdir, tmp_path): 45 | """ Ensure 'process_file' behaves correctly when the file contents 46 | should be drawn from the filedata content field.""" 47 | 48 | test_file = tmpdir.join("testfile") 49 | filedata = { 50 | 'path': str(tmp_path) + '/testfile', 51 | 'content': 'This is some test content' 52 | } 53 | process_file('inputs', filedata) 54 | 55 | assert open(str(tmp_path) + '/testfile', 'r').read() == filedata['content'] 56 | 57 | 58 | def test_copyContent_dir(mocker): 59 | """Ensure that 'os.listdir' is called when 'copyContent' is called.""" 60 | 61 | mock_os_listdir = mocker.patch('os.listdir') 62 | copyContent('.', '/test_dst') 63 | 64 | mock_os_listdir.assert_called_once_with('.') 65 | -------------------------------------------------------------------------------- /source/tesk-core/tests/test_filer_http_pytest.py: -------------------------------------------------------------------------------- 1 | """Tests for 'filer.py' HTTP functionalities using 'pytest'.""" 2 | 3 | from requests import Response, put 4 | import os 5 | from unittest import mock 6 | 7 | from tesk_core.filer import ( 8 | HTTPTransput, 9 | Type 10 | ) 11 | 12 | PATH_DOWN = 'test_download_file.txt' 13 | PATH_UP = 'tests/test_filer_http_pytest.py' 14 | SUCCESS = 200 15 | FAIL = 300 16 | URL = 'http://www.foo.bar' 17 | FTYPE = 'FILE' 18 | 19 | resp = Response() 20 | resp._content = b'{ "foo" : "bar" }' 21 | 22 | 23 | def test_download_file(mocker): 24 | """ Ensure a file gets properly downloaded.""" 25 | 26 | resp.status_code = SUCCESS 27 | http_obj = HTTPTransput(PATH_DOWN, URL, FTYPE) 28 | mocker.patch('requests.get', return_value=resp) 29 | 30 | with mock.patch( 31 | 'builtins.open', 32 | mock.mock_open(read_data=resp._content), 33 | create=False 34 | ) as m: 35 | assert 0 == http_obj.download_file() 36 | assert open(PATH_DOWN, 'rb').read() == resp._content 37 | 38 | 39 | def test_download_file_error(mocker, caplog): 40 | """ Ensure download error returns the correct value and log message.""" 41 | 42 | resp.status_code = FAIL 43 | http_obj = HTTPTransput(PATH_DOWN, URL, FTYPE) 44 | mocker.patch('requests.get', return_value=resp) 45 | 46 | assert 1 == http_obj.download_file() 47 | assert 'Got status code: {}'.format(FAIL) in caplog.text 48 | 49 | 50 | def test_upload_file(mocker): 51 | """ Ensure a file gets properly uploaded.""" 52 | 53 | resp.status_code = SUCCESS 54 | http_obj = HTTPTransput(PATH_UP, URL, FTYPE) 55 | mocker.patch('requests.put', return_value=resp) 56 | 57 | assert 0 == http_obj.upload_file() 58 | 59 | 60 | def test_upload_file_error(mocker, caplog): 61 | """ Ensure upload error returns the correct value and log message.""" 62 | 63 | resp.status_code = FAIL 64 | http_obj = HTTPTransput(PATH_UP, URL, FTYPE) 65 | mocker.patch('requests.put', return_value=resp) 66 | 67 | assert 1 == http_obj.upload_file() 68 | assert 'Got status code: {}'.format(FAIL) in caplog.text 69 | 70 | 71 | def test_upload_dir(mocker, fs): 72 | """ Ensure that each file inside nexted directories gets successfully 73 | uploaded.""" 74 | 75 | # Tele2 Speedtest Service, free upload /download test server 76 | endpoint = "http://speedtest.tele2.net/upload.php" 77 | resp.status_code = 200 78 | 79 | fs.create_dir('dir1') 80 | fs.create_dir('dir1/dir2') 81 | fs.create_file('dir1/file1', contents="this is random") 82 | fs.create_file('dir1/dir2/file2', contents="not really") 83 | fs.create_file('dir1/dir2/file4.txt', contents="took me a while") 84 | 85 | 86 | mock_put = mocker.patch('requests.put', return_value=resp) 87 | 88 | http_obj = HTTPTransput( 89 | "dir1", 90 | endpoint + "/dir1", 91 | Type.Directory 92 | ) 93 | 94 | assert http_obj.upload_dir() == 0 95 | 96 | # We emply the 'list.sorted' trick to ignore calls order because the 97 | # 'assert_has_calls' method would not work in this setting 98 | assert sorted(mock_put.mock_calls) == sorted([ 99 | mock.call(endpoint + '/dir1/dir2/file2', data="not really"), 100 | mock.call(endpoint + '/dir1/dir2/file4.txt', data="took me a while"), 101 | mock.call(endpoint + '/dir1/file1', data="this is random"), 102 | ]) 103 | 104 | 105 | def test_upload_dir_error(mocker, fs): 106 | """ Ensure 'upload_dir' error returns the correct value. """ 107 | 108 | fs.create_dir('dir2') 109 | 110 | # Tele2 Speedtest Service, free upload /download test server 111 | endpoint1 = "http://speedtest.tele2.net/upload.php" 112 | 113 | # Non-existent endpoint 114 | endpoint2 = "http://somerandomendpoint.fail" 115 | 116 | http_obj1 = HTTPTransput( 117 | "dir1", 118 | endpoint1 + "/dir1", 119 | Type.Directory 120 | ) 121 | 122 | http_obj2 = HTTPTransput( 123 | "dir2", 124 | endpoint2 + "/dir1", 125 | Type.Directory 126 | ) 127 | 128 | assert http_obj1.upload_dir() == 1 129 | assert http_obj2.upload_dir() == 1 130 | -------------------------------------------------------------------------------- /source/tesk-core/tests/test_s3_filer.py: -------------------------------------------------------------------------------- 1 | import os 2 | import pytest 3 | import boto3 4 | from tesk_core.filer_s3 import S3Transput 5 | #from tesk_core.extract_endpoint import extract_endpoint 6 | from moto import mock_s3 7 | from unittest.mock import patch, mock_open 8 | 9 | @pytest.fixture() 10 | def moto_boto(): 11 | with mock_s3(): 12 | boto3.client('s3', endpoint_url="http://s3.amazonaws.com") 13 | 14 | client = boto3.resource('s3',endpoint_url="http://s3.amazonaws.com") 15 | client.create_bucket(Bucket='tesk') 16 | client.Bucket('tesk').put_object(Bucket='tesk', Key='folder/file.txt', Body='') 17 | client.Bucket('tesk').put_object(Bucket='tesk', Key='folder1/folder2/file.txt', Body='') 18 | yield 19 | 20 | @pytest.mark.parametrize("path, url, ftype,expected", [ 21 | ("/home/user/filer_test/file.txt", "s3://tesk/folder/file.txt","FILE", 22 | ("tesk","folder/file.txt")), 23 | ("/home/user/filer_test/file.txt", "s3://tesk/folder1/folder2","DIRECTORY", 24 | ("tesk","folder1/folder2")), 25 | ]) 26 | def test_get_bucket_name_and_file_path( moto_boto, path, url, ftype,expected): 27 | """ 28 | Check if the bucket name and path is extracted correctly for file and folders 29 | """ 30 | trans = S3Transput(path, url, ftype) 31 | assert trans.get_bucket_name_and_file_path() == expected 32 | 33 | @pytest.mark.parametrize("path, url, ftype,expected", [ 34 | ("/home/user/filer_test/file.txt", "s3://tesk/folder/file.txt","FILE",0), 35 | ("/home/user/filer_test/file.txt", "s3://mybucket/folder/file.txt","FILE",1), 36 | ("/home/user/filer_test/", "s3://tesk/folder1/folder2","DIRECTORY",0), 37 | ("/home/user/filer_test/", "s3://mybucket/folder1/folder2","DIRECTORY",1) 38 | ]) 39 | def test_check_if_bucket_exists(moto_boto, path, url, ftype, expected): 40 | """ 41 | Check if the bucket exists 42 | """ 43 | client = boto3.resource('s3', endpoint_url="http://s3.amazonaws.com") 44 | trans = S3Transput(path, url, ftype) 45 | assert trans.check_if_bucket_exists(client) == expected 46 | 47 | # @patch('tesk_core.filer.os.makedirs') 48 | # @patch('builtins.open') 49 | # @patch('s3transfer.utils.OSUtils.rename_file') 50 | @pytest.mark.parametrize("path, url, ftype,expected", [ 51 | ("/home/user/filer_test/file.txt", "s3://tesk/folder/file.txt","FILE",0), 52 | ("/home/user/filer_test/file.txt", "s3://tesk/folder/file_new.txt","FILE",1), 53 | ]) 54 | def test_s3_download_file( moto_boto, path, url, ftype, expected, fs, caplog): 55 | """ 56 | Checking for successful/failed file download from Object storage server 57 | """ 58 | with S3Transput(path, url, ftype) as trans: 59 | assert trans.download_file() == expected 60 | if expected: 61 | assert "Not Found" in caplog.text 62 | else: 63 | assert os.path.exists(path) == True 64 | 65 | 66 | 67 | @patch('tesk_core.filer.os.makedirs') 68 | @patch('builtins.open') 69 | @patch('s3transfer.utils.OSUtils.rename_file') 70 | #@patch("tesk_core.filer_s3.extract_endpoint", return_value="http://s3.amazonaws.com") 71 | @pytest.mark.parametrize("path, url, ftype,expected", [ 72 | ("filer_test/", "s3://tesk/folder1/","DIRECTORY",0), 73 | ("filer_test/", "s3://tesk/folder10/folder20","DIRECTORY",1) 74 | ]) 75 | def test_s3_download_directory( mock_makedirs, mock_open, mock_rename, path, url, ftype, 76 | expected, moto_boto, caplog): 77 | """ 78 | test case to check directory download from Object storage server 79 | """ 80 | with S3Transput(path, url, ftype) as trans: 81 | assert trans.download_dir() == expected 82 | print(mock_rename.mock_calls) 83 | if expected: 84 | assert "Invalid file path" in caplog.text 85 | else: 86 | ''' 87 | s3 object path s3://tesk/folder1/ will contain 'folder2', checking if the 'folder2' 88 | is present in the download folder. 89 | ''' 90 | mock_rename.assert_called_once_with('filer_test/folder2', exist_ok=True) 91 | 92 | 93 | @pytest.mark.parametrize("path, url, ftype,expected", [ 94 | ("/home/user/filer_test/file.txt", "s3://tesk/folder/file.txt","FILE",0), 95 | ("/home/user/filer_test/file_new.txt", "s3://tesk/folder/file.txt","FILE",1), 96 | ]) 97 | def test_s3_upload_file( moto_boto, path, url, ftype, expected,fs, caplog): 98 | """ 99 | Testing successful/failed file upload to object storage server 100 | """ 101 | fs.create_file("/home/user/filer_test/file.txt") 102 | client = boto3.resource('s3', endpoint_url="http://s3.amazonaws.com") 103 | trans = S3Transput(path, url, ftype) 104 | trans.bucket_obj = client.Bucket(trans.bucket) 105 | assert trans.upload_file() == expected 106 | if expected: 107 | assert "File upload failed for" in caplog.text 108 | else: 109 | ''' 110 | Checking if the file was uploaded, if the object is found, load() method will return None 111 | otherwise an exception will be raised. 112 | ''' 113 | assert client.Object('tesk', 'folder/file.txt').load() == None 114 | 115 | 116 | 117 | @pytest.mark.parametrize("path, url, ftype,expected", [ 118 | ("tests", "s3://tesk/folder1/folder2","DIRECTORY",0), 119 | ("/home/user/filer_test_new/", "s3://tesk/folder1/folder2","DIRECTORY",1) 120 | ]) 121 | def test_s3_upload_directory(path, url, ftype, expected, moto_boto, caplog): 122 | """ 123 | Checking for successful and failed Directory upload to object storage server 124 | """ 125 | client = boto3.resource('s3', endpoint_url="http://s3.amazonaws.com") 126 | trans = S3Transput(path, url, ftype) 127 | trans.bucket_obj = client.Bucket(trans.bucket) 128 | assert trans.upload_dir() == expected 129 | if expected: 130 | assert "File upload failed for" in caplog.text 131 | else: 132 | ''' 133 | Checking if the file was uploaded, if the object is found load() method will return None 134 | otherwise an exception will be raised. 135 | ''' 136 | assert client.Object('tesk', 'folder1/folder2/test_filer.py').load() == None 137 | 138 | def test_upload_directory_for_unknown_file_type(moto_boto, fs, monkeypatch, caplog): 139 | """ 140 | Checking whether an exception is raised when the object type is neither file or directory 141 | If the exception is raised, an error message will be logged. 142 | """ 143 | monkeypatch.setattr(os.path, 'isfile', lambda _:False) 144 | fs.create_file("/home/user/filer_test/text.txt") 145 | url, ftype = "s3://tesk/folder10/folder20","DIRECTORY" 146 | path = "/home/user/filer_test/" 147 | trans = S3Transput(path, url, ftype) 148 | client = boto3.resource('s3', endpoint_url="http://s3.amazonaws.com") 149 | trans.bucket_obj = client.Bucket(trans.bucket) 150 | trans.upload_dir() 151 | assert "Object is neither file or directory" in caplog.text 152 | 153 | 154 | @patch("tesk_core.filer.os.path.exists", return_value=1) 155 | def test_extract_url_from_config_file(mock_path_exists): 156 | """ 157 | Testing extraction of endpoint url from default file location 158 | """ 159 | read_data = '\n'.join(["[default]", "endpoint_url = http://s3-aws-region.amazonaws.com"]) 160 | with patch("builtins.open", mock_open(read_data=read_data), create=True) as mock_file: 161 | mock_file.return_value.__iter__.return_value = read_data.splitlines() 162 | #assert extract_endpoint() == "http://s3-aws-region.amazonaws.com" 163 | #mock_file.assert_called_once_with("~/.aws/config", encoding=None) 164 | 165 | @patch.dict(os.environ, {"AWS_CONFIG_FILE": "~/.aws/config"}) 166 | def test_extract_url_from_environ_variable(): 167 | """ 168 | Testing successful extraction of endpoint url read from file path saved on enviornment variable 169 | """ 170 | read_data = '\n'.join(["[default]","endpoint_url = http://s3-aws-region.amazonaws.com"]) 171 | with patch("builtins.open", mock_open(read_data=read_data),create=True) as mock_file: 172 | mock_file.return_value.__iter__.return_value = read_data.splitlines() 173 | #assert (extract_endpoint() == "http://s3-aws-region.amazonaws.com") 174 | #mock_file.assert_called_once_with(os.environ["AWS_CONFIG_FILE"], encoding=None) 175 | -------------------------------------------------------------------------------- /source/tesk-core/tests/test_taskmaster.py: -------------------------------------------------------------------------------- 1 | import json 2 | import os 3 | import unittest 4 | from unittest.mock import patch 5 | from argparse import Namespace 6 | from tesk_core import taskmaster 7 | from tesk_core.filer_class import Filer 8 | from kubernetes.client.rest import ApiException 9 | from tesk_core.taskmaster import init_pvc, PVC, run_executor,\ 10 | generate_mounts, append_mount, dirname, run_task,newParser 11 | 12 | 13 | class TaskmasterTest(unittest.TestCase): 14 | 15 | def setUp(self): 16 | self.data = json.loads(open(os.path.join(os.path.dirname(__file__), "resources/inputFile.json")).read()) 17 | self.task_name = self.data['executors'][0]['metadata']['labels']['taskmaster-name'] 18 | taskmaster.args = Namespace( debug = False, file = None, filer_version = 'v0.1.9', json = 'json' 19 | ,namespace='default', poll_interval=5, state_file='/tmp/.teskstate' 20 | , localKubeConfig=False, pull_policy_always=False 21 | , filer_name= "eu.gcr.io/tes-wes/filer", pod_timeout = 240 22 | ) 23 | self.filer = Filer(self.task_name + '-filer', self.data, taskmaster.args.filer_name, 24 | taskmaster.args.filer_version, taskmaster.args.pull_policy_always) 25 | self.pvc = PVC(self.task_name + '-pvc', self.data['resources']['disk_gb'], taskmaster.args.namespace) 26 | 27 | taskmaster.created_jobs = [] 28 | 29 | @patch("tesk_core.taskmaster.PVC.create") 30 | @patch("tesk_core.taskmaster.Job.run_to_completion", return_value="Complete") 31 | @patch("tesk_core.taskmaster.logger") 32 | def test_pvc_creation(self, mock_logger, mock_run_to_compl, mock_pvc_create): 33 | """ 34 | Testing to check if the PVC volume was created successfully 35 | """ 36 | self.assertIsInstance(init_pvc(self.data, self.filer), PVC) 37 | 38 | @patch("kubernetes.client.CoreV1Api.read_namespaced_persistent_volume_claim") 39 | @patch("kubernetes.client.CoreV1Api.create_namespaced_persistent_volume_claim", side_effect=ApiException(status=409, 40 | reason="conflict")) 41 | def test_create_pvc_check_for_conflict_exception(self, mock_create_namespaced_pvc, 42 | mock_read_namespaced_pvc): 43 | self.pvc.create() 44 | mock_read_namespaced_pvc.assert_called_once() 45 | 46 | @patch("kubernetes.client.CoreV1Api.create_namespaced_persistent_volume_claim", side_effect=ApiException(status=500, 47 | reason="Random error")) 48 | def test_create_pvc_check_for_other_exceptions(self, mock_create_namespaced_pvc): 49 | with self.assertRaises(ApiException): 50 | self.pvc.create() 51 | 52 | 53 | @patch("tesk_core.taskmaster.PVC.delete") 54 | @patch("tesk_core.taskmaster.PVC.create") 55 | @patch("tesk_core.taskmaster.Job.run_to_completion", return_value="error") 56 | @patch("tesk_core.taskmaster.logger") 57 | def test_pvc_failure(self, mock_logger, run_to_compl, mock_pvc_create, mock_pvc_delete): 58 | """ 59 | Testcase for finding if the PVC creation failed with exit 0 60 | """ 61 | 62 | self.assertRaises(SystemExit, init_pvc, self.data, self.filer) 63 | 64 | @patch("tesk_core.taskmaster.PVC.delete") 65 | @patch("tesk_core.taskmaster.Job.delete") 66 | @patch("tesk_core.taskmaster.Job.run_to_completion", return_value="Error") 67 | @patch("tesk_core.taskmaster.logger") 68 | def test_run_executor_failure(self, mock_logger, mock_run_to_compl, mock_job_delete, mock_pvc_delete): 69 | """ 70 | 71 | """ 72 | self.assertRaises(SystemExit, run_executor, self.data['executors'][0],taskmaster.args.namespace) 73 | 74 | @patch("tesk_core.taskmaster.PVC") 75 | @patch("tesk_core.taskmaster.Job.run_to_completion", return_value="Complete") 76 | @patch("tesk_core.taskmaster.logger") 77 | def test_run_executor_complete(self, mock_logger, mock_run_to_compl, mock_pvc): 78 | """ 79 | 80 | """ 81 | self.assertEqual(run_executor(self.data['executors'][0], taskmaster.args.namespace,mock_pvc),None) 82 | 83 | 84 | 85 | @patch("tesk_core.taskmaster.logger") 86 | def test_generate_mount(self, mock_logger): 87 | """ 88 | 89 | """ 90 | self.assertIsInstance(generate_mounts(self.data, self.pvc),list) 91 | 92 | @patch("tesk_core.taskmaster.logger") 93 | def test_append_mount(self, mock_logger): 94 | """ 95 | 96 | """ 97 | volume_mounts = [] 98 | task_volume_name = 'task-volume' 99 | for aninput in self.data['inputs']: 100 | dirnm = dirname(aninput) 101 | append_mount(volume_mounts, task_volume_name, dirnm, self.pvc) 102 | self.assertEqual(volume_mounts,[{'name': task_volume_name, 'mountPath': '/some/volume', 'subPath': 'dir0'}]) 103 | 104 | 105 | @patch('tesk_core.taskmaster.logger') 106 | @patch('tesk_core.taskmaster.PVC.create') 107 | @patch('tesk_core.taskmaster.PVC.delete') 108 | @patch('tesk_core.taskmaster.Job.run_to_completion', return_value='Complete' ) 109 | def test_run_task(self, mock_job, mock_pvc_create, mock_pvc_delete, mock_logger): 110 | """ 111 | 112 | """ 113 | run_task(self.data, taskmaster.args.filer_name, taskmaster.args.filer_version) 114 | 115 | def test_localKubeConfig(self): 116 | """ 117 | 118 | """ 119 | parser = newParser() 120 | args = parser.parse_args(['json', '--localKubeConfig']) 121 | self.assertEqual(args 122 | , Namespace(debug=False, file=None, filer_version='v0.1.9', json='json', namespace='default', 123 | poll_interval=5, state_file='/tmp/.teskstate' 124 | , localKubeConfig=True 125 | , pull_policy_always=False 126 | , filer_name='eu.gcr.io/tes-wes/filer' 127 | , pod_timeout=240 128 | ) 129 | ) 130 | 131 | if __name__ == "__main__": 132 | #import sys;sys.argv = ['', 'Test.testName'] 133 | unittest.main() -------------------------------------------------------------------------------- /source/tesk-core/tox.ini: -------------------------------------------------------------------------------- 1 | [tox] 2 | envlist = 3 | py{38,39,310,311}-unit, 4 | py{38,39,310,311}-lint 5 | skip_missing_interpreters = True 6 | 7 | [gh-actions] 8 | python = 9 | 3.7: py37 10 | 3.8: py38 11 | 3.9: py39 12 | 3.10: py310 13 | 14 | [testenv] 15 | passenv = CI, TRAVIS, TRAVIS_* 16 | deps = 17 | py{38,39,310,311}: .[test] 18 | py{38,39,310,311}-unit: pytest-cov 19 | codecov 20 | py{38,39,310,311}-lint: pylint 21 | commands = 22 | py{38,39,310,311}-unit: pytest -v --cov-report xml --cov tesk_core {posargs} tests 23 | py{38,39,310,311}-unit: codecov 24 | py{38,39,310,311}-lint: python -m pylint --exit-zero -d missing-docstring,line-too-long,C tesk_core 25 | py{38,39,310,311}-lint: python -m pylint -E tesk_core 26 | --------------------------------------------------------------------------------