├── .dockerignore ├── .gitignore ├── Dockerfile ├── LICENSE ├── Makefile ├── PROJECT ├── README.md ├── VERSION ├── api └── v1alpha1 │ ├── groupversion_info.go │ ├── jobflow_types.go │ ├── jobtemplate_types.go │ └── zz_generated.deepcopy.go ├── config ├── certmanager │ ├── certificate.yaml │ ├── kustomization.yaml │ └── kustomizeconfig.yaml ├── crd │ ├── bases │ │ ├── flow.volcano.sh_jobflows.yaml │ │ └── flow.volcano.sh_jobtemplates.yaml │ ├── kustomization.yaml │ ├── kustomizeconfig.yaml │ └── patches │ │ ├── cainjection_in_jobflows.yaml │ │ ├── cainjection_in_jobtemplates.yaml │ │ ├── webhook_in_jobflows.yaml │ │ └── webhook_in_jobtemplates.yaml ├── default │ ├── kustomization.yaml │ ├── manager_auth_proxy_patch.yaml │ ├── manager_config_patch.yaml │ ├── manager_webhook_patch.yaml │ └── webhookcainjection_patch.yaml ├── manager │ ├── controller_manager_config.yaml │ ├── kustomization.yaml │ └── manager.yaml ├── prometheus │ ├── kustomization.yaml │ └── monitor.yaml ├── rbac │ ├── auth_proxy_client_clusterrole.yaml │ ├── auth_proxy_role.yaml │ ├── auth_proxy_role_binding.yaml │ ├── auth_proxy_service.yaml │ ├── jobflow_editor_role.yaml │ ├── jobflow_viewer_role.yaml │ ├── jobtemplate_editor_role.yaml │ ├── jobtemplate_viewer_role.yaml │ ├── kustomization.yaml │ ├── leader_election_role.yaml │ ├── leader_election_role_binding.yaml │ ├── role.yaml │ ├── role_binding.yaml │ └── service_account.yaml ├── samples │ ├── batch_v1alpha1_jobflow.yaml │ └── batch_v1alpha1_jobtemplate.yaml └── webhook │ ├── kustomization.yaml │ ├── kustomizeconfig.yaml │ ├── manifests.yaml │ ├── secret.yaml │ └── service.yaml ├── controllers ├── jobflow.go ├── jobflow_test.go ├── jobtemplate.go └── suite_test.go ├── deploy └── jobflow.yaml ├── docs ├── community │ └── roadmap.md ├── design │ ├── jobflow.md │ └── jobtemplate.md ├── images │ └── jobflow.gif └── scanning-report │ └── fossas-report.html ├── example ├── JobFlow.yaml ├── JobTemplate.yaml └── README.md ├── go.mod ├── go.sum ├── hack ├── boilerplate.go.txt └── run-e2e.sh ├── main.go ├── test └── e2e │ ├── jobflow-admission │ ├── admission.go │ ├── e2e_test.go │ └── main_test.go │ ├── jobflow-controller │ ├── e2e_test.go │ ├── jobflow.go │ └── main_test.go │ ├── jobtemplate-admission │ ├── admission.go │ ├── e2e_test.go │ └── main_test.go │ ├── jobtemplate-controller │ ├── e2e_test.go │ ├── jobtemplate.go │ └── main_test.go │ └── util │ └── util.go ├── utils ├── utils.go ├── utils_test.go ├── validate_dag.go └── validate_util.go └── webhooks ├── admission ├── jobflow │ └── validate │ │ ├── validate_jobflow.go │ │ └── validate_jobflow_test.go └── template │ └── validate │ ├── validate_template.go │ └── validate_template_test.go ├── router ├── admission.go ├── interface.go └── server.go ├── schema └── schema.go ├── server.go └── util └── util.go /.dockerignore: -------------------------------------------------------------------------------- 1 | # More info: https://docs.docker.com/engine/reference/builder/#dockerignore-file 2 | # Ignore all files which are not go type 3 | !**/*.go 4 | !**/*.mod 5 | !**/*.sum 6 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | 2 | # Binaries for programs and plugins 3 | *.exe 4 | *.exe~ 5 | *.dll 6 | *.so 7 | *.dylib 8 | bin 9 | testbin/* 10 | 11 | # Test binary, build with `go test -c` 12 | *.test 13 | 14 | # Output of the go coverage tool, specifically when used with LiteIDE 15 | *.out 16 | 17 | # Kubernetes Generated files - skip generated files, except for vendored files 18 | 19 | !vendor/**/zz_generated.* 20 | 21 | # editor and IDE paraphernalia 22 | .idea 23 | *.swp 24 | *.swo 25 | *~ 26 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM alpine:3.14.3 2 | 3 | COPY bin/manager /manager 4 | 5 | ENTRYPOINT ["/manager"] -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright [yyyy] [name of copyright owner] 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | # image VERSION 2 | VERSION:=$(shell cat VERSION) 3 | 4 | # Image URL to use all building/pushing image targets 5 | IMG = beyondcent/jobflow:$(VERSION) 6 | # Produce CRDs that work back to Kubernetes 1.11 (no version conversion) 7 | CRD_OPTIONS ?= "crd:trivialVersions=true,preserveUnknownFields=false" 8 | 9 | # Get the currently used golang install path (in GOPATH/bin, unless GOBIN is set) 10 | ifeq (,$(shell go env GOBIN)) 11 | GOBIN=$(shell go env GOPATH)/bin 12 | else 13 | GOBIN=$(shell go env GOBIN) 14 | endif 15 | 16 | # Setting SHELL to bash allows bash commands to be executed by recipes. 17 | # This is a requirement for 'setup-envtest.sh' in the test target. 18 | # Options are set to exit when a recipe line exits non-zero or a piped command fails. 19 | SHELL = /usr/bin/env bash -o pipefail 20 | .SHELLFLAGS = -ec 21 | 22 | all: build 23 | 24 | ##@ General 25 | 26 | # The help target prints out all targets with their descriptions organized 27 | # beneath their categories. The categories are represented by '##@' and the 28 | # target descriptions by '##'. The awk commands is responsible for reading the 29 | # entire set of makefiles included in this invocation, looking for lines of the 30 | # file as xyz: ## something, and then pretty-format the target and help. Then, 31 | # if there's a line with ##@ something, that gets pretty-printed as a category. 32 | # More info on the usage of ANSI control characters for terminal formatting: 33 | # https://en.wikipedia.org/wiki/ANSI_escape_code#SGR_parameters 34 | # More info on the awk command: 35 | # http://linuxcommand.org/lc3_adv_awk.php 36 | 37 | help: ## Display this help. 38 | @awk 'BEGIN {FS = ":.*##"; printf "\nUsage:\n make \033[36m\033[0m\n"} /^[a-zA-Z_0-9-]+:.*?##/ { printf " \033[36m%-15s\033[0m %s\n", $$1, $$2 } /^##@/ { printf "\n\033[1m%s\033[0m\n", substr($$0, 5) } ' $(MAKEFILE_LIST) 39 | 40 | ##@ Development 41 | 42 | manifests: controller-gen ## Generate WebhookConfiguration, ClusterRole and CustomResourceDefinition objects. 43 | $(CONTROLLER_GEN) $(CRD_OPTIONS) rbac:roleName=manager-role webhook paths="./..." output:crd:artifacts:config=config/crd/bases 44 | 45 | generate: controller-gen ## Generate code containing DeepCopy, DeepCopyInto, and DeepCopyObject method implementations. 46 | $(CONTROLLER_GEN) object:headerFile="hack/boilerplate.go.txt" paths="./..." 47 | 48 | fmt: ## Run go fmt against code. 49 | go fmt ./... 50 | 51 | vet: ## Run go vet against code. 52 | go vet ./... 53 | 54 | ENVTEST_ASSETS_DIR=$(shell pwd)/testbin 55 | test: manifests generate fmt vet ## Run tests. 56 | mkdir -p ${ENVTEST_ASSETS_DIR} 57 | test -f ${ENVTEST_ASSETS_DIR}/setup-envtest.sh || curl -sSLo ${ENVTEST_ASSETS_DIR}/setup-envtest.sh https://raw.githubusercontent.com/kubernetes-sigs/controller-runtime/v0.8.3/hack/setup-envtest.sh 58 | source ${ENVTEST_ASSETS_DIR}/setup-envtest.sh; fetch_envtest_tools $(ENVTEST_ASSETS_DIR); setup_envtest_env $(ENVTEST_ASSETS_DIR); go test ./... -coverprofile cover.out 59 | 60 | ##@ Build 61 | 62 | build: generate fmt vet ## Build manager binary. 63 | CGO_ENABLED=0 go build -o bin/manager main.go 64 | 65 | run: manifests generate fmt vet ## Run a controller from your host. 66 | go run ./main.go 67 | 68 | docker-build: ## Build docker image with the manager. 69 | docker build --no-cache -t ${IMG} . 70 | 71 | docker-push: ## Push docker image with the manager. 72 | docker push ${IMG} 73 | 74 | ##@ Deployment 75 | 76 | install: manifests kustomize ## Install CRDs into the K8s cluster specified in ~/.kube/config. 77 | $(KUSTOMIZE) build config/crd | kubectl apply -f - 78 | 79 | uninstall: manifests kustomize ## Uninstall CRDs from the K8s cluster specified in ~/.kube/config. 80 | $(KUSTOMIZE) build config/crd | kubectl delete -f - 81 | 82 | deploy: manifests kustomize ## Deploy controller to the K8s cluster specified in ~/.kube/config. 83 | cd config/manager && $(KUSTOMIZE) edit set image controller=${IMG} 84 | $(KUSTOMIZE) build config/default | kubectl apply -f - 85 | 86 | yaml: manifests kustomize ## Deploy controller to the K8s cluster specified in ~/.kube/config. 87 | cd config/manager && $(KUSTOMIZE) edit set image controller=${IMG} 88 | $(KUSTOMIZE) build config/default -o deploy/jobflow.yaml 89 | 90 | undeploy: ## Undeploy controller from the K8s cluster specified in ~/.kube/config. 91 | $(KUSTOMIZE) build config/default | kubectl delete -f - 92 | 93 | e2e: 94 | ./hack/run-e2e.sh 95 | 96 | e2e-test-jobflow-controller: 97 | E2E_TYPE=JOBFLOWCONTROLLER ./hack/run-e2e.sh 98 | 99 | e2e-test-jobtemplate-controller: 100 | E2E_TYPE=JOBTEMPLATECONTROLLER ./hack/run-e2e.sh 101 | 102 | e2e-test-jobflow-admission: 103 | E2E_TYPE=JOBFLOWADMISSION ./hack/run-e2e.sh 104 | 105 | e2e-test-jobtemplate-admission: 106 | E2E_TYPE=JOBTEMPLATEADMISSION ./hack/run-e2e.sh 107 | 108 | 109 | CONTROLLER_GEN = $(shell pwd)/bin/controller-gen 110 | controller-gen: ## Download controller-gen locally if necessary. 111 | $(call go-get-tool,$(CONTROLLER_GEN),sigs.k8s.io/controller-tools/cmd/controller-gen@v0.4.1) 112 | 113 | KUSTOMIZE = $(shell pwd)/bin/kustomize 114 | kustomize: ## Download kustomize locally if necessary. 115 | $(call go-get-tool,$(KUSTOMIZE),sigs.k8s.io/kustomize/kustomize/v3@v3.8.7) 116 | 117 | # go-get-tool will 'go get' any package $2 and install it to $1. 118 | PROJECT_DIR := $(shell dirname $(abspath $(lastword $(MAKEFILE_LIST)))) 119 | define go-get-tool 120 | @[ -f $(1) ] || { \ 121 | set -e ;\ 122 | TMP_DIR=$$(mktemp -d) ;\ 123 | cd $$TMP_DIR ;\ 124 | go mod init tmp ;\ 125 | echo "Downloading $(2)" ;\ 126 | GOBIN=$(PROJECT_DIR)/bin go get $(2) ;\ 127 | rm -rf $$TMP_DIR ;\ 128 | } 129 | endef 130 | -------------------------------------------------------------------------------- /PROJECT: -------------------------------------------------------------------------------- 1 | domain: volcano.sh 2 | layout: 3 | - go.kubebuilder.io/v3 4 | projectName: jobflow 5 | repo: jobflow 6 | resources: 7 | - api: 8 | crdVersion: v1 9 | namespaced: true 10 | controller: true 11 | domain: volcano.sh 12 | group: flow 13 | kind: JobFlow 14 | path: jobflow/api/v1alpha1 15 | version: v1alpha1 16 | - api: 17 | crdVersion: v1 18 | namespaced: true 19 | controller: true 20 | domain: volcano.sh 21 | group: flow 22 | kind: JobTemplate 23 | path: jobflow/api/v1alpha1 24 | version: v1alpha1 25 | version: "3" 26 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # JobFlow 2 | 3 | ![jobflowAnimation](./docs/images/jobflow.gif) 4 | 5 | ## Project Status 6 | 7 | This project is being donated to the volcano community 8 | 9 | ## Introduction 10 | 11 | Volcano is an CNCF sandbox project aiming for running tranditional batch jobs on Kubernetes. It abstracts those batch jobs into an CRD called VCJob and has an excellet scheduler to imporve resource utilization. However, to solve an real-world issue, we need many VCJobs to cooperate each other and orchestrate them mannualy or by another Job Orchestruating Platrom to get the job done finally.We present an new way of orchestruing VCJobs called JobFlow. We proposed two concepts to running multiple batch jobs automatically named JobTemplate and JobFlow so end users can easily declare their jobs and run them using complex controlling primitives, for example, sequential or parallel executing, if-then-else statement, switch-case statement, loop executing and so on. 12 | 13 | JobFlow helps migrating AI, BigData, HPC workloads to the cloudnative world. Though there are already some workload flow engines, they are not designed for batch job workloads. Those jobs typically have a complex running dependencies and take long time to run, for example days or weeks. JobFlow helps the end users to declaire their jobs as an jobTemplate and then reuse them accordingly. Also, JobFlow orchestruating those jobs using complex controlling primitives and lanch those jobs automatically. This can significantly reduce the time consumption of an complex job and improve resource utilization. Finally, JobFlow is not an generally purposed workflow engine, it knows the details of VCJobs. End user can have a better understanding of their jobs, for example, job's running state, beginning and ending timestamps, the next jobs to run, pod-failure-ratio and so on. 14 | 15 | ## Demo video 16 | 17 | https://www.bilibili.com/video/BV1c44y1Y7FX 18 | 19 | ## Deploy 20 | ``` 21 | kubectl apply -f https://raw.githubusercontent.com/BoCloud/JobFlow/main/deploy/jobflow.yaml 22 | ``` 23 | 24 | ## Donation Self-Check Form 25 | 26 | | ID | Item | Description | Required | Compliance Conditions | Note | complete | 27 | | ---- | --------------------- | ------------------------------------------------------------ | -------- | ------------------------------------------------------------ | ------------------------------- | ---- | 28 | | 1 | Code of Conduct | The conduct for the source code | Y | [Contributor Covenant Code of Conduct](https://www.contributor-covenant.org/) | Submit the code scanning report | [yes](./docs/scanning-report/fossas-report.html) | 29 | | 2 | License | The License the project obeys | Y | [Apache 2.0](https://github.com/volcano-sh/volcano/blob/master/LICENSE) | | [yes](./LICENSE) | 30 | | 3 | Readme | Brief introduction of the project along with the source code | Y | | | [yes](./README.md) | 31 | | 4 | CI/CD | The CI/CD to judge the compliance for all PRs | Y | [Github Action](https://docs.github.com/en/actions) | | yes| 32 | | 5 | Security | Security policy including vulnerability discovery and disposal | Y | [Security Release Process](https://github.com/volcano-sh/volcano/blob/master/SECURITY.md) | Submit security scanning report | yes[![Total alerts](https://img.shields.io/lgtm/alerts/g/BoCloud/JobFlow.svg?logo=lgtm&logoWidth=18)](https://lgtm.com/projects/g/BoCloud/JobFlow/alerts/) | 33 | | 6 | Roadmap | Roadmap file about the important features in the feature | Y | | | [yes](./docs/community/roadmap.md) | 34 | | 7 | Design Documentations | Documentations about the record of feature designs | Y | | | [yes](./docs/design) | -------------------------------------------------------------------------------- /VERSION: -------------------------------------------------------------------------------- 1 | v0.0.1 -------------------------------------------------------------------------------- /api/v1alpha1/groupversion_info.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2021. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | // Package v1alpha1 contains API Schema definitions for the flow v1alpha1 API group 18 | //+kubebuilder:object:generate=true 19 | //+groupName=flow.volcano.sh 20 | package v1alpha1 21 | 22 | import ( 23 | "k8s.io/apimachinery/pkg/runtime/schema" 24 | "sigs.k8s.io/controller-runtime/pkg/scheme" 25 | ) 26 | 27 | var ( 28 | // GroupVersion is group version used to register these objects 29 | GroupVersion = schema.GroupVersion{Group: "flow.volcano.sh", Version: "v1alpha1"} 30 | 31 | // SchemeBuilder is used to add go types to the GroupVersionKind scheme 32 | SchemeBuilder = &scheme.Builder{GroupVersion: GroupVersion} 33 | 34 | // AddToScheme adds the types in this group-version to the given scheme. 35 | AddToScheme = SchemeBuilder.AddToScheme 36 | ) 37 | -------------------------------------------------------------------------------- /api/v1alpha1/jobflow_types.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2021. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package v1alpha1 18 | 19 | import ( 20 | v1 "k8s.io/api/core/v1" 21 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 22 | "volcano.sh/apis/pkg/apis/batch/v1alpha1" 23 | ) 24 | 25 | // EDIT THIS FILE! THIS IS SCAFFOLDING FOR YOU TO OWN! 26 | // NOTE: json tags are required. Any new fields you add must have json tags for the fields to be serialized. 27 | 28 | // JobFlowSpec defines the desired state of JobFlow 29 | type JobFlowSpec struct { 30 | // INSERT ADDITIONAL SPEC FIELDS - desired state of cluster 31 | // Important: Run "make" to regenerate code after modifying this file 32 | 33 | // Foo is an example field of JobFlow. Edit jobflow_types.go to remove/update 34 | Flows []Flow `json:"flows,omitempty"` 35 | JobRetainPolicy string `json:"jobRetainPolicy,omitempty"` 36 | } 37 | 38 | // Flow defines the dependent of jobs 39 | type Flow struct { 40 | Name string `json:"name"` 41 | DependsOn *DependsOn `json:"dependsOn,omitempty"` 42 | } 43 | 44 | type DependsOn struct { 45 | Targets []string `json:"targets,omitempty"` 46 | Probe *Probe `json:"probe,omitempty"` 47 | } 48 | 49 | type Probe struct { 50 | HttpGetList []HttpGet `json:"httpGetList,omitempty"` 51 | TcpSocketList []TcpSocket `json:"tcpSocketList,omitempty"` 52 | TaskStatusList []TaskStatus `json:"taskStatusList,omitempty"` 53 | } 54 | 55 | type HttpGet struct { 56 | TaskName string `json:"taskName,omitempty"` 57 | Path string `json:"path,omitempty"` 58 | Port int `json:"port,omitempty"` 59 | HTTPHeader v1.HTTPHeader `json:"httpHeader,omitempty"` 60 | } 61 | 62 | type TcpSocket struct { 63 | TaskName string `json:"taskName,omitempty"` 64 | Port int `json:"port"` 65 | } 66 | 67 | type TaskStatus struct { 68 | TaskName string `json:"taskName,omitempty"` 69 | Phase string `json:"phase,omitempty"` 70 | } 71 | 72 | // JobFlowStatus defines the observed state of JobFlow 73 | type JobFlowStatus struct { 74 | // INSERT ADDITIONAL STATUS FIELD - define observed state of cluster 75 | // Important: Run "make" to regenerate code after modifying this file 76 | 77 | PendingJobs []string `json:"pendingJobs,omitempty"` 78 | RunningJobs []string `json:"runningJobs,omitempty"` 79 | FailedJobs []string `json:"failedJobs,omitempty"` 80 | CompletedJobs []string `json:"completedJobs,omitempty"` 81 | TerminatedJobs []string `json:"terminatedJobs,omitempty"` 82 | UnKnowJobs []string `json:"unKnowJobs,omitempty"` 83 | JobStatusList []JobStatus `json:"jobStatusList,omitempty"` 84 | Conditions map[string]Condition `json:"conditions,omitempty"` 85 | State State `json:"state,omitempty"` 86 | } 87 | 88 | type JobStatus struct { 89 | Name string `json:"name,omitempty"` 90 | State v1alpha1.JobPhase `json:"state,omitempty"` 91 | StartTimestamp metav1.Time `json:"startTimestamp,omitempty"` 92 | EndTimestamp metav1.Time `json:"endTimestamp,omitempty"` 93 | RestartCount int32 `json:"restartCount,omitempty"` 94 | RunningHistories []JobRunningHistory `json:"runningHistories,omitempty"` 95 | } 96 | 97 | type JobRunningHistory struct { 98 | StartTimestamp metav1.Time `json:"startTimestamp,omitempty"` 99 | EndTimestamp metav1.Time `json:"endTimestamp,omitempty"` 100 | State v1alpha1.JobPhase `json:"state,omitempty"` 101 | } 102 | 103 | type State struct { 104 | Phase Phase `json:"phase,omitempty"` 105 | } 106 | 107 | type Phase string 108 | 109 | const ( 110 | Retain = "retain" 111 | Delete = "delete" 112 | ) 113 | 114 | const ( 115 | Succeed Phase = "Succeed" 116 | Terminating Phase = "Terminating" 117 | Failed Phase = "Failed" 118 | Running Phase = "Running" 119 | Pending Phase = "Pending" 120 | ) 121 | 122 | type Condition struct { 123 | Phase v1alpha1.JobPhase `json:"phase,omitempty"` 124 | CreateTimestamp metav1.Time `json:"createTime,omitempty"` 125 | RunningDuration *metav1.Duration `json:"runningDuration,omitempty"` 126 | TaskStatusCount map[string]v1alpha1.TaskState `json:"taskStatusCount,omitempty"` 127 | } 128 | 129 | //+kubebuilder:object:root=true 130 | //+kubebuilder:printcolumn:name="Status",type="string",JSONPath=".status.state.phase" 131 | // +kubebuilder:printcolumn:name="Age",type="date",JSONPath=".metadata.creationTimestamp" 132 | // +kubebuilder:resource:path=jobflows,shortName=jf 133 | //+kubebuilder:subresource:status 134 | 135 | // JobFlow is the Schema for the jobflows API 136 | type JobFlow struct { 137 | metav1.TypeMeta `json:",inline"` 138 | metav1.ObjectMeta `json:"metadata,omitempty"` 139 | 140 | Spec JobFlowSpec `json:"spec,omitempty"` 141 | Status JobFlowStatus `json:"status,omitempty"` 142 | } 143 | 144 | //+kubebuilder:object:root=true 145 | 146 | // JobFlowList contains a list of JobFlow 147 | type JobFlowList struct { 148 | metav1.TypeMeta `json:",inline"` 149 | metav1.ListMeta `json:"metadata,omitempty"` 150 | Items []JobFlow `json:"items"` 151 | } 152 | 153 | func init() { 154 | SchemeBuilder.Register(&JobFlow{}, &JobFlowList{}) 155 | } 156 | -------------------------------------------------------------------------------- /api/v1alpha1/jobtemplate_types.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2021. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package v1alpha1 18 | 19 | import ( 20 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 21 | "volcano.sh/apis/pkg/apis/batch/v1alpha1" 22 | ) 23 | 24 | // EDIT THIS FILE! THIS IS SCAFFOLDING FOR YOU TO OWN! 25 | // NOTE: json tags are required. Any new fields you add must have json tags for the fields to be serialized. 26 | 27 | // JobTemplateSpec defines the desired state of JobTemplate 28 | type JobTemplateSpec struct { 29 | // INSERT ADDITIONAL SPEC FIELDS - desired state of cluster 30 | // Important: Run "make" to regenerate code after modifying this file 31 | 32 | // Foo is an example field of JobTemplate. Edit jobtemplate_types.go to remove/update 33 | v1alpha1.JobSpec 34 | } 35 | 36 | // JobTemplateStatus defines the observed state of JobTemplate 37 | type JobTemplateStatus struct { 38 | // INSERT ADDITIONAL STATUS FIELD - define observed state of cluster 39 | // Important: Run "make" to regenerate code after modifying this file 40 | 41 | //Describes the Jobs generated from the JobTemplate 42 | JobDependsOnList []string `json:"jobDependsOnList,omitempty"` 43 | } 44 | 45 | //+kubebuilder:object:root=true 46 | // +kubebuilder:resource:path=jobtemplates,shortName=jt 47 | //+kubebuilder:subresource:status 48 | 49 | // JobTemplate is the Schema for the jobtemplates API 50 | type JobTemplate struct { 51 | metav1.TypeMeta `json:",inline"` 52 | metav1.ObjectMeta `json:"metadata,omitempty"` 53 | 54 | Spec v1alpha1.JobSpec `json:"spec,omitempty"` 55 | Status JobTemplateStatus `json:"status,omitempty"` 56 | } 57 | 58 | //+kubebuilder:object:root=true 59 | 60 | // JobTemplateList contains a list of JobTemplate 61 | type JobTemplateList struct { 62 | metav1.TypeMeta `json:",inline"` 63 | metav1.ListMeta `json:"metadata,omitempty"` 64 | Items []JobTemplate `json:"items"` 65 | } 66 | 67 | func init() { 68 | SchemeBuilder.Register(&JobTemplate{}, &JobTemplateList{}) 69 | } 70 | -------------------------------------------------------------------------------- /api/v1alpha1/zz_generated.deepcopy.go: -------------------------------------------------------------------------------- 1 | // +build !ignore_autogenerated 2 | 3 | /* 4 | Copyright 2021. 5 | 6 | Licensed under the Apache License, Version 2.0 (the "License"); 7 | you may not use this file except in compliance with the License. 8 | You may obtain a copy of the License at 9 | 10 | http://www.apache.org/licenses/LICENSE-2.0 11 | 12 | Unless required by applicable law or agreed to in writing, software 13 | distributed under the License is distributed on an "AS IS" BASIS, 14 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | See the License for the specific language governing permissions and 16 | limitations under the License. 17 | */ 18 | 19 | // Code generated by controller-gen. DO NOT EDIT. 20 | 21 | package v1alpha1 22 | 23 | import ( 24 | "k8s.io/apimachinery/pkg/apis/meta/v1" 25 | runtime "k8s.io/apimachinery/pkg/runtime" 26 | batchv1alpha1 "volcano.sh/apis/pkg/apis/batch/v1alpha1" 27 | ) 28 | 29 | // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. 30 | func (in *Condition) DeepCopyInto(out *Condition) { 31 | *out = *in 32 | in.CreateTimestamp.DeepCopyInto(&out.CreateTimestamp) 33 | if in.RunningDuration != nil { 34 | in, out := &in.RunningDuration, &out.RunningDuration 35 | *out = new(v1.Duration) 36 | **out = **in 37 | } 38 | if in.TaskStatusCount != nil { 39 | in, out := &in.TaskStatusCount, &out.TaskStatusCount 40 | *out = make(map[string]batchv1alpha1.TaskState, len(*in)) 41 | for key, val := range *in { 42 | (*out)[key] = *val.DeepCopy() 43 | } 44 | } 45 | } 46 | 47 | // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Condition. 48 | func (in *Condition) DeepCopy() *Condition { 49 | if in == nil { 50 | return nil 51 | } 52 | out := new(Condition) 53 | in.DeepCopyInto(out) 54 | return out 55 | } 56 | 57 | // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. 58 | func (in *DependsOn) DeepCopyInto(out *DependsOn) { 59 | *out = *in 60 | if in.Targets != nil { 61 | in, out := &in.Targets, &out.Targets 62 | *out = make([]string, len(*in)) 63 | copy(*out, *in) 64 | } 65 | if in.Probe != nil { 66 | in, out := &in.Probe, &out.Probe 67 | *out = new(Probe) 68 | (*in).DeepCopyInto(*out) 69 | } 70 | } 71 | 72 | // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DependsOn. 73 | func (in *DependsOn) DeepCopy() *DependsOn { 74 | if in == nil { 75 | return nil 76 | } 77 | out := new(DependsOn) 78 | in.DeepCopyInto(out) 79 | return out 80 | } 81 | 82 | // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. 83 | func (in *Flow) DeepCopyInto(out *Flow) { 84 | *out = *in 85 | if in.DependsOn != nil { 86 | in, out := &in.DependsOn, &out.DependsOn 87 | *out = new(DependsOn) 88 | (*in).DeepCopyInto(*out) 89 | } 90 | } 91 | 92 | // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Flow. 93 | func (in *Flow) DeepCopy() *Flow { 94 | if in == nil { 95 | return nil 96 | } 97 | out := new(Flow) 98 | in.DeepCopyInto(out) 99 | return out 100 | } 101 | 102 | // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. 103 | func (in *HttpGet) DeepCopyInto(out *HttpGet) { 104 | *out = *in 105 | out.HTTPHeader = in.HTTPHeader 106 | } 107 | 108 | // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new HttpGet. 109 | func (in *HttpGet) DeepCopy() *HttpGet { 110 | if in == nil { 111 | return nil 112 | } 113 | out := new(HttpGet) 114 | in.DeepCopyInto(out) 115 | return out 116 | } 117 | 118 | // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. 119 | func (in *JobFlow) DeepCopyInto(out *JobFlow) { 120 | *out = *in 121 | out.TypeMeta = in.TypeMeta 122 | in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) 123 | in.Spec.DeepCopyInto(&out.Spec) 124 | in.Status.DeepCopyInto(&out.Status) 125 | } 126 | 127 | // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new JobFlow. 128 | func (in *JobFlow) DeepCopy() *JobFlow { 129 | if in == nil { 130 | return nil 131 | } 132 | out := new(JobFlow) 133 | in.DeepCopyInto(out) 134 | return out 135 | } 136 | 137 | // DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. 138 | func (in *JobFlow) DeepCopyObject() runtime.Object { 139 | if c := in.DeepCopy(); c != nil { 140 | return c 141 | } 142 | return nil 143 | } 144 | 145 | // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. 146 | func (in *JobFlowList) DeepCopyInto(out *JobFlowList) { 147 | *out = *in 148 | out.TypeMeta = in.TypeMeta 149 | in.ListMeta.DeepCopyInto(&out.ListMeta) 150 | if in.Items != nil { 151 | in, out := &in.Items, &out.Items 152 | *out = make([]JobFlow, len(*in)) 153 | for i := range *in { 154 | (*in)[i].DeepCopyInto(&(*out)[i]) 155 | } 156 | } 157 | } 158 | 159 | // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new JobFlowList. 160 | func (in *JobFlowList) DeepCopy() *JobFlowList { 161 | if in == nil { 162 | return nil 163 | } 164 | out := new(JobFlowList) 165 | in.DeepCopyInto(out) 166 | return out 167 | } 168 | 169 | // DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. 170 | func (in *JobFlowList) DeepCopyObject() runtime.Object { 171 | if c := in.DeepCopy(); c != nil { 172 | return c 173 | } 174 | return nil 175 | } 176 | 177 | // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. 178 | func (in *JobFlowSpec) DeepCopyInto(out *JobFlowSpec) { 179 | *out = *in 180 | if in.Flows != nil { 181 | in, out := &in.Flows, &out.Flows 182 | *out = make([]Flow, len(*in)) 183 | for i := range *in { 184 | (*in)[i].DeepCopyInto(&(*out)[i]) 185 | } 186 | } 187 | } 188 | 189 | // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new JobFlowSpec. 190 | func (in *JobFlowSpec) DeepCopy() *JobFlowSpec { 191 | if in == nil { 192 | return nil 193 | } 194 | out := new(JobFlowSpec) 195 | in.DeepCopyInto(out) 196 | return out 197 | } 198 | 199 | // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. 200 | func (in *JobFlowStatus) DeepCopyInto(out *JobFlowStatus) { 201 | *out = *in 202 | if in.PendingJobs != nil { 203 | in, out := &in.PendingJobs, &out.PendingJobs 204 | *out = make([]string, len(*in)) 205 | copy(*out, *in) 206 | } 207 | if in.RunningJobs != nil { 208 | in, out := &in.RunningJobs, &out.RunningJobs 209 | *out = make([]string, len(*in)) 210 | copy(*out, *in) 211 | } 212 | if in.FailedJobs != nil { 213 | in, out := &in.FailedJobs, &out.FailedJobs 214 | *out = make([]string, len(*in)) 215 | copy(*out, *in) 216 | } 217 | if in.CompletedJobs != nil { 218 | in, out := &in.CompletedJobs, &out.CompletedJobs 219 | *out = make([]string, len(*in)) 220 | copy(*out, *in) 221 | } 222 | if in.TerminatedJobs != nil { 223 | in, out := &in.TerminatedJobs, &out.TerminatedJobs 224 | *out = make([]string, len(*in)) 225 | copy(*out, *in) 226 | } 227 | if in.UnKnowJobs != nil { 228 | in, out := &in.UnKnowJobs, &out.UnKnowJobs 229 | *out = make([]string, len(*in)) 230 | copy(*out, *in) 231 | } 232 | if in.JobStatusList != nil { 233 | in, out := &in.JobStatusList, &out.JobStatusList 234 | *out = make([]JobStatus, len(*in)) 235 | for i := range *in { 236 | (*in)[i].DeepCopyInto(&(*out)[i]) 237 | } 238 | } 239 | if in.Conditions != nil { 240 | in, out := &in.Conditions, &out.Conditions 241 | *out = make(map[string]Condition, len(*in)) 242 | for key, val := range *in { 243 | (*out)[key] = *val.DeepCopy() 244 | } 245 | } 246 | out.State = in.State 247 | } 248 | 249 | // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new JobFlowStatus. 250 | func (in *JobFlowStatus) DeepCopy() *JobFlowStatus { 251 | if in == nil { 252 | return nil 253 | } 254 | out := new(JobFlowStatus) 255 | in.DeepCopyInto(out) 256 | return out 257 | } 258 | 259 | // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. 260 | func (in *JobRunningHistory) DeepCopyInto(out *JobRunningHistory) { 261 | *out = *in 262 | in.StartTimestamp.DeepCopyInto(&out.StartTimestamp) 263 | in.EndTimestamp.DeepCopyInto(&out.EndTimestamp) 264 | } 265 | 266 | // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new JobRunningHistory. 267 | func (in *JobRunningHistory) DeepCopy() *JobRunningHistory { 268 | if in == nil { 269 | return nil 270 | } 271 | out := new(JobRunningHistory) 272 | in.DeepCopyInto(out) 273 | return out 274 | } 275 | 276 | // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. 277 | func (in *JobStatus) DeepCopyInto(out *JobStatus) { 278 | *out = *in 279 | in.StartTimestamp.DeepCopyInto(&out.StartTimestamp) 280 | in.EndTimestamp.DeepCopyInto(&out.EndTimestamp) 281 | if in.RunningHistories != nil { 282 | in, out := &in.RunningHistories, &out.RunningHistories 283 | *out = make([]JobRunningHistory, len(*in)) 284 | for i := range *in { 285 | (*in)[i].DeepCopyInto(&(*out)[i]) 286 | } 287 | } 288 | } 289 | 290 | // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new JobStatus. 291 | func (in *JobStatus) DeepCopy() *JobStatus { 292 | if in == nil { 293 | return nil 294 | } 295 | out := new(JobStatus) 296 | in.DeepCopyInto(out) 297 | return out 298 | } 299 | 300 | // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. 301 | func (in *JobTemplate) DeepCopyInto(out *JobTemplate) { 302 | *out = *in 303 | out.TypeMeta = in.TypeMeta 304 | in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) 305 | in.Spec.DeepCopyInto(&out.Spec) 306 | in.Status.DeepCopyInto(&out.Status) 307 | } 308 | 309 | // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new JobTemplate. 310 | func (in *JobTemplate) DeepCopy() *JobTemplate { 311 | if in == nil { 312 | return nil 313 | } 314 | out := new(JobTemplate) 315 | in.DeepCopyInto(out) 316 | return out 317 | } 318 | 319 | // DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. 320 | func (in *JobTemplate) DeepCopyObject() runtime.Object { 321 | if c := in.DeepCopy(); c != nil { 322 | return c 323 | } 324 | return nil 325 | } 326 | 327 | // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. 328 | func (in *JobTemplateList) DeepCopyInto(out *JobTemplateList) { 329 | *out = *in 330 | out.TypeMeta = in.TypeMeta 331 | in.ListMeta.DeepCopyInto(&out.ListMeta) 332 | if in.Items != nil { 333 | in, out := &in.Items, &out.Items 334 | *out = make([]JobTemplate, len(*in)) 335 | for i := range *in { 336 | (*in)[i].DeepCopyInto(&(*out)[i]) 337 | } 338 | } 339 | } 340 | 341 | // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new JobTemplateList. 342 | func (in *JobTemplateList) DeepCopy() *JobTemplateList { 343 | if in == nil { 344 | return nil 345 | } 346 | out := new(JobTemplateList) 347 | in.DeepCopyInto(out) 348 | return out 349 | } 350 | 351 | // DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. 352 | func (in *JobTemplateList) DeepCopyObject() runtime.Object { 353 | if c := in.DeepCopy(); c != nil { 354 | return c 355 | } 356 | return nil 357 | } 358 | 359 | // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. 360 | func (in *JobTemplateSpec) DeepCopyInto(out *JobTemplateSpec) { 361 | *out = *in 362 | in.JobSpec.DeepCopyInto(&out.JobSpec) 363 | } 364 | 365 | // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new JobTemplateSpec. 366 | func (in *JobTemplateSpec) DeepCopy() *JobTemplateSpec { 367 | if in == nil { 368 | return nil 369 | } 370 | out := new(JobTemplateSpec) 371 | in.DeepCopyInto(out) 372 | return out 373 | } 374 | 375 | // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. 376 | func (in *JobTemplateStatus) DeepCopyInto(out *JobTemplateStatus) { 377 | *out = *in 378 | if in.JobDependsOnList != nil { 379 | in, out := &in.JobDependsOnList, &out.JobDependsOnList 380 | *out = make([]string, len(*in)) 381 | copy(*out, *in) 382 | } 383 | } 384 | 385 | // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new JobTemplateStatus. 386 | func (in *JobTemplateStatus) DeepCopy() *JobTemplateStatus { 387 | if in == nil { 388 | return nil 389 | } 390 | out := new(JobTemplateStatus) 391 | in.DeepCopyInto(out) 392 | return out 393 | } 394 | 395 | // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. 396 | func (in *Probe) DeepCopyInto(out *Probe) { 397 | *out = *in 398 | if in.HttpGetList != nil { 399 | in, out := &in.HttpGetList, &out.HttpGetList 400 | *out = make([]HttpGet, len(*in)) 401 | copy(*out, *in) 402 | } 403 | if in.TcpSocketList != nil { 404 | in, out := &in.TcpSocketList, &out.TcpSocketList 405 | *out = make([]TcpSocket, len(*in)) 406 | copy(*out, *in) 407 | } 408 | if in.TaskStatusList != nil { 409 | in, out := &in.TaskStatusList, &out.TaskStatusList 410 | *out = make([]TaskStatus, len(*in)) 411 | copy(*out, *in) 412 | } 413 | } 414 | 415 | // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Probe. 416 | func (in *Probe) DeepCopy() *Probe { 417 | if in == nil { 418 | return nil 419 | } 420 | out := new(Probe) 421 | in.DeepCopyInto(out) 422 | return out 423 | } 424 | 425 | // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. 426 | func (in *State) DeepCopyInto(out *State) { 427 | *out = *in 428 | } 429 | 430 | // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new State. 431 | func (in *State) DeepCopy() *State { 432 | if in == nil { 433 | return nil 434 | } 435 | out := new(State) 436 | in.DeepCopyInto(out) 437 | return out 438 | } 439 | 440 | // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. 441 | func (in *TaskStatus) DeepCopyInto(out *TaskStatus) { 442 | *out = *in 443 | } 444 | 445 | // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TaskStatus. 446 | func (in *TaskStatus) DeepCopy() *TaskStatus { 447 | if in == nil { 448 | return nil 449 | } 450 | out := new(TaskStatus) 451 | in.DeepCopyInto(out) 452 | return out 453 | } 454 | 455 | // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. 456 | func (in *TcpSocket) DeepCopyInto(out *TcpSocket) { 457 | *out = *in 458 | } 459 | 460 | // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TcpSocket. 461 | func (in *TcpSocket) DeepCopy() *TcpSocket { 462 | if in == nil { 463 | return nil 464 | } 465 | out := new(TcpSocket) 466 | in.DeepCopyInto(out) 467 | return out 468 | } 469 | -------------------------------------------------------------------------------- /config/certmanager/certificate.yaml: -------------------------------------------------------------------------------- 1 | # The following manifests contain a self-signed issuer CR and a certificate CR. 2 | # More document can be found at https://docs.cert-manager.io 3 | # WARNING: Targets CertManager 0.11 check https://docs.cert-manager.io/en/latest/tasks/upgrading/index.html for 4 | # breaking changes 5 | apiVersion: cert-manager.io/v1alpha2 6 | kind: Issuer 7 | metadata: 8 | name: selfsigned-issuer 9 | namespace: system 10 | spec: 11 | selfSigned: {} 12 | --- 13 | apiVersion: cert-manager.io/v1alpha2 14 | kind: Certificate 15 | metadata: 16 | name: serving-cert # this name should match the one appeared in kustomizeconfig.yaml 17 | namespace: system 18 | spec: 19 | # $(SERVICE_NAME) and $(SERVICE_NAMESPACE) will be substituted by kustomize 20 | dnsNames: 21 | - $(SERVICE_NAME).$(SERVICE_NAMESPACE).svc 22 | - $(SERVICE_NAME).$(SERVICE_NAMESPACE).svc.cluster.local 23 | issuerRef: 24 | kind: Issuer 25 | name: selfsigned-issuer 26 | secretName: webhook-server-cert # this secret will not be prefixed, since it's not managed by kustomize 27 | -------------------------------------------------------------------------------- /config/certmanager/kustomization.yaml: -------------------------------------------------------------------------------- 1 | resources: 2 | - certificate.yaml 3 | 4 | configurations: 5 | - kustomizeconfig.yaml 6 | -------------------------------------------------------------------------------- /config/certmanager/kustomizeconfig.yaml: -------------------------------------------------------------------------------- 1 | # This configuration is for teaching kustomize how to update name ref and var substitution 2 | nameReference: 3 | - kind: Issuer 4 | group: cert-manager.io 5 | fieldSpecs: 6 | - kind: Certificate 7 | group: cert-manager.io 8 | path: spec/issuerRef/name 9 | 10 | varReference: 11 | - kind: Certificate 12 | group: cert-manager.io 13 | path: spec/commonName 14 | - kind: Certificate 15 | group: cert-manager.io 16 | path: spec/dnsNames 17 | -------------------------------------------------------------------------------- /config/crd/bases/flow.volcano.sh_jobflows.yaml: -------------------------------------------------------------------------------- 1 | 2 | --- 3 | apiVersion: apiextensions.k8s.io/v1 4 | kind: CustomResourceDefinition 5 | metadata: 6 | annotations: 7 | controller-gen.kubebuilder.io/version: v0.4.1 8 | creationTimestamp: null 9 | name: jobflows.flow.volcano.sh 10 | spec: 11 | group: flow.volcano.sh 12 | names: 13 | kind: JobFlow 14 | listKind: JobFlowList 15 | plural: jobflows 16 | shortNames: 17 | - jf 18 | singular: jobflow 19 | scope: Namespaced 20 | versions: 21 | - additionalPrinterColumns: 22 | - jsonPath: .status.state.phase 23 | name: Status 24 | type: string 25 | - jsonPath: .metadata.creationTimestamp 26 | name: Age 27 | type: date 28 | name: v1alpha1 29 | schema: 30 | openAPIV3Schema: 31 | description: JobFlow is the Schema for the jobflows API 32 | properties: 33 | apiVersion: 34 | description: 'APIVersion defines the versioned schema of this representation 35 | of an object. Servers should convert recognized schemas to the latest 36 | internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' 37 | type: string 38 | kind: 39 | description: 'Kind is a string value representing the REST resource this 40 | object represents. Servers may infer this from the endpoint the client 41 | submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' 42 | type: string 43 | metadata: 44 | type: object 45 | spec: 46 | description: JobFlowSpec defines the desired state of JobFlow 47 | properties: 48 | flows: 49 | description: Foo is an example field of JobFlow. Edit jobflow_types.go 50 | to remove/update 51 | items: 52 | description: Flow defines the dependent of jobs 53 | properties: 54 | dependsOn: 55 | properties: 56 | probe: 57 | properties: 58 | httpGetList: 59 | items: 60 | properties: 61 | httpHeader: 62 | description: HTTPHeader describes a custom header 63 | to be used in HTTP probes 64 | properties: 65 | name: 66 | description: The header field name 67 | type: string 68 | value: 69 | description: The header field value 70 | type: string 71 | required: 72 | - name 73 | - value 74 | type: object 75 | path: 76 | type: string 77 | port: 78 | type: integer 79 | taskName: 80 | type: string 81 | type: object 82 | type: array 83 | taskStatusList: 84 | items: 85 | properties: 86 | phase: 87 | type: string 88 | taskName: 89 | type: string 90 | type: object 91 | type: array 92 | tcpSocketList: 93 | items: 94 | properties: 95 | port: 96 | type: integer 97 | taskName: 98 | type: string 99 | required: 100 | - port 101 | type: object 102 | type: array 103 | type: object 104 | targets: 105 | items: 106 | type: string 107 | type: array 108 | type: object 109 | name: 110 | type: string 111 | required: 112 | - name 113 | type: object 114 | type: array 115 | jobRetainPolicy: 116 | type: string 117 | type: object 118 | status: 119 | description: JobFlowStatus defines the observed state of JobFlow 120 | properties: 121 | completedJobs: 122 | items: 123 | type: string 124 | type: array 125 | conditions: 126 | additionalProperties: 127 | properties: 128 | createTime: 129 | format: date-time 130 | type: string 131 | phase: 132 | description: JobPhase defines the phase of the job. 133 | type: string 134 | runningDuration: 135 | type: string 136 | taskStatusCount: 137 | additionalProperties: 138 | description: TaskState contains details for the current state 139 | of the task. 140 | properties: 141 | phase: 142 | additionalProperties: 143 | format: int32 144 | type: integer 145 | description: The phase of Task. 146 | type: object 147 | type: object 148 | type: object 149 | type: object 150 | type: object 151 | failedJobs: 152 | items: 153 | type: string 154 | type: array 155 | jobStatusList: 156 | items: 157 | properties: 158 | endTimestamp: 159 | format: date-time 160 | type: string 161 | name: 162 | type: string 163 | restartCount: 164 | format: int32 165 | type: integer 166 | runningHistories: 167 | items: 168 | properties: 169 | endTimestamp: 170 | format: date-time 171 | type: string 172 | startTimestamp: 173 | format: date-time 174 | type: string 175 | state: 176 | description: JobPhase defines the phase of the job. 177 | type: string 178 | type: object 179 | type: array 180 | startTimestamp: 181 | format: date-time 182 | type: string 183 | state: 184 | description: JobPhase defines the phase of the job. 185 | type: string 186 | type: object 187 | type: array 188 | pendingJobs: 189 | items: 190 | type: string 191 | type: array 192 | runningJobs: 193 | items: 194 | type: string 195 | type: array 196 | state: 197 | properties: 198 | phase: 199 | type: string 200 | type: object 201 | terminatedJobs: 202 | items: 203 | type: string 204 | type: array 205 | unKnowJobs: 206 | items: 207 | type: string 208 | type: array 209 | type: object 210 | type: object 211 | served: true 212 | storage: true 213 | subresources: 214 | status: {} 215 | status: 216 | acceptedNames: 217 | kind: "" 218 | plural: "" 219 | conditions: [] 220 | storedVersions: [] 221 | -------------------------------------------------------------------------------- /config/crd/kustomization.yaml: -------------------------------------------------------------------------------- 1 | # This kustomization.yaml is not intended to be run by itself, 2 | # since it depends on service name and namespace that are out of this kustomize package. 3 | # It should be run by config/default 4 | resources: 5 | - bases/flow.volcano.sh_jobflows.yaml 6 | - bases/flow.volcano.sh_jobtemplates.yaml 7 | #+kubebuilder:scaffold:crdkustomizeresource 8 | 9 | patchesStrategicMerge: 10 | # [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix. 11 | # patches here are for enabling the conversion webhook for each CRD 12 | #- patches/webhook_in_jobflows.yaml 13 | #- patches/webhook_in_jobtemplates.yaml 14 | #+kubebuilder:scaffold:crdkustomizewebhookpatch 15 | 16 | # [CERTMANAGER] To enable webhook, uncomment all the sections with [CERTMANAGER] prefix. 17 | # patches here are for enabling the CA injection for each CRD 18 | #- patches/cainjection_in_jobflows.yaml 19 | #- patches/cainjection_in_jobtemplates.yaml 20 | #+kubebuilder:scaffold:crdkustomizecainjectionpatch 21 | 22 | # the following config is for teaching kustomize how to do kustomization for CRDs. 23 | configurations: 24 | - kustomizeconfig.yaml 25 | -------------------------------------------------------------------------------- /config/crd/kustomizeconfig.yaml: -------------------------------------------------------------------------------- 1 | # This file is for teaching kustomize how to substitute name and namespace reference in CRD 2 | nameReference: 3 | - kind: Service 4 | version: v1 5 | fieldSpecs: 6 | - kind: CustomResourceDefinition 7 | version: v1 8 | group: apiextensions.k8s.io 9 | path: spec/conversion/webhook/clientConfig/service/name 10 | 11 | namespace: 12 | - kind: CustomResourceDefinition 13 | version: v1 14 | group: apiextensions.k8s.io 15 | path: spec/conversion/webhook/clientConfig/service/namespace 16 | create: false 17 | 18 | varReference: 19 | - path: metadata/annotations 20 | -------------------------------------------------------------------------------- /config/crd/patches/cainjection_in_jobflows.yaml: -------------------------------------------------------------------------------- 1 | # The following patch adds a directive for certmanager to inject CA into the CRD 2 | apiVersion: apiextensions.k8s.io/v1 3 | kind: CustomResourceDefinition 4 | metadata: 5 | annotations: 6 | cert-manager.io/inject-ca-from: $(CERTIFICATE_NAMESPACE)/$(CERTIFICATE_NAME) 7 | name: jobflows.flow.volcano.sh 8 | -------------------------------------------------------------------------------- /config/crd/patches/cainjection_in_jobtemplates.yaml: -------------------------------------------------------------------------------- 1 | # The following patch adds a directive for certmanager to inject CA into the CRD 2 | apiVersion: apiextensions.k8s.io/v1 3 | kind: CustomResourceDefinition 4 | metadata: 5 | annotations: 6 | cert-manager.io/inject-ca-from: $(CERTIFICATE_NAMESPACE)/$(CERTIFICATE_NAME) 7 | name: jobtemplates.flow.volcano.sh 8 | -------------------------------------------------------------------------------- /config/crd/patches/webhook_in_jobflows.yaml: -------------------------------------------------------------------------------- 1 | # The following patch enables a conversion webhook for the CRD 2 | apiVersion: apiextensions.k8s.io/v1 3 | kind: CustomResourceDefinition 4 | metadata: 5 | name: jobflows.flow.volcano.sh 6 | spec: 7 | conversion: 8 | strategy: Webhook 9 | webhook: 10 | clientConfig: 11 | service: 12 | namespace: system 13 | name: webhook-service 14 | path: /convert 15 | conversionReviewVersions: 16 | - v1 17 | -------------------------------------------------------------------------------- /config/crd/patches/webhook_in_jobtemplates.yaml: -------------------------------------------------------------------------------- 1 | # The following patch enables a conversion webhook for the CRD 2 | apiVersion: apiextensions.k8s.io/v1 3 | kind: CustomResourceDefinition 4 | metadata: 5 | name: jobtemplates.flow.volcano.sh 6 | spec: 7 | conversion: 8 | strategy: Webhook 9 | webhook: 10 | clientConfig: 11 | service: 12 | namespace: system 13 | name: webhook-service 14 | path: /convert 15 | conversionReviewVersions: 16 | - v1 17 | -------------------------------------------------------------------------------- /config/default/kustomization.yaml: -------------------------------------------------------------------------------- 1 | # Adds namespace to all resources. 2 | namespace: kube-system 3 | 4 | # Value of this field is prepended to the 5 | # names of all resources, e.g. a deployment named 6 | # "wordpress" becomes "alices-wordpress". 7 | # Note that it should also match with the prefix (text before '-') of the namespace 8 | # field above. 9 | namePrefix: jobflow- 10 | 11 | # Labels to add to all resources and selectors. 12 | #commonLabels: 13 | # someName: someValue 14 | 15 | bases: 16 | - ../crd 17 | - ../rbac 18 | - ../manager 19 | # [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix including the one in 20 | # crd/kustomization.yaml 21 | - ../webhook 22 | # [CERTMANAGER] To enable cert-manager, uncomment all sections with 'CERTMANAGER'. 'WEBHOOK' components are required. 23 | # - ../certmanager 24 | # [PROMETHEUS] To enable prometheus monitor, uncomment all sections with 'PROMETHEUS'. 25 | #- ../prometheus 26 | 27 | patchesStrategicMerge: 28 | # Protect the /metrics endpoint by putting it behind auth. 29 | # If you want your controller-manager to expose the /metrics 30 | # endpoint w/o any authn/z, please comment the following line. 31 | - manager_auth_proxy_patch.yaml 32 | 33 | # [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix including the one in 34 | # crd/kustomization.yaml 35 | - manager_webhook_patch.yaml 36 | 37 | # [CERTMANAGER] To enable cert-manager, uncomment all sections with 'CERTMANAGER'. 38 | # Uncomment 'CERTMANAGER' sections in crd/kustomization.yaml to enable the CA injection in the admission webhooks. 39 | # 'CERTMANAGER' needs to be enabled to use ca injection 40 | # - webhookcainjection_patch.yaml 41 | 42 | # the following config is for teaching kustomize how to do var substitution 43 | #vars: 44 | # [CERTMANAGER] To enable cert-manager, uncomment all sections with 'CERTMANAGER' prefix. 45 | # - name: CERTIFICATE_NAMESPACE # namespace of the certificate CR 46 | # objref: 47 | # kind: Certificate 48 | # group: cert-manager.io 49 | # version: v1alpha2 50 | # name: serving-cert # this name should match the one in certificate.yaml 51 | # fieldref: 52 | # fieldpath: metadata.namespace 53 | # - name: CERTIFICATE_NAME 54 | # objref: 55 | # kind: Certificate 56 | # group: cert-manager.io 57 | # version: v1alpha2 58 | # name: serving-cert # this name should match the one in certificate.yaml 59 | # - name: SERVICE_NAMESPACE # namespace of the service 60 | # objref: 61 | # kind: Service 62 | # version: v1 63 | # name: webhook-service 64 | # fieldref: 65 | # fieldpath: metadata.namespace 66 | # - name: SERVICE_NAME 67 | # objref: 68 | # kind: Service 69 | # version: v1 70 | # name: webhook-service 71 | -------------------------------------------------------------------------------- /config/default/manager_auth_proxy_patch.yaml: -------------------------------------------------------------------------------- 1 | # This patch inject a sidecar container which is a HTTP proxy for the 2 | # controller manager, it performs RBAC authorization against the Kubernetes API using SubjectAccessReviews. 3 | apiVersion: apps/v1 4 | kind: Deployment 5 | metadata: 6 | name: controller-manager 7 | namespace: system 8 | spec: 9 | template: 10 | spec: 11 | containers: 12 | # - name: kube-rbac-proxy 13 | # image: gcr.io/kubebuilder/kube-rbac-proxy:v0.8.0 14 | # args: 15 | # - "--secure-listen-address=0.0.0.0:8443" 16 | # - "--upstream=http://127.0.0.1:8080/" 17 | # - "--logtostderr=true" 18 | # - "--v=10" 19 | # ports: 20 | # - containerPort: 8443 21 | # name: https 22 | - name: manager 23 | args: 24 | - "--health-probe-bind-address=:8081" 25 | - "--metrics-bind-address=127.0.0.1:8080" 26 | - "--leader-elect" 27 | -------------------------------------------------------------------------------- /config/default/manager_config_patch.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: apps/v1 2 | kind: Deployment 3 | metadata: 4 | name: controller-manager 5 | namespace: system 6 | spec: 7 | template: 8 | spec: 9 | containers: 10 | - name: manager 11 | args: 12 | - "--config=controller_manager_config.yaml" 13 | volumeMounts: 14 | - name: manager-config 15 | mountPath: /controller_manager_config.yaml 16 | subPath: controller_manager_config.yaml 17 | volumes: 18 | - name: manager-config 19 | configMap: 20 | name: manager-config 21 | -------------------------------------------------------------------------------- /config/default/manager_webhook_patch.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: apps/v1 2 | kind: Deployment 3 | metadata: 4 | name: controller-manager 5 | namespace: system 6 | spec: 7 | template: 8 | spec: 9 | containers: 10 | - name: manager 11 | ports: 12 | - containerPort: 8725 13 | name: webhook-server 14 | protocol: TCP 15 | volumeMounts: 16 | - mountPath: /tmp/k8s-webhook-server/serving-certs 17 | name: cert 18 | readOnly: true 19 | volumes: 20 | - name: cert 21 | secret: 22 | defaultMode: 420 23 | secretName: jobflow-webhook-server-cert 24 | -------------------------------------------------------------------------------- /config/default/webhookcainjection_patch.yaml: -------------------------------------------------------------------------------- 1 | # This patch add annotation to admission webhook config and 2 | # the variables $(CERTIFICATE_NAMESPACE) and $(CERTIFICATE_NAME) will be substituted by kustomize. 3 | apiVersion: admissionregistration.k8s.io/v1 4 | kind: MutatingWebhookConfiguration 5 | metadata: 6 | name: mutating-webhook-configuration 7 | webhooks: 8 | - clientConfig: 9 | caBundle: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSURSVENDQWkyZ0F3SUJBZ0lVRVVJaXJ0R05KS1FzMGZldXFPMExRRFlvSzN3d0RRWUpLb1pJaHZjTkFRRUwKQlFBd01qRXdNQzRHQTFVRUF3d25hbTlpWm14dmR5MTNaV0pvYjI5ckxYTmxjblpwWTJVdWEzVmlaUzF6ZVhOMApaVzB1YzNaak1CNFhEVEl4TVRJeU1ERXdNamN6TmxvWERUTXhNVEl4T0RFd01qY3pObG93TWpFd01DNEdBMVVFCkF3d25hbTlpWm14dmR5MTNaV0pvYjI5ckxYTmxjblpwWTJVdWEzVmlaUzF6ZVhOMFpXMHVjM1pqTUlJQklqQU4KQmdrcWhraUc5dzBCQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBelg5Q09NUk8xMUF6aCt2eEYrOHVrK2IyUjVqcgpLZ014TmcrNDVONkdEeWp5bFFkNmdPSUJQUE5pd2NrRFQzYlduNWlacG1Ib3JEVkZ0cGRxS1R2T0ROSVdFWWFYCnZRc3dMY0wrUGoxTHhFWTc4T3pkTE95OURCTkF3eDdIWkNYR2U2UzhlWDZqOWJ1T2NOMVhzckFoYllQdFlyNTQKdkhkM2NWTU5VS09nYWltUDRNSFJ6ZnBTNi9OVzNnZ1c3bHNVVjRvM2N0WFRtbTVDRFdwMzhnODNpVWFYZlJIMgo2S045V3hZbGp6YzRFd0N2WHE3eTZaTDBZdEZOQW9DZ3RFRGhsOG5Dc1RoSEN0Uk04OS9keVBpb1ZJN2ZCNWVOCnlwZnlNWjJVcVlZSFFFdXFKa3NmNGZNR2x4dFRSSDBkYXJ5MCtkR0FBVWY4bi9xYzVjNUNBRFI4SlFJREFRQUIKbzFNd1VUQWRCZ05WSFE0RUZnUVVlRXdlbFVLZStYNXRLamlpREkzVFRGVGdHZFl3SHdZRFZSMGpCQmd3Rm9BVQplRXdlbFVLZStYNXRLamlpREkzVFRGVGdHZFl3RHdZRFZSMFRBUUgvQkFVd0F3RUIvekFOQmdrcWhraUc5dzBCCkFRc0ZBQU9DQVFFQWFFalFWN1I5SHJpby9VZVRqQXBRK1JXQ2NNWVJWNXIrU25RNGV5cVBIOHlZQjMxNmdKazkKL01ZNXV6RTJhK0NkSUdHQmRtQzgvRHBPTDdUTnZSNXlCYkNNZmRwMzdMTXlSNlVJRXEvTmVUWTBYRi9WcFFPMApqSEI0ZHMwVGFBaGJwN3pobFRsYytjK3JyenlqWXJ1N3VYMHg1ekhFUkhFRFpHRjRCVThlcTcyOVBxcWMxbTdSCjlwMWZVMFF5b3NKZXJhVmt6czJYUHgzZmV2bXZHMm55dlM0UGFNdVRXbXltMkpnZndqZmduUG5SaUNvL3R5d1YKWks1a2hUQTZ6VGZaZzJnaUt0Z2JuN1NhcWtsNVFOcHNaaFU2NjNkN0dIQmgyZThpV1ZhQXNXK0xvV2xLNFFtLwprNmplQktZbzRvZHJwSXdUNEt4OTdYSzYzcklnUSt3NklRPT0KLS0tLS1FTkQgQ0VSVElGSUNBVEUtLS0tLQo= 10 | name: mjobtemplate.kb.io 11 | - clientConfig: 12 | caBundle: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSURSVENDQWkyZ0F3SUJBZ0lVRVVJaXJ0R05KS1FzMGZldXFPMExRRFlvSzN3d0RRWUpLb1pJaHZjTkFRRUwKQlFBd01qRXdNQzRHQTFVRUF3d25hbTlpWm14dmR5MTNaV0pvYjI5ckxYTmxjblpwWTJVdWEzVmlaUzF6ZVhOMApaVzB1YzNaak1CNFhEVEl4TVRJeU1ERXdNamN6TmxvWERUTXhNVEl4T0RFd01qY3pObG93TWpFd01DNEdBMVVFCkF3d25hbTlpWm14dmR5MTNaV0pvYjI5ckxYTmxjblpwWTJVdWEzVmlaUzF6ZVhOMFpXMHVjM1pqTUlJQklqQU4KQmdrcWhraUc5dzBCQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBelg5Q09NUk8xMUF6aCt2eEYrOHVrK2IyUjVqcgpLZ014TmcrNDVONkdEeWp5bFFkNmdPSUJQUE5pd2NrRFQzYlduNWlacG1Ib3JEVkZ0cGRxS1R2T0ROSVdFWWFYCnZRc3dMY0wrUGoxTHhFWTc4T3pkTE95OURCTkF3eDdIWkNYR2U2UzhlWDZqOWJ1T2NOMVhzckFoYllQdFlyNTQKdkhkM2NWTU5VS09nYWltUDRNSFJ6ZnBTNi9OVzNnZ1c3bHNVVjRvM2N0WFRtbTVDRFdwMzhnODNpVWFYZlJIMgo2S045V3hZbGp6YzRFd0N2WHE3eTZaTDBZdEZOQW9DZ3RFRGhsOG5Dc1RoSEN0Uk04OS9keVBpb1ZJN2ZCNWVOCnlwZnlNWjJVcVlZSFFFdXFKa3NmNGZNR2x4dFRSSDBkYXJ5MCtkR0FBVWY4bi9xYzVjNUNBRFI4SlFJREFRQUIKbzFNd1VUQWRCZ05WSFE0RUZnUVVlRXdlbFVLZStYNXRLamlpREkzVFRGVGdHZFl3SHdZRFZSMGpCQmd3Rm9BVQplRXdlbFVLZStYNXRLamlpREkzVFRGVGdHZFl3RHdZRFZSMFRBUUgvQkFVd0F3RUIvekFOQmdrcWhraUc5dzBCCkFRc0ZBQU9DQVFFQWFFalFWN1I5SHJpby9VZVRqQXBRK1JXQ2NNWVJWNXIrU25RNGV5cVBIOHlZQjMxNmdKazkKL01ZNXV6RTJhK0NkSUdHQmRtQzgvRHBPTDdUTnZSNXlCYkNNZmRwMzdMTXlSNlVJRXEvTmVUWTBYRi9WcFFPMApqSEI0ZHMwVGFBaGJwN3pobFRsYytjK3JyenlqWXJ1N3VYMHg1ekhFUkhFRFpHRjRCVThlcTcyOVBxcWMxbTdSCjlwMWZVMFF5b3NKZXJhVmt6czJYUHgzZmV2bXZHMm55dlM0UGFNdVRXbXltMkpnZndqZmduUG5SaUNvL3R5d1YKWks1a2hUQTZ6VGZaZzJnaUt0Z2JuN1NhcWtsNVFOcHNaaFU2NjNkN0dIQmgyZThpV1ZhQXNXK0xvV2xLNFFtLwprNmplQktZbzRvZHJwSXdUNEt4OTdYSzYzcklnUSt3NklRPT0KLS0tLS1FTkQgQ0VSVElGSUNBVEUtLS0tLQo= 13 | name: mjobflow.kb.io 14 | --- 15 | apiVersion: admissionregistration.k8s.io/v1 16 | kind: ValidatingWebhookConfiguration 17 | metadata: 18 | name: validating-webhook-configuration 19 | webhooks: 20 | - clientConfig: 21 | caBundle: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSURSVENDQWkyZ0F3SUJBZ0lVRVVJaXJ0R05KS1FzMGZldXFPMExRRFlvSzN3d0RRWUpLb1pJaHZjTkFRRUwKQlFBd01qRXdNQzRHQTFVRUF3d25hbTlpWm14dmR5MTNaV0pvYjI5ckxYTmxjblpwWTJVdWEzVmlaUzF6ZVhOMApaVzB1YzNaak1CNFhEVEl4TVRJeU1ERXdNamN6TmxvWERUTXhNVEl4T0RFd01qY3pObG93TWpFd01DNEdBMVVFCkF3d25hbTlpWm14dmR5MTNaV0pvYjI5ckxYTmxjblpwWTJVdWEzVmlaUzF6ZVhOMFpXMHVjM1pqTUlJQklqQU4KQmdrcWhraUc5dzBCQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBelg5Q09NUk8xMUF6aCt2eEYrOHVrK2IyUjVqcgpLZ014TmcrNDVONkdEeWp5bFFkNmdPSUJQUE5pd2NrRFQzYlduNWlacG1Ib3JEVkZ0cGRxS1R2T0ROSVdFWWFYCnZRc3dMY0wrUGoxTHhFWTc4T3pkTE95OURCTkF3eDdIWkNYR2U2UzhlWDZqOWJ1T2NOMVhzckFoYllQdFlyNTQKdkhkM2NWTU5VS09nYWltUDRNSFJ6ZnBTNi9OVzNnZ1c3bHNVVjRvM2N0WFRtbTVDRFdwMzhnODNpVWFYZlJIMgo2S045V3hZbGp6YzRFd0N2WHE3eTZaTDBZdEZOQW9DZ3RFRGhsOG5Dc1RoSEN0Uk04OS9keVBpb1ZJN2ZCNWVOCnlwZnlNWjJVcVlZSFFFdXFKa3NmNGZNR2x4dFRSSDBkYXJ5MCtkR0FBVWY4bi9xYzVjNUNBRFI4SlFJREFRQUIKbzFNd1VUQWRCZ05WSFE0RUZnUVVlRXdlbFVLZStYNXRLamlpREkzVFRGVGdHZFl3SHdZRFZSMGpCQmd3Rm9BVQplRXdlbFVLZStYNXRLamlpREkzVFRGVGdHZFl3RHdZRFZSMFRBUUgvQkFVd0F3RUIvekFOQmdrcWhraUc5dzBCCkFRc0ZBQU9DQVFFQWFFalFWN1I5SHJpby9VZVRqQXBRK1JXQ2NNWVJWNXIrU25RNGV5cVBIOHlZQjMxNmdKazkKL01ZNXV6RTJhK0NkSUdHQmRtQzgvRHBPTDdUTnZSNXlCYkNNZmRwMzdMTXlSNlVJRXEvTmVUWTBYRi9WcFFPMApqSEI0ZHMwVGFBaGJwN3pobFRsYytjK3JyenlqWXJ1N3VYMHg1ekhFUkhFRFpHRjRCVThlcTcyOVBxcWMxbTdSCjlwMWZVMFF5b3NKZXJhVmt6czJYUHgzZmV2bXZHMm55dlM0UGFNdVRXbXltMkpnZndqZmduUG5SaUNvL3R5d1YKWks1a2hUQTZ6VGZaZzJnaUt0Z2JuN1NhcWtsNVFOcHNaaFU2NjNkN0dIQmgyZThpV1ZhQXNXK0xvV2xLNFFtLwprNmplQktZbzRvZHJwSXdUNEt4OTdYSzYzcklnUSt3NklRPT0KLS0tLS1FTkQgQ0VSVElGSUNBVEUtLS0tLQo= 22 | name: vjobtemplate.kb.io 23 | - clientConfig: 24 | caBundle: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSURSVENDQWkyZ0F3SUJBZ0lVRVVJaXJ0R05KS1FzMGZldXFPMExRRFlvSzN3d0RRWUpLb1pJaHZjTkFRRUwKQlFBd01qRXdNQzRHQTFVRUF3d25hbTlpWm14dmR5MTNaV0pvYjI5ckxYTmxjblpwWTJVdWEzVmlaUzF6ZVhOMApaVzB1YzNaak1CNFhEVEl4TVRJeU1ERXdNamN6TmxvWERUTXhNVEl4T0RFd01qY3pObG93TWpFd01DNEdBMVVFCkF3d25hbTlpWm14dmR5MTNaV0pvYjI5ckxYTmxjblpwWTJVdWEzVmlaUzF6ZVhOMFpXMHVjM1pqTUlJQklqQU4KQmdrcWhraUc5dzBCQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBelg5Q09NUk8xMUF6aCt2eEYrOHVrK2IyUjVqcgpLZ014TmcrNDVONkdEeWp5bFFkNmdPSUJQUE5pd2NrRFQzYlduNWlacG1Ib3JEVkZ0cGRxS1R2T0ROSVdFWWFYCnZRc3dMY0wrUGoxTHhFWTc4T3pkTE95OURCTkF3eDdIWkNYR2U2UzhlWDZqOWJ1T2NOMVhzckFoYllQdFlyNTQKdkhkM2NWTU5VS09nYWltUDRNSFJ6ZnBTNi9OVzNnZ1c3bHNVVjRvM2N0WFRtbTVDRFdwMzhnODNpVWFYZlJIMgo2S045V3hZbGp6YzRFd0N2WHE3eTZaTDBZdEZOQW9DZ3RFRGhsOG5Dc1RoSEN0Uk04OS9keVBpb1ZJN2ZCNWVOCnlwZnlNWjJVcVlZSFFFdXFKa3NmNGZNR2x4dFRSSDBkYXJ5MCtkR0FBVWY4bi9xYzVjNUNBRFI4SlFJREFRQUIKbzFNd1VUQWRCZ05WSFE0RUZnUVVlRXdlbFVLZStYNXRLamlpREkzVFRGVGdHZFl3SHdZRFZSMGpCQmd3Rm9BVQplRXdlbFVLZStYNXRLamlpREkzVFRGVGdHZFl3RHdZRFZSMFRBUUgvQkFVd0F3RUIvekFOQmdrcWhraUc5dzBCCkFRc0ZBQU9DQVFFQWFFalFWN1I5SHJpby9VZVRqQXBRK1JXQ2NNWVJWNXIrU25RNGV5cVBIOHlZQjMxNmdKazkKL01ZNXV6RTJhK0NkSUdHQmRtQzgvRHBPTDdUTnZSNXlCYkNNZmRwMzdMTXlSNlVJRXEvTmVUWTBYRi9WcFFPMApqSEI0ZHMwVGFBaGJwN3pobFRsYytjK3JyenlqWXJ1N3VYMHg1ekhFUkhFRFpHRjRCVThlcTcyOVBxcWMxbTdSCjlwMWZVMFF5b3NKZXJhVmt6czJYUHgzZmV2bXZHMm55dlM0UGFNdVRXbXltMkpnZndqZmduUG5SaUNvL3R5d1YKWks1a2hUQTZ6VGZaZzJnaUt0Z2JuN1NhcWtsNVFOcHNaaFU2NjNkN0dIQmgyZThpV1ZhQXNXK0xvV2xLNFFtLwprNmplQktZbzRvZHJwSXdUNEt4OTdYSzYzcklnUSt3NklRPT0KLS0tLS1FTkQgQ0VSVElGSUNBVEUtLS0tLQo= 25 | name: vjobflow.kb.io -------------------------------------------------------------------------------- /config/manager/controller_manager_config.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: controller-runtime.sigs.k8s.io/v1alpha1 2 | kind: ControllerManagerConfig 3 | health: 4 | healthProbeBindAddress: :8081 5 | metrics: 6 | bindAddress: 127.0.0.1:8080 7 | webhook: 8 | port: 8725 9 | leaderElection: 10 | leaderElect: true 11 | resourceName: 1b1c5f74.volcano.sh 12 | -------------------------------------------------------------------------------- /config/manager/kustomization.yaml: -------------------------------------------------------------------------------- 1 | resources: 2 | - manager.yaml 3 | 4 | generatorOptions: 5 | disableNameSuffixHash: true 6 | 7 | configMapGenerator: 8 | - files: 9 | - controller_manager_config.yaml 10 | name: manager-config 11 | apiVersion: kustomize.config.k8s.io/v1beta1 12 | kind: Kustomization 13 | images: 14 | - name: controller 15 | newName: beyondcent/jobflow 16 | newTag: v0.0.1 17 | -------------------------------------------------------------------------------- /config/manager/manager.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: apps/v1 2 | kind: Deployment 3 | metadata: 4 | name: controller-manager 5 | namespace: system 6 | labels: 7 | control-plane: controller-manager 8 | spec: 9 | selector: 10 | matchLabels: 11 | control-plane: controller-manager 12 | replicas: 1 13 | template: 14 | metadata: 15 | labels: 16 | control-plane: controller-manager 17 | spec: 18 | securityContext: 19 | runAsNonRoot: false 20 | containers: 21 | - command: 22 | - /manager 23 | args: 24 | - --leader-elect 25 | image: controller:latest 26 | name: manager 27 | securityContext: 28 | allowPrivilegeEscalation: false 29 | livenessProbe: 30 | httpGet: 31 | path: /healthz 32 | port: 8081 33 | initialDelaySeconds: 15 34 | periodSeconds: 20 35 | readinessProbe: 36 | httpGet: 37 | path: /readyz 38 | port: 8081 39 | initialDelaySeconds: 5 40 | periodSeconds: 10 41 | resources: 42 | limits: 43 | cpu: 100m 44 | memory: 30Mi 45 | requests: 46 | cpu: 100m 47 | memory: 20Mi 48 | serviceAccountName: controller-manager 49 | terminationGracePeriodSeconds: 10 50 | -------------------------------------------------------------------------------- /config/prometheus/kustomization.yaml: -------------------------------------------------------------------------------- 1 | #resources: 2 | #- monitor.yaml 3 | -------------------------------------------------------------------------------- /config/prometheus/monitor.yaml: -------------------------------------------------------------------------------- 1 | 2 | # Prometheus Monitor Service (Metrics) 3 | apiVersion: monitoring.coreos.com/v1 4 | kind: ServiceMonitor 5 | metadata: 6 | labels: 7 | control-plane: controller-manager 8 | name: controller-manager-metrics-monitor 9 | namespace: system 10 | spec: 11 | endpoints: 12 | - path: /metrics 13 | port: https 14 | scheme: https 15 | bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token 16 | tlsConfig: 17 | insecureSkipVerify: true 18 | selector: 19 | matchLabels: 20 | control-plane: controller-manager 21 | -------------------------------------------------------------------------------- /config/rbac/auth_proxy_client_clusterrole.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: rbac.authorization.k8s.io/v1 2 | kind: ClusterRole 3 | metadata: 4 | name: metrics-reader 5 | rules: 6 | - nonResourceURLs: 7 | - "/metrics" 8 | verbs: 9 | - get 10 | -------------------------------------------------------------------------------- /config/rbac/auth_proxy_role.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: rbac.authorization.k8s.io/v1 2 | kind: ClusterRole 3 | metadata: 4 | name: proxy-role 5 | rules: 6 | - apiGroups: 7 | - authentication.k8s.io 8 | resources: 9 | - tokenreviews 10 | verbs: 11 | - create 12 | - apiGroups: 13 | - authorization.k8s.io 14 | resources: 15 | - subjectaccessreviews 16 | verbs: 17 | - create 18 | -------------------------------------------------------------------------------- /config/rbac/auth_proxy_role_binding.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: rbac.authorization.k8s.io/v1 2 | kind: ClusterRoleBinding 3 | metadata: 4 | name: proxy-rolebinding 5 | roleRef: 6 | apiGroup: rbac.authorization.k8s.io 7 | kind: ClusterRole 8 | name: proxy-role 9 | subjects: 10 | - kind: ServiceAccount 11 | name: controller-manager 12 | namespace: system 13 | -------------------------------------------------------------------------------- /config/rbac/auth_proxy_service.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Service 3 | metadata: 4 | labels: 5 | control-plane: controller-manager 6 | name: controller-manager-metrics-service 7 | namespace: system 8 | spec: 9 | ports: 10 | - name: https 11 | port: 8443 12 | targetPort: https 13 | selector: 14 | control-plane: controller-manager 15 | -------------------------------------------------------------------------------- /config/rbac/jobflow_editor_role.yaml: -------------------------------------------------------------------------------- 1 | # permissions for end users to edit jobflows. 2 | apiVersion: rbac.authorization.k8s.io/v1 3 | kind: ClusterRole 4 | metadata: 5 | name: jobflow-editor-role 6 | rules: 7 | - apiGroups: 8 | - flow.volcano.sh 9 | resources: 10 | - jobflows 11 | verbs: 12 | - create 13 | - delete 14 | - get 15 | - list 16 | - patch 17 | - update 18 | - watch 19 | - apiGroups: 20 | - flow.volcano.sh 21 | resources: 22 | - jobflows/status 23 | verbs: 24 | - get 25 | -------------------------------------------------------------------------------- /config/rbac/jobflow_viewer_role.yaml: -------------------------------------------------------------------------------- 1 | # permissions for end users to view jobflows. 2 | apiVersion: rbac.authorization.k8s.io/v1 3 | kind: ClusterRole 4 | metadata: 5 | name: jobflow-viewer-role 6 | rules: 7 | - apiGroups: 8 | - flow.volcano.sh 9 | resources: 10 | - jobflows 11 | verbs: 12 | - get 13 | - list 14 | - watch 15 | - apiGroups: 16 | - flow.volcano.sh 17 | resources: 18 | - jobflows/status 19 | verbs: 20 | - get 21 | -------------------------------------------------------------------------------- /config/rbac/jobtemplate_editor_role.yaml: -------------------------------------------------------------------------------- 1 | # permissions for end users to edit jobtemplates. 2 | apiVersion: rbac.authorization.k8s.io/v1 3 | kind: ClusterRole 4 | metadata: 5 | name: jobtemplate-editor-role 6 | rules: 7 | - apiGroups: 8 | - flow.volcano.sh 9 | resources: 10 | - jobtemplates 11 | verbs: 12 | - create 13 | - delete 14 | - get 15 | - list 16 | - patch 17 | - update 18 | - watch 19 | - apiGroups: 20 | - flow.volcano.sh 21 | resources: 22 | - jobtemplates/status 23 | verbs: 24 | - get 25 | -------------------------------------------------------------------------------- /config/rbac/jobtemplate_viewer_role.yaml: -------------------------------------------------------------------------------- 1 | # permissions for end users to view jobtemplates. 2 | apiVersion: rbac.authorization.k8s.io/v1 3 | kind: ClusterRole 4 | metadata: 5 | name: jobtemplate-viewer-role 6 | rules: 7 | - apiGroups: 8 | - flow.volcano.sh 9 | resources: 10 | - jobtemplates 11 | verbs: 12 | - get 13 | - list 14 | - watch 15 | - apiGroups: 16 | - flow.volcano.sh 17 | resources: 18 | - jobtemplates/status 19 | verbs: 20 | - get 21 | -------------------------------------------------------------------------------- /config/rbac/kustomization.yaml: -------------------------------------------------------------------------------- 1 | resources: 2 | # All RBAC will be applied under this service account in 3 | # the deployment namespace. You may comment out this resource 4 | # if your manager will use a service account that exists at 5 | # runtime. Be sure to update RoleBinding and ClusterRoleBinding 6 | # subjects if changing service account names. 7 | - service_account.yaml 8 | - role.yaml 9 | - role_binding.yaml 10 | - leader_election_role.yaml 11 | - leader_election_role_binding.yaml 12 | # Comment the following 4 lines if you want to disable 13 | # the auth proxy (https://github.com/brancz/kube-rbac-proxy) 14 | # which protects your /metrics endpoint. 15 | - auth_proxy_service.yaml 16 | - auth_proxy_role.yaml 17 | - auth_proxy_role_binding.yaml 18 | - auth_proxy_client_clusterrole.yaml 19 | -------------------------------------------------------------------------------- /config/rbac/leader_election_role.yaml: -------------------------------------------------------------------------------- 1 | # permissions to do leader election. 2 | apiVersion: rbac.authorization.k8s.io/v1 3 | kind: Role 4 | metadata: 5 | name: leader-election-role 6 | rules: 7 | - apiGroups: 8 | - "" 9 | resources: 10 | - configmaps 11 | verbs: 12 | - get 13 | - list 14 | - watch 15 | - create 16 | - update 17 | - patch 18 | - delete 19 | - apiGroups: 20 | - coordination.k8s.io 21 | resources: 22 | - leases 23 | verbs: 24 | - get 25 | - list 26 | - watch 27 | - create 28 | - update 29 | - patch 30 | - delete 31 | - apiGroups: 32 | - "" 33 | resources: 34 | - events 35 | verbs: 36 | - create 37 | - patch 38 | -------------------------------------------------------------------------------- /config/rbac/leader_election_role_binding.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: rbac.authorization.k8s.io/v1 2 | kind: RoleBinding 3 | metadata: 4 | name: leader-election-rolebinding 5 | roleRef: 6 | apiGroup: rbac.authorization.k8s.io 7 | kind: Role 8 | name: leader-election-role 9 | subjects: 10 | - kind: ServiceAccount 11 | name: controller-manager 12 | namespace: system 13 | -------------------------------------------------------------------------------- /config/rbac/role.yaml: -------------------------------------------------------------------------------- 1 | 2 | --- 3 | apiVersion: rbac.authorization.k8s.io/v1 4 | kind: ClusterRole 5 | metadata: 6 | creationTimestamp: null 7 | name: manager-role 8 | rules: 9 | - apiGroups: 10 | - "" 11 | resources: 12 | - events 13 | verbs: 14 | - create 15 | - patch 16 | - apiGroups: 17 | - admissionregistration.k8s.io 18 | resources: 19 | - mutatingwebhookconfigurations 20 | verbs: 21 | - create 22 | - delete 23 | - get 24 | - list 25 | - update 26 | - watch 27 | - apiGroups: 28 | - admissionregistration.k8s.io 29 | resources: 30 | - validatingwebhookconfigurations 31 | verbs: 32 | - create 33 | - delete 34 | - get 35 | - list 36 | - update 37 | - watch 38 | - apiGroups: 39 | - batch.volcano.sh 40 | resources: 41 | - jobs 42 | verbs: 43 | - create 44 | - delete 45 | - get 46 | - list 47 | - patch 48 | - update 49 | - watch 50 | - apiGroups: 51 | - batch.volcano.sh 52 | resources: 53 | - jobs/status 54 | verbs: 55 | - get 56 | - patch 57 | - update 58 | - apiGroups: 59 | - flow.volcano.sh 60 | resources: 61 | - jobflows 62 | verbs: 63 | - create 64 | - delete 65 | - get 66 | - list 67 | - patch 68 | - update 69 | - watch 70 | - apiGroups: 71 | - flow.volcano.sh 72 | resources: 73 | - jobflows/finalizers 74 | verbs: 75 | - update 76 | - apiGroups: 77 | - flow.volcano.sh 78 | resources: 79 | - jobflows/status 80 | verbs: 81 | - get 82 | - patch 83 | - update 84 | - apiGroups: 85 | - flow.volcano.sh 86 | resources: 87 | - jobtemplates 88 | verbs: 89 | - create 90 | - delete 91 | - get 92 | - list 93 | - patch 94 | - update 95 | - watch 96 | - apiGroups: 97 | - flow.volcano.sh 98 | resources: 99 | - jobtemplates/finalizers 100 | verbs: 101 | - update 102 | - apiGroups: 103 | - flow.volcano.sh 104 | resources: 105 | - jobtemplates/status 106 | verbs: 107 | - get 108 | - patch 109 | - update 110 | -------------------------------------------------------------------------------- /config/rbac/role_binding.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: rbac.authorization.k8s.io/v1 2 | kind: ClusterRoleBinding 3 | metadata: 4 | name: manager-rolebinding 5 | roleRef: 6 | apiGroup: rbac.authorization.k8s.io 7 | kind: ClusterRole 8 | name: manager-role 9 | subjects: 10 | - kind: ServiceAccount 11 | name: controller-manager 12 | namespace: system 13 | -------------------------------------------------------------------------------- /config/rbac/service_account.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: ServiceAccount 3 | metadata: 4 | name: controller-manager 5 | namespace: system 6 | -------------------------------------------------------------------------------- /config/samples/batch_v1alpha1_jobflow.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: flow.volcano.sh/v1alpha1 2 | kind: JobFlow 3 | metadata: 4 | name: jobflow-sample 5 | spec: 6 | # Add fields here 7 | foo: bar 8 | -------------------------------------------------------------------------------- /config/samples/batch_v1alpha1_jobtemplate.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: flow.volcano.sh/v1alpha1 2 | kind: JobTemplate 3 | metadata: 4 | name: jobtemplate-sample 5 | spec: 6 | # Add fields here 7 | foo: bar 8 | -------------------------------------------------------------------------------- /config/webhook/kustomization.yaml: -------------------------------------------------------------------------------- 1 | resources: 2 | #- manifests.yaml 3 | - service.yaml 4 | - secret.yaml 5 | 6 | #configurations: 7 | #- kustomizeconfig.yaml 8 | -------------------------------------------------------------------------------- /config/webhook/kustomizeconfig.yaml: -------------------------------------------------------------------------------- 1 | # the following config is for teaching kustomize where to look at when substituting vars. 2 | # It requires kustomize v2.1.0 or newer to work properly. 3 | nameReference: 4 | - kind: Service 5 | version: v1 6 | fieldSpecs: 7 | - kind: MutatingWebhookConfiguration 8 | group: admissionregistration.k8s.io 9 | path: webhooks/clientConfig/service/name 10 | - kind: ValidatingWebhookConfiguration 11 | group: admissionregistration.k8s.io 12 | path: webhooks/clientConfig/service/name 13 | 14 | namespace: 15 | - kind: MutatingWebhookConfiguration 16 | group: admissionregistration.k8s.io 17 | path: webhooks/clientConfig/service/namespace 18 | create: true 19 | - kind: ValidatingWebhookConfiguration 20 | group: admissionregistration.k8s.io 21 | path: webhooks/clientConfig/service/namespace 22 | create: true 23 | 24 | varReference: 25 | - path: metadata/annotations 26 | -------------------------------------------------------------------------------- /config/webhook/manifests.yaml: -------------------------------------------------------------------------------- 1 | 2 | --- 3 | apiVersion: admissionregistration.k8s.io/v1 4 | kind: MutatingWebhookConfiguration 5 | metadata: 6 | creationTimestamp: null 7 | name: mutating-webhook-configuration 8 | webhooks: 9 | - admissionReviewVersions: 10 | - v1 11 | - v1alpha1 12 | clientConfig: 13 | service: 14 | name: webhook-service 15 | namespace: system 16 | path: /mutate-batch-volcano-sh-v1alpha1-jobflow 17 | failurePolicy: Fail 18 | name: mjobflow.kb.io 19 | rules: 20 | - apiGroups: 21 | - flow.volcano.sh 22 | apiVersions: 23 | - v1alpha1 24 | operations: 25 | - CREATE 26 | - UPDATE 27 | resources: 28 | - jobflows 29 | sideEffects: None 30 | - admissionReviewVersions: 31 | - v1 32 | - v1alpha1 33 | clientConfig: 34 | service: 35 | name: webhook-service 36 | namespace: system 37 | path: /mutate-batch-volcano-sh-v1alpha1-jobtemplate 38 | failurePolicy: Fail 39 | name: mjobtemplate.kb.io 40 | rules: 41 | - apiGroups: 42 | - flow.volcano.sh 43 | apiVersions: 44 | - v1alpha1 45 | operations: 46 | - CREATE 47 | - UPDATE 48 | resources: 49 | - jobtemplates 50 | sideEffects: None 51 | 52 | --- 53 | apiVersion: admissionregistration.k8s.io/v1 54 | kind: ValidatingWebhookConfiguration 55 | metadata: 56 | creationTimestamp: null 57 | name: validating-webhook-configuration 58 | webhooks: 59 | - admissionReviewVersions: 60 | - v1 61 | - v1alpha1 62 | clientConfig: 63 | service: 64 | name: webhook-service 65 | namespace: system 66 | path: /validate-batch-volcano-sh-v1alpha1-jobflow 67 | failurePolicy: Fail 68 | name: vjobflow.kb.io 69 | rules: 70 | - apiGroups: 71 | - flow.volcano.sh 72 | apiVersions: 73 | - v1alpha1 74 | operations: 75 | - CREATE 76 | - UPDATE 77 | resources: 78 | - jobflows 79 | sideEffects: None 80 | - admissionReviewVersions: 81 | - v1 82 | - v1alpha1 83 | clientConfig: 84 | service: 85 | name: webhook-service 86 | namespace: system 87 | path: /validate-batch-volcano-sh-v1alpha1-jobtemplate 88 | failurePolicy: Fail 89 | name: vjobtemplate.kb.io 90 | rules: 91 | - apiGroups: 92 | - flow.volcano.sh 93 | apiVersions: 94 | - v1alpha1 95 | operations: 96 | - CREATE 97 | - UPDATE 98 | resources: 99 | - jobtemplates 100 | sideEffects: None 101 | -------------------------------------------------------------------------------- /config/webhook/secret.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | data: 3 | ca.crt: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSURSVENDQWkyZ0F3SUJBZ0lVU1JDRDhBSjVjSTdJUnVJRXFWV01mQ1JSMXQwd0RRWUpLb1pJaHZjTkFRRUwKQlFBd01qRXdNQzRHQTFVRUF3d25hbTlpWm14dmR5MTNaV0pvYjI5ckxYTmxjblpwWTJVdWEzVmlaUzF6ZVhOMApaVzB1YzNaak1CNFhEVEl5TURRd056QTNNemcwTTFvWERUTXlNRFF3TkRBM016ZzBNMW93TWpFd01DNEdBMVVFCkF3d25hbTlpWm14dmR5MTNaV0pvYjI5ckxYTmxjblpwWTJVdWEzVmlaUzF6ZVhOMFpXMHVjM1pqTUlJQklqQU4KQmdrcWhraUc5dzBCQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBeENSQkNKdW1STm1aZzBsUTloa1g1WWQ5dWRZVAp5WTdTU3BSYkUxc0NzM2xWRmJvNExlL2FUSFdUNStST211bzd6NG9qWUhqTVBKOXpHZGtyR2twc0RwM3RNWnRnCnF0Z013T2psbUFPL2VUay9IZHNFZEt1N3J1UnQwaHJOT2tpNTZ6Q3RBNndCWlRjZkdZQ3Y1ZHh2WUFPV01XbnAKMTlxV3V2NkxnaWF5aHlRMzVyZkp1Sk9mVzN6TUNaV1h6Y3V6K2lEc2luZnk3dWxMQk9jdWZtU0IxRGEyM3BIQgpZY3RhVW1tQ041eXRMU3RnSlNKWS9RN1lRNFNDRm9VU1ZDVTg5Y3NaOC9yYmI0TkFBdFVRNVJhT0hDOU1kS3VUCnlNNjNNdkJDREovZkNKQ1d3UmtiQnBMT0orcUNzdzFwZW1ZTTJBSHhWdDh6MmFraWxYQXJvUCs3eFFJREFRQUIKbzFNd1VUQWRCZ05WSFE0RUZnUVV1S2MzYWptQkdBc29wTkU1c3NTMytTd1o2bUV3SHdZRFZSMGpCQmd3Rm9BVQp1S2MzYWptQkdBc29wTkU1c3NTMytTd1o2bUV3RHdZRFZSMFRBUUgvQkFVd0F3RUIvekFOQmdrcWhraUc5dzBCCkFRc0ZBQU9DQVFFQXNDYzQwNm9pQlBERkV5YjRTbHdTamhQZUMyK09McjNkTTdBbmExSW10eFZSdWRvcnoyUC8KNkJvd0I1c2Z2clU5OTFQeVVaRG8yZEVUNWxvZ2d2R3lxWVNWOG9Va292ZExjd0hJcDVPOWx1SlRUM0xVcUM5RQpucjRsMEZ5NXJzSnJsNmpXcjEydzNHVE1IQTlNdlRadTBmSWhYRUdUWHUwVWpDL1ZBTlpMeUJYV2tjUnUwa3dJCkUvM3VtZkgxM1JNSnB4OU1xS2JUK1RzOHo0YW5BOTNvVkNiQ1k2R2F3Z0FybmdUOWhpazRKQ0lycTVWY1BaV3MKM0lORy9SRVRQVW5QM3UzNFRHMHF0S2ViUmQwTlBQdmlBOWVVQjFFc0dmNHpIMEE1Z2NIMXNYeHIwM25DOUErNwo2akFvUEpucWVQTTR3alBTL0F0TmpZZlJBZWNjK2dnajlnPT0KLS0tLS1FTkQgQ0VSVElGSUNBVEUtLS0tLQo= 4 | tls.crt: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSURsVENDQW4yZ0F3SUJBZ0lVVHRYVHFKVS9yeHVqVHJDVWk1bFY2TEttczFrd0RRWUpLb1pJaHZjTkFRRUwKQlFBd01qRXdNQzRHQTFVRUF3d25hbTlpWm14dmR5MTNaV0pvYjI5ckxYTmxjblpwWTJVdWEzVmlaUzF6ZVhOMApaVzB1YzNaak1CNFhEVEl5TURRd056QTNNemcwTTFvWERUSXlNRFV3TnpBM016ZzBNMW93TWpFd01DNEdBMVVFCkF3d25hbTlpWm14dmR5MTNaV0pvYjI5ckxYTmxjblpwWTJVdWEzVmlaUzF6ZVhOMFpXMHVjM1pqTUlJQklqQU4KQmdrcWhraUc5dzBCQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBMVdvbXJ3THJVcTlHcDdBV0d6ZjZ4a0UwaDZKYgpZVVpGd2tBSWltNW5vWlZHNStXS1ptTlUveXJQRm1CYWJHVW1ZQWJ0TWM4Um1iVzM4OVp2N0IwOGNIUk93K3JVClp6T0I4d3I2akE1bVh5M0VFOUlKYmtPYnZvS3AyMm54WGJqTHhJSzRBOE5VWHJUY1ByWTNjeEtDbHlKNEx4ZEIKdjF4K3FhNUlJU1AzMWV4QkZBNzNnMkQwbis1Z0pJZENiREtZUVVsQWgvdjhpMVZ3RUV5V1o4b2tUVVRTa25IUwp2OVl1b2VjYUJvcWN1cW5pZXcwaTYrcVlkdlhkUVFQYmR2M3BEWWkrUHVFeVBDbXZuYVUra09BWjd0WmZpc3RmCjVKbEJPQmlORXIzcFgyU0hwRmpnSjFLRmZpZllVMW05RG5rdVdRbkttTHZ6YXpEeHd1T1JNUmsxZndJREFRQUIKbzRHaU1JR2ZNQWtHQTFVZEV3UUNNQUF3Q3dZRFZSMFBCQVFEQWdYZ01CTUdBMVVkSlFRTU1Bb0dDQ3NHQVFVRgpCd01CTUhBR0ExVWRFUVJwTUdlQ0YycHZZbVpzYjNjdGQyVmlhRzl2YXkxelpYSjJhV05sZ2lOcWIySm1iRzkzCkxYZGxZbWh2YjJzdGMyVnlkbWxqWlM1cmRXSmxMWE41YzNSbGJZSW5hbTlpWm14dmR5MTNaV0pvYjI5ckxYTmwKY25acFkyVXVhM1ZpWlMxemVYTjBaVzB1YzNaak1BMEdDU3FHU0liM0RRRUJDd1VBQTRJQkFRQzZqRTJkUmFTVQp1SFBYUmIyV1huMS9iSmZ5UDJOUUIyUlZmSzd2QXRWMHRnVFd2cWtzN3c0WWNWTCs1VXpZVThUVytxdnJTWGJmCi80WWtUOTZxckVwTThMTC9TRE1VbCsyYUpoZmJXMUNDelVzZTA3bFNsRU80ZDQwMmVKVE9qNkRkTzFUMGE1YkwKQzdUSnN4d0dEaFFyNGw1OEhPZFZDZk94UUpidzloUnZENHFtd24rY3BETmFTTEhEcmpYREtMN2s0SjdtMzRiZQpuZUgrNGtSTTd2d2NuemNwclZUZW9nOEZSeDR1d3U2WTBZRGRuM0ZDQTg2YzVvaERFcWtBOWhmKzhid2xDV3RwClEyVXBOWHdjOS82NE9sWURvUWpVanpETldUbzJCR1QydWM5OTUweFhta1gzd0twbGZJTjhHaE4wbG1oelZEbUsKSDRIOFo3ZUFBZm9ECi0tLS0tRU5EIENFUlRJRklDQVRFLS0tLS0K 5 | tls.key: LS0tLS1CRUdJTiBSU0EgUFJJVkFURSBLRVktLS0tLQpNSUlFcEFJQkFBS0NBUUVBMVdvbXJ3THJVcTlHcDdBV0d6ZjZ4a0UwaDZKYllVWkZ3a0FJaW01bm9aVkc1K1dLClptTlUveXJQRm1CYWJHVW1ZQWJ0TWM4Um1iVzM4OVp2N0IwOGNIUk93K3JVWnpPQjh3cjZqQTVtWHkzRUU5SUoKYmtPYnZvS3AyMm54WGJqTHhJSzRBOE5VWHJUY1ByWTNjeEtDbHlKNEx4ZEJ2MXgrcWE1SUlTUDMxZXhCRkE3MwpnMkQwbis1Z0pJZENiREtZUVVsQWgvdjhpMVZ3RUV5V1o4b2tUVVRTa25IU3Y5WXVvZWNhQm9xY3VxbmlldzBpCjYrcVlkdlhkUVFQYmR2M3BEWWkrUHVFeVBDbXZuYVUra09BWjd0WmZpc3RmNUpsQk9CaU5FcjNwWDJTSHBGamcKSjFLRmZpZllVMW05RG5rdVdRbkttTHZ6YXpEeHd1T1JNUmsxZndJREFRQUJBb0lCQUZ1K0YwVVMzekNiVXFkUgpoaDlMVUUyYnlFWXFZRmdQeW5zMzViUUV4QjN0T3g4RFNSeTh4TlF3aGZlYmtpT1kxQWRoMHVPZm5YUlRidkd1CmxFeVBRT1VpeXAxa1BNemtrbzkxZjhGbmkxQ05Md1pLTzIyZ09McU1GeExRbDBidjR3RG1KTTdSVUZxcUt5ZjIKVXgzdnJoM2tYbi8zNG5hdC8zRllEMy9YbzNuZEhhY1BnQm1NTlhyN0IxbmRmVXJsMDcwN1BCMG1HbVh0TEE3awp2ODJUWjdLbHdnUTc5OWhkaEEwbUMxckphaXB1VW4zYnRpTzl2K0ZFdklhdElpOE80K1duS3ZOM25vYytFTGVvCnBNMGZLSEZqUUlmRHo3ek8rRlloNDhwRTcrekZvN3pkL2QyZEVQYXZmdERhazlhbEE3a3RVbG9pNmpIbTNkM0YKQzJLZUtla0NnWUVBK241U2lPYzVub3N6VXErSEVzcndpYXZXdU53R1laTFF3WkZtK3N6eWVaMmY5TG11UWlOMApvK09BZnNsYUZYRzdtdjZlRDlhdGNBdm1Ec2N4NlgycW0yWE55SkRMY0Y5dnFYdTR4elpQbThlQVNsaFhmUks2Ci9HcUpTUFlsdmMzNXQyMnZNSEx0cktjM2lPOVQyaU90Z3ZhQ2laQ0lvMTZZbUxlSm9SSW53U1VDZ1lFQTJoc3AKK3VQbVNNSVZGdjF1RzBwVENNU0xSMjhxMUpJNVZiRE5palc5NVBNYjFUazRNNE95dndONkpiamFUaGd4dlYzSwpxTmlZWFZ0MFNnOTlqdVFCM2wyQ2FxcHErUGNlUURMNTVydFVSM0paa0ZKb1ZRMWw0S2hJckEyYzV4RGhhdWF4CmNKTGgrK2hSUXNRZTJiT0RhdXdITUs1Tm0weEdOTDhJWnNjN3ROTUNnWUVBenZ0Y3BhVXlsay85ZTZCd28xV3YKaG9MSWJYM1pnL3kxcEl6S0pBai9md0NCTU0zUk1QTnRLUk1PbFRVNXk2aHIxYm40ejZ1YktvK2FiTEdxQzM1OApYK1d5TWIxN2JRSmZHUk9UYm9EeExRNmZjazhuRThGTFl0R0JXUm1UdkErYi9UYVQ0UnZHU3JqdGlhZ0FpS3FjCmNDL1RVMnByalZyWUNyRDE5M015Q0VVQ2dZQmI0Q3Q3ODNxelZZWjZ5OEVSSCtzQWU0TE1VYWp5S0xLY1JVRWcKSW1sZXc0WUsrUEtTeUx5SU9GZkJBakI3eXpkUXRPekUyWkM5YXVQK3VxM0Nmb3ZHOXc4VURidklLcGtFcERTZgpISFJ4TUZ0SUwxNmh6V1lJRC91azlvc016eENWN3AzNmRQVmJIMDd6MkJmQ3p4cmg5SkZHMFhZQm9Fekd0VjQ5CnBWbWlYd0tCZ1FDNzNVS0IvSktGaExia3ZrZk9Qdld2STBJZnVodm4wNmdDc01FQ2VMUjMvNXVuRDhWY0dBNmoKNW9SN1p0NW5xQnZqTytJT3RMSnpncG4zeXVpU2QrdGZaVXpaTDZPYzNqVkU0c0tpejlXSUk5WG9WT2Z3ZWx6bQprZkhuVG5MWlJuNXk0Z3prVEdyd1lxdmRGRGJyZ0poVWo3YlRWU0syYkZwc2JKMlVFYi9rTVE9PQotLS0tLUVORCBSU0EgUFJJVkFURSBLRVktLS0tLQo= 6 | kind: Secret 7 | metadata: 8 | name: webhook-server-cert 9 | namespace: kube-system 10 | type: Opaque -------------------------------------------------------------------------------- /config/webhook/service.yaml: -------------------------------------------------------------------------------- 1 | 2 | apiVersion: v1 3 | kind: Service 4 | metadata: 5 | name: webhook-service 6 | namespace: system 7 | spec: 8 | ports: 9 | - port: 443 10 | targetPort: 8725 11 | selector: 12 | control-plane: controller-manager 13 | -------------------------------------------------------------------------------- /controllers/jobflow.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2021. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package controllers 18 | 19 | import ( 20 | "context" 21 | "fmt" 22 | "strings" 23 | "time" 24 | 25 | corev1 "k8s.io/api/core/v1" 26 | "k8s.io/apimachinery/pkg/api/errors" 27 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 28 | "k8s.io/apimachinery/pkg/runtime" 29 | "k8s.io/apimachinery/pkg/types" 30 | "k8s.io/client-go/tools/record" 31 | "k8s.io/client-go/util/workqueue" 32 | "k8s.io/klog" 33 | ctrl "sigs.k8s.io/controller-runtime" 34 | "sigs.k8s.io/controller-runtime/pkg/client" 35 | "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" 36 | "sigs.k8s.io/controller-runtime/pkg/event" 37 | "sigs.k8s.io/controller-runtime/pkg/handler" 38 | "sigs.k8s.io/controller-runtime/pkg/reconcile" 39 | "sigs.k8s.io/controller-runtime/pkg/source" 40 | "volcano.sh/apis/pkg/apis/batch/v1alpha1" 41 | 42 | jobflowv1alpha1 "jobflow/api/v1alpha1" 43 | "jobflow/utils" 44 | ) 45 | 46 | // JobFlowReconciler reconciles a JobFlow object 47 | type JobFlowReconciler struct { 48 | client.Client 49 | Scheme *runtime.Scheme 50 | Recorder record.EventRecorder 51 | } 52 | 53 | // +kubebuilder:rbac:groups=flow.volcano.sh,resources=jobflows,verbs=get;list;watch;create;update;patch;delete 54 | // +kubebuilder:rbac:groups=flow.volcano.sh,resources=jobflows/status,verbs=get;update;patch 55 | // +kubebuilder:rbac:groups=flow.volcano.sh,resources=jobflows/finalizers,verbs=update 56 | // +kubebuilder:rbac:groups=batch.volcano.sh,resources=jobs,verbs=get;list;watch;create;update;patch;delete 57 | // +kubebuilder:rbac:groups=batch.volcano.sh,resources=jobs/status,verbs=get;update;patch 58 | // +kubebuilder:rbac:groups="",resources=events,verbs=create;patch 59 | // +kubebuilder:rbac:groups=admissionregistration.k8s.io,resources=mutatingwebhookconfigurations,verbs=get;list;watch;create;update;delete 60 | // +kubebuilder:rbac:groups=admissionregistration.k8s.io,resources=validatingwebhookconfigurations,verbs=get;list;watch;create;update;delete 61 | 62 | // Reconcile is part of the main kubernetes reconciliation loop which aims to 63 | // move the current state of the cluster closer to the desired state. 64 | // TODO(user): Modify the Reconcile function to compare the state specified by 65 | // the JobFlow object against the actual cluster state, and then 66 | // perform operations to make the cluster state reflect the state specified by 67 | // the user. 68 | // 69 | // For more details, check Reconcile and its Result here: 70 | // - https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.8.3/pkg/reconcile 71 | func (r *JobFlowReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { 72 | klog.Info("start jobFlow Reconcile..........") 73 | klog.Info(fmt.Sprintf("req.%v", req)) 74 | 75 | scheduledResult := ctrl.Result{} 76 | // load JobFlow by namespace 77 | jobFlow := &jobflowv1alpha1.JobFlow{} 78 | time.Sleep(time.Second) 79 | err := r.Get(ctx, req.NamespacedName, jobFlow) 80 | if err != nil { 81 | // If no instance is found, it will be returned directly 82 | if errors.IsNotFound(err) { 83 | klog.Info(fmt.Sprintf("not found jobFlow : %v", req.Name)) 84 | return scheduledResult, nil 85 | } 86 | klog.Error(err, err.Error()) 87 | r.Recorder.Eventf(jobFlow, corev1.EventTypeWarning, "Created", err.Error()) 88 | return scheduledResult, err 89 | } 90 | // JobRetainPolicy Judging whether jobs are necessary to delete 91 | if jobFlow.Spec.JobRetainPolicy == jobflowv1alpha1.Delete && jobFlow.Status.State.Phase == jobflowv1alpha1.Succeed { 92 | if err := r.deleteAllJobsCreateByJobFlow(ctx, jobFlow); err != nil { 93 | klog.Error(err, "delete jobs create by JobFlow error！") 94 | return scheduledResult, err 95 | } 96 | return scheduledResult, err 97 | } 98 | 99 | // deploy job by dependence order. 100 | if err = r.deployJob(ctx, *jobFlow); err != nil { 101 | klog.Error(err, "") 102 | return scheduledResult, err 103 | } 104 | 105 | // update status 106 | if err = r.updateStatus(ctx, jobFlow); err != nil { 107 | klog.Error(err, "update jobFlow status error!") 108 | return scheduledResult, err 109 | } 110 | klog.Info("end jobFlow Reconcile........") 111 | return scheduledResult, nil 112 | } 113 | 114 | func getJobName(jobFlowName string, jobTemplateName string) string { 115 | return jobFlowName + "-" + jobTemplateName 116 | } 117 | 118 | const ( 119 | JobFlow = "JobFlow" 120 | ) 121 | 122 | //deploy job by dependence order. 123 | func (r *JobFlowReconciler) deployJob(ctx context.Context, jobFlow jobflowv1alpha1.JobFlow) error { 124 | // load jobTemplate by flow and deploy it 125 | for _, flow := range jobFlow.Spec.Flows { 126 | job := &v1alpha1.Job{} 127 | jobName := getJobName(jobFlow.Name, flow.Name) 128 | namespacedNameJob := types.NamespacedName{ 129 | Namespace: jobFlow.Namespace, 130 | Name: jobName, 131 | } 132 | if err := r.Get(ctx, namespacedNameJob, job); err != nil { 133 | if errors.IsNotFound(err) { 134 | // If it is not distributed, judge whether the dependency of the VcJob meets the requirements 135 | if flow.DependsOn == nil || flow.DependsOn.Targets == nil { 136 | if err := r.loadJobTemplateAndSetJob(jobFlow, flow, jobName, job); err != nil { 137 | return err 138 | } 139 | if err = r.Create(ctx, job); err != nil { 140 | if errors.IsAlreadyExists(err) { 141 | continue 142 | } 143 | return err 144 | } 145 | r.Recorder.Eventf(&jobFlow, corev1.EventTypeNormal, "Created", fmt.Sprintf("create a job named %v!", job.Name)) 146 | } else { 147 | // query dependency meets the requirements 148 | flag := true 149 | for _, targetName := range flow.DependsOn.Targets { 150 | job = &v1alpha1.Job{} 151 | targetJobName := getJobName(jobFlow.Name, targetName) 152 | namespacedName := types.NamespacedName{ 153 | Namespace: jobFlow.Namespace, 154 | Name: targetJobName, 155 | } 156 | if err = r.Get(ctx, namespacedName, job); err != nil { 157 | if err != nil { 158 | if errors.IsNotFound(err) { 159 | klog.Info(fmt.Sprintf("No %v Job found！", namespacedName.Name)) 160 | flag = false 161 | break 162 | } 163 | return err 164 | } 165 | } 166 | if job.Status.State.Phase != v1alpha1.Completed { 167 | flag = false 168 | } 169 | } 170 | if flag { 171 | if err := r.loadJobTemplateAndSetJob(jobFlow, flow, jobName, job); err != nil { 172 | return err 173 | } 174 | if err = r.Create(ctx, job); err != nil { 175 | if errors.IsAlreadyExists(err) { 176 | break 177 | } 178 | return err 179 | } 180 | r.Recorder.Eventf(&jobFlow, corev1.EventTypeNormal, "Created", fmt.Sprintf("create a job named %v!", job.Name)) 181 | } 182 | } 183 | continue 184 | } 185 | return err 186 | } 187 | } 188 | return nil 189 | } 190 | 191 | func (r *JobFlowReconciler) loadJobTemplateAndSetJob(jobFlow jobflowv1alpha1.JobFlow, flow jobflowv1alpha1.Flow, jobName string, job *v1alpha1.Job) error { 192 | // load jobTemplate 193 | jobTemplate := &jobflowv1alpha1.JobTemplate{} 194 | namespacedNameTemplate := types.NamespacedName{ 195 | Namespace: jobFlow.Namespace, 196 | Name: flow.Name, 197 | } 198 | if err := r.Get(context.TODO(), namespacedNameTemplate, jobTemplate); err != nil { 199 | klog.Error(err, "not found the jobTemplate！") 200 | return err 201 | } 202 | *job = v1alpha1.Job{ 203 | ObjectMeta: metav1.ObjectMeta{ 204 | Name: jobName, 205 | Namespace: jobFlow.Namespace, 206 | Annotations: map[string]string{utils.CreateByJobTemplate: utils.GetConnectionOfJobAndJobTemplate(jobFlow.Namespace, flow.Name)}, 207 | }, 208 | Spec: jobTemplate.Spec, 209 | Status: v1alpha1.JobStatus{}, 210 | } 211 | if err := controllerutil.SetControllerReference(&jobFlow, job, r.Scheme); err != nil { 212 | return err 213 | } 214 | return nil 215 | } 216 | 217 | // update status 218 | func (r *JobFlowReconciler) updateStatus(ctx context.Context, jobFlow *jobflowv1alpha1.JobFlow) error { 219 | klog.Info(fmt.Sprintf("start to update jobFlow status! jobFlowName: %v, jobFlowNamespace: %v ", jobFlow.Name, jobFlow.Namespace)) 220 | allJobList := new(v1alpha1.JobList) 221 | err := r.List(ctx, allJobList) 222 | if err != nil { 223 | klog.Error(err, "") 224 | return err 225 | } 226 | jobFlowStatus, err := getAllJobStatus(jobFlow, allJobList) 227 | if err != nil { 228 | return err 229 | } 230 | jobFlow.Status = *jobFlowStatus 231 | jobFlow.CreationTimestamp = metav1.Time{} 232 | jobFlow.UID = "" 233 | if err = r.Status().Update(ctx, jobFlow); err != nil { 234 | if errors.IsNotFound(err) { 235 | return nil 236 | } 237 | return err 238 | } 239 | return nil 240 | } 241 | 242 | // getAllJobStatus Get the information of all created jobs 243 | func getAllJobStatus(jobFlow *jobflowv1alpha1.JobFlow, allJobList *v1alpha1.JobList) (*jobflowv1alpha1.JobFlowStatus, error) { 244 | jobListRes := make([]v1alpha1.Job, 0) 245 | for _, job := range allJobList.Items { 246 | for _, reference := range job.OwnerReferences { 247 | if reference.Kind == JobFlow && strings.Contains(reference.APIVersion, "volcano") && reference.Name == jobFlow.Name { 248 | jobListRes = append(jobListRes, job) 249 | } 250 | } 251 | } 252 | conditions := make(map[string]jobflowv1alpha1.Condition) 253 | pendingJobs := make([]string, 0) 254 | runningJobs := make([]string, 0) 255 | FailedJobs := make([]string, 0) 256 | CompletedJobs := make([]string, 0) 257 | TerminatedJobs := make([]string, 0) 258 | UnKnowJobs := make([]string, 0) 259 | jobList := make([]string, 0) 260 | 261 | state := new(jobflowv1alpha1.State) 262 | for _, flow := range jobFlow.Spec.Flows { 263 | jobList = append(jobList, getJobName(jobFlow.Name, flow.Name)) 264 | } 265 | statusListJobMap := map[v1alpha1.JobPhase]*[]string{ 266 | v1alpha1.Pending: &pendingJobs, 267 | v1alpha1.Running: &runningJobs, 268 | v1alpha1.Completing: &CompletedJobs, 269 | v1alpha1.Completed: &CompletedJobs, 270 | v1alpha1.Terminating: &TerminatedJobs, 271 | v1alpha1.Terminated: &TerminatedJobs, 272 | v1alpha1.Failed: &FailedJobs, 273 | } 274 | for _, job := range jobListRes { 275 | if jobListRes, ok := statusListJobMap[job.Status.State.Phase]; ok { 276 | *jobListRes = append(*jobListRes, job.Name) 277 | } else { 278 | UnKnowJobs = append(UnKnowJobs, job.Name) 279 | } 280 | conditions[job.Name] = jobflowv1alpha1.Condition{ 281 | Phase: job.Status.State.Phase, 282 | CreateTimestamp: job.CreationTimestamp, 283 | RunningDuration: job.Status.RunningDuration, 284 | TaskStatusCount: job.Status.TaskStatusCount, 285 | } 286 | 287 | } 288 | jobStatusList := make([]jobflowv1alpha1.JobStatus, 0) 289 | if jobFlow.Status.JobStatusList != nil { 290 | jobStatusList = jobFlow.Status.JobStatusList 291 | } 292 | for _, job := range jobListRes { 293 | runningHistories := getRunningHistories(jobStatusList, job) 294 | endTimeStamp := metav1.Time{} 295 | if job.Status.RunningDuration != nil { 296 | endTimeStamp = job.CreationTimestamp 297 | endTimeStamp = metav1.Time{Time: endTimeStamp.Add(job.Status.RunningDuration.Duration)} 298 | } 299 | jobStatus := jobflowv1alpha1.JobStatus{ 300 | Name: job.Name, 301 | State: job.Status.State.Phase, 302 | StartTimestamp: job.CreationTimestamp, 303 | EndTimestamp: endTimeStamp, 304 | RestartCount: job.Status.RetryCount, 305 | RunningHistories: runningHistories, 306 | } 307 | jobFlag := true 308 | for i := range jobStatusList { 309 | if jobStatusList[i].Name == jobStatus.Name { 310 | jobFlag = false 311 | jobStatusList[i] = jobStatus 312 | } 313 | } 314 | if jobFlag { 315 | jobStatusList = append(jobStatusList, jobStatus) 316 | } 317 | } 318 | if jobFlow.DeletionTimestamp != nil { 319 | state.Phase = jobflowv1alpha1.Terminating 320 | } else { 321 | if len(jobList) != len(CompletedJobs) { 322 | if len(FailedJobs) > 0 { 323 | state.Phase = jobflowv1alpha1.Failed 324 | } else if len(runningJobs) > 0 || len(CompletedJobs) > 0 { 325 | state.Phase = jobflowv1alpha1.Running 326 | } else { 327 | state.Phase = jobflowv1alpha1.Pending 328 | } 329 | } else { 330 | state.Phase = jobflowv1alpha1.Succeed 331 | } 332 | } 333 | 334 | jobFlowStatus := jobflowv1alpha1.JobFlowStatus{ 335 | PendingJobs: pendingJobs, 336 | RunningJobs: runningJobs, 337 | FailedJobs: FailedJobs, 338 | CompletedJobs: CompletedJobs, 339 | TerminatedJobs: TerminatedJobs, 340 | UnKnowJobs: UnKnowJobs, 341 | JobStatusList: jobStatusList, 342 | Conditions: conditions, 343 | State: *state, 344 | } 345 | return &jobFlowStatus, nil 346 | } 347 | 348 | func getRunningHistories(jobStatusList []jobflowv1alpha1.JobStatus, job v1alpha1.Job) []jobflowv1alpha1.JobRunningHistory { 349 | klog.Infof("start insert %+v RunningHistories", job.Name) 350 | runningHistories := make([]jobflowv1alpha1.JobRunningHistory, 0) 351 | flag := true 352 | for _, jobStatusGet := range jobStatusList { 353 | if jobStatusGet.Name == job.Name { 354 | if jobStatusGet.RunningHistories != nil { 355 | flag = false 356 | runningHistories = jobStatusGet.RunningHistories 357 | // State change 358 | if runningHistories[len(runningHistories)-1].State != job.Status.State.Phase { 359 | runningHistories[len(runningHistories)-1].EndTimestamp = metav1.Time{ 360 | Time: time.Now(), 361 | } 362 | runningHistories = append(runningHistories, jobflowv1alpha1.JobRunningHistory{ 363 | StartTimestamp: metav1.Time{Time: time.Now()}, 364 | EndTimestamp: metav1.Time{}, 365 | State: job.Status.State.Phase, 366 | }) 367 | } 368 | } 369 | } 370 | } 371 | if flag && job.Status.State.Phase != "" { 372 | runningHistories = append(runningHistories, jobflowv1alpha1.JobRunningHistory{ 373 | StartTimestamp: metav1.Time{ 374 | Time: time.Now(), 375 | }, 376 | EndTimestamp: metav1.Time{}, 377 | State: job.Status.State.Phase, 378 | }) 379 | } 380 | return runningHistories 381 | } 382 | 383 | func (r *JobFlowReconciler) deleteAllJobsCreateByJobFlow(ctx context.Context, jobFlow *jobflowv1alpha1.JobFlow) error { 384 | jobList := new(v1alpha1.JobList) 385 | if err := r.List(ctx, jobList, client.InNamespace(jobFlow.Namespace)); err != nil { 386 | return err 387 | } 388 | for _, item := range jobList.Items { 389 | if len(item.OwnerReferences) > 0 { 390 | for _, reference := range item.OwnerReferences { 391 | if reference.Kind == jobFlow.Kind && reference.Name == jobFlow.Name { 392 | if err := r.Delete(ctx, &item); err != nil { 393 | return err 394 | } 395 | } 396 | } 397 | } 398 | } 399 | return nil 400 | } 401 | 402 | // SetupWithManager sets up the controller with the Manager. 403 | func (r *JobFlowReconciler) SetupWithManager(mgr ctrl.Manager) error { 404 | return ctrl.NewControllerManagedBy(mgr). 405 | For(&jobflowv1alpha1.JobFlow{}). 406 | Watches(&source.Kind{Type: &v1alpha1.Job{}}, handler.Funcs{UpdateFunc: r.jobUpdateHandler}). 407 | Complete(r) 408 | } 409 | 410 | func (r *JobFlowReconciler) jobUpdateHandler(e event.UpdateEvent, q workqueue.RateLimitingInterface) { 411 | references := e.ObjectOld.GetOwnerReferences() 412 | for _, owner := range references { 413 | if owner.Kind == "JobFlow" && strings.Contains(owner.APIVersion, "volcano") { 414 | klog.Info(fmt.Sprintf("Listen to the update event of the job！jobName: %v, jobFlowName: %v, nameSpace: %v", e.ObjectOld.GetName(), owner.Name, e.ObjectOld.GetNamespace())) 415 | q.AddRateLimited(reconcile.Request{ 416 | NamespacedName: types.NamespacedName{Name: owner.Name, Namespace: e.ObjectOld.GetNamespace()}, 417 | }) 418 | } 419 | } 420 | } 421 | -------------------------------------------------------------------------------- /controllers/jobflow_test.go: -------------------------------------------------------------------------------- 1 | package controllers 2 | 3 | import ( 4 | "reflect" 5 | "testing" 6 | "time" 7 | 8 | jobflowv1alpha1 "jobflow/api/v1alpha1" 9 | "k8s.io/apimachinery/pkg/apis/meta/v1" 10 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 11 | "volcano.sh/apis/pkg/apis/batch/v1alpha1" 12 | ) 13 | 14 | func TestGetJobNameFunc(t *testing.T) { 15 | type args struct { 16 | jobFlowName string 17 | jobTemplateName string 18 | } 19 | tests := []struct { 20 | name string 21 | args args 22 | want string 23 | }{ 24 | { 25 | name: "GetJobName success case", 26 | args: args{ 27 | jobFlowName: "jobFlowA", 28 | jobTemplateName: "jobTemplateA", 29 | }, 30 | want: "jobFlowA-jobTemplateA", 31 | }, 32 | } 33 | for _, tt := range tests { 34 | t.Run(tt.name, func(t *testing.T) { 35 | if got := getJobName(tt.args.jobFlowName, tt.args.jobTemplateName); got != tt.want { 36 | t.Errorf("getJobName() = %v, want %v", got, tt.want) 37 | } 38 | }) 39 | } 40 | } 41 | 42 | func TestGetRunningHistoriesFunc(t *testing.T) { 43 | type args struct { 44 | jobStatusList []jobflowv1alpha1.JobStatus 45 | job v1alpha1.Job 46 | } 47 | startTime := time.Now() 48 | endTime := startTime.Add(1 * time.Second) 49 | tests := []struct { 50 | name string 51 | args args 52 | want []jobflowv1alpha1.JobRunningHistory 53 | }{ 54 | { 55 | name: "GetRunningHistories success case", 56 | args: args{ 57 | jobStatusList: []jobflowv1alpha1.JobStatus{ 58 | { 59 | Name: "vcJobA", 60 | State: v1alpha1.Completed, 61 | StartTimestamp: v1.Time{Time: startTime}, 62 | EndTimestamp: v1.Time{Time: endTime}, 63 | RestartCount: 0, 64 | RunningHistories: []jobflowv1alpha1.JobRunningHistory{ 65 | { 66 | StartTimestamp: v1.Time{Time: startTime}, 67 | EndTimestamp: v1.Time{Time: endTime}, 68 | State: v1alpha1.Completed, 69 | }, 70 | }, 71 | }, 72 | }, 73 | job: v1alpha1.Job{ 74 | TypeMeta: v1.TypeMeta{}, 75 | ObjectMeta: v1.ObjectMeta{Name: "vcJobA"}, 76 | Spec: v1alpha1.JobSpec{}, 77 | Status: v1alpha1.JobStatus{ 78 | State: v1alpha1.JobState{ 79 | Phase: v1alpha1.Completed, 80 | Reason: "", 81 | Message: "", 82 | LastTransitionTime: v1.Time{}, 83 | }, 84 | }, 85 | }, 86 | }, 87 | want: []jobflowv1alpha1.JobRunningHistory{ 88 | { 89 | StartTimestamp: v1.Time{Time: startTime}, 90 | EndTimestamp: v1.Time{Time: endTime}, 91 | State: v1alpha1.Completed, 92 | }, 93 | }, 94 | }, 95 | } 96 | for _, tt := range tests { 97 | t.Run(tt.name, func(t *testing.T) { 98 | if got := getRunningHistories(tt.args.jobStatusList, tt.args.job); !reflect.DeepEqual(got, tt.want) { 99 | t.Errorf("getRunningHistories() = %v, want %v", got, tt.want) 100 | } 101 | }) 102 | } 103 | } 104 | 105 | func TestGetAllJobStatusFunc(t *testing.T) { 106 | type args struct { 107 | jobFlow *jobflowv1alpha1.JobFlow 108 | allJobList *v1alpha1.JobList 109 | } 110 | createJobATime := time.Now() 111 | jobFlowName := "jobFlowA" 112 | createJobBTime := createJobATime.Add(time.Second) 113 | tests := []struct { 114 | name string 115 | args args 116 | want *jobflowv1alpha1.JobFlowStatus 117 | wantErr bool 118 | }{ 119 | { 120 | name: "GetAllJobStatus success case", 121 | args: args{ 122 | jobFlow: &jobflowv1alpha1.JobFlow{ 123 | TypeMeta: v1.TypeMeta{}, 124 | ObjectMeta: v1.ObjectMeta{ 125 | Name: jobFlowName, 126 | }, 127 | Spec: jobflowv1alpha1.JobFlowSpec{ 128 | Flows: []jobflowv1alpha1.Flow{ 129 | { 130 | Name: "A", 131 | DependsOn: nil, 132 | }, 133 | { 134 | Name: "B", 135 | DependsOn: &jobflowv1alpha1.DependsOn{ 136 | Targets: []string{"A"}, 137 | }, 138 | }, 139 | }, 140 | JobRetainPolicy: "", 141 | }, 142 | Status: jobflowv1alpha1.JobFlowStatus{}, 143 | }, 144 | allJobList: &v1alpha1.JobList{ 145 | Items: []v1alpha1.Job{ 146 | { 147 | TypeMeta: v1.TypeMeta{}, 148 | ObjectMeta: v1.ObjectMeta{ 149 | Name: "jobFlowA-A", 150 | CreationTimestamp: v1.Time{Time: createJobATime}, 151 | OwnerReferences: []v1.OwnerReference{{ 152 | APIVersion: "volcano", 153 | Kind: JobFlow, 154 | Name: jobFlowName, 155 | }}, 156 | }, 157 | Spec: v1alpha1.JobSpec{}, 158 | Status: v1alpha1.JobStatus{ 159 | State: v1alpha1.JobState{Phase: v1alpha1.Completed}, 160 | RetryCount: 1, 161 | RunningDuration: &metav1.Duration{Duration: time.Second}, 162 | }, 163 | }, 164 | { 165 | TypeMeta: v1.TypeMeta{}, 166 | ObjectMeta: v1.ObjectMeta{ 167 | Name: "jobFlowA-B", 168 | CreationTimestamp: v1.Time{Time: createJobBTime}, 169 | OwnerReferences: []v1.OwnerReference{{ 170 | APIVersion: "volcano", 171 | Kind: JobFlow, 172 | Name: jobFlowName, 173 | }}, 174 | }, 175 | Spec: v1alpha1.JobSpec{}, 176 | Status: v1alpha1.JobStatus{ 177 | State: v1alpha1.JobState{Phase: v1alpha1.Running}, 178 | }, 179 | }, 180 | }, 181 | }, 182 | }, 183 | want: &jobflowv1alpha1.JobFlowStatus{ 184 | PendingJobs: []string{}, 185 | RunningJobs: []string{"jobFlowA-B"}, 186 | FailedJobs: []string{}, 187 | CompletedJobs: []string{"jobFlowA-A"}, 188 | TerminatedJobs: []string{}, 189 | UnKnowJobs: []string{}, 190 | JobStatusList: []jobflowv1alpha1.JobStatus{ 191 | { 192 | Name: "jobFlowA-A", 193 | State: v1alpha1.Completed, 194 | StartTimestamp: metav1.Time{Time: createJobATime}, 195 | EndTimestamp: metav1.Time{Time: createJobATime.Add(time.Second)}, 196 | RestartCount: 1, 197 | RunningHistories: []jobflowv1alpha1.JobRunningHistory{ 198 | { 199 | StartTimestamp: metav1.Time{}, 200 | EndTimestamp: metav1.Time{}, 201 | State: v1alpha1.Completed, 202 | }, 203 | }, 204 | }, 205 | { 206 | Name: "jobFlowA-B", 207 | State: v1alpha1.Running, 208 | StartTimestamp: metav1.Time{Time: createJobBTime}, 209 | EndTimestamp: metav1.Time{}, 210 | RestartCount: 0, 211 | RunningHistories: []jobflowv1alpha1.JobRunningHistory{ 212 | { 213 | StartTimestamp: metav1.Time{}, 214 | EndTimestamp: metav1.Time{}, 215 | State: v1alpha1.Running, 216 | }, 217 | }, 218 | }, 219 | }, 220 | Conditions: map[string]jobflowv1alpha1.Condition{ 221 | "jobFlowA-A": { 222 | Phase: v1alpha1.Completed, 223 | CreateTimestamp: metav1.Time{Time: createJobATime}, 224 | RunningDuration: &v1.Duration{Duration: time.Second}, 225 | }, 226 | "jobFlowA-B": { 227 | Phase: v1alpha1.Running, 228 | CreateTimestamp: metav1.Time{Time: createJobBTime}, 229 | }, 230 | }, 231 | State: jobflowv1alpha1.State{Phase: jobflowv1alpha1.Running}, 232 | }, 233 | wantErr: false, 234 | }, 235 | } 236 | for _, tt := range tests { 237 | t.Run(tt.name, func(t *testing.T) { 238 | got, err := getAllJobStatus(tt.args.jobFlow, tt.args.allJobList) 239 | if (err != nil) != tt.wantErr { 240 | t.Errorf("getAllJobStatus() error = %v, wantErr %v", err, tt.wantErr) 241 | return 242 | } 243 | got.JobStatusList[0].RunningHistories[0].StartTimestamp = metav1.Time{} 244 | got.JobStatusList[1].RunningHistories[0].StartTimestamp = metav1.Time{} 245 | if !reflect.DeepEqual(got, tt.want) { 246 | t.Errorf("getAllJobStatus() got = %v, want %v", got, tt.want) 247 | } 248 | }) 249 | } 250 | } 251 | -------------------------------------------------------------------------------- /controllers/jobtemplate.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2021. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package controllers 18 | 19 | import ( 20 | "context" 21 | "fmt" 22 | "strings" 23 | "time" 24 | 25 | batchv1alpha1 "jobflow/api/v1alpha1" 26 | jobflowv1alpha1 "jobflow/api/v1alpha1" 27 | "jobflow/utils" 28 | 29 | corev1 "k8s.io/api/core/v1" 30 | "k8s.io/apimachinery/pkg/api/errors" 31 | "k8s.io/apimachinery/pkg/runtime" 32 | "k8s.io/apimachinery/pkg/types" 33 | "k8s.io/client-go/tools/record" 34 | "k8s.io/client-go/util/workqueue" 35 | "k8s.io/klog" 36 | ctrl "sigs.k8s.io/controller-runtime" 37 | "sigs.k8s.io/controller-runtime/pkg/client" 38 | "sigs.k8s.io/controller-runtime/pkg/event" 39 | "sigs.k8s.io/controller-runtime/pkg/handler" 40 | "sigs.k8s.io/controller-runtime/pkg/reconcile" 41 | "sigs.k8s.io/controller-runtime/pkg/source" 42 | "volcano.sh/apis/pkg/apis/batch/v1alpha1" 43 | ) 44 | 45 | // JobTemplateReconciler reconciles a Job object 46 | type JobTemplateReconciler struct { 47 | client.Client 48 | Scheme *runtime.Scheme 49 | Recorder record.EventRecorder 50 | } 51 | 52 | //+kubebuilder:rbac:groups=flow.volcano.sh,resources=jobtemplates,verbs=get;list;watch;create;update;patch;delete 53 | //+kubebuilder:rbac:groups=flow.volcano.sh,resources=jobtemplates/status,verbs=get;update;patch 54 | //+kubebuilder:rbac:groups=flow.volcano.sh,resources=jobtemplates/finalizers,verbs=update 55 | // +kubebuilder:rbac:groups=batch.volcano.sh,resources=jobs,verbs=get;list;watch;create;update;patch;delete 56 | // +kubebuilder:rbac:groups=batch.volcano.sh,resources=jobs/status,verbs=get;update;patch 57 | // +kubebuilder:rbac:groups="",resources=events,verbs=create;patch 58 | 59 | // Reconcile is part of the main kubernetes reconciliation loop which aims to 60 | // move the current state of the cluster closer to the desired state. 61 | // TODO(user): Modify the Reconcile function to compare the state specified by 62 | // the Job object against the actual cluster state, and then 63 | // perform operations to make the cluster state reflect the state specified by 64 | // the user. 65 | // 66 | // For more details, check Reconcile and its Result here: 67 | // - https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.8.3/pkg/reconcile 68 | func (r *JobTemplateReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { 69 | klog.Info("start jobTemplate Reconcile..........") 70 | klog.Info(fmt.Sprintf("event for jobTemplate: %v", req.Name)) 71 | // your logic here 72 | scheduledResult := ctrl.Result{} 73 | 74 | // load JobTemplate by namespace 75 | jobTemplate := &jobflowv1alpha1.JobTemplate{} 76 | time.Sleep(time.Second) 77 | err := r.Get(ctx, req.NamespacedName, jobTemplate) 78 | if err != nil { 79 | //If no instance is found, it will be returned directly 80 | if errors.IsNotFound(err) { 81 | klog.Info(fmt.Sprintf("not fount jobTemplate : %v", req.Name)) 82 | return scheduledResult, nil 83 | } 84 | klog.Error(err, err.Error()) 85 | r.Recorder.Eventf(jobTemplate, corev1.EventTypeWarning, "Created", err.Error()) 86 | return scheduledResult, err 87 | } 88 | // search the jobs created by JobTemplate 89 | jobList := &v1alpha1.JobList{} 90 | err = r.List(ctx, jobList) 91 | if err != nil { 92 | klog.Error(err, "") 93 | return scheduledResult, err 94 | } 95 | filterJobList := make([]v1alpha1.Job, 0) 96 | for _, item := range jobList.Items { 97 | if item.Annotations[utils.CreateByJobTemplate] == utils.GetConnectionOfJobAndJobTemplate(req.Namespace, req.Name) { 98 | filterJobList = append(filterJobList, item) 99 | } 100 | } 101 | if len(filterJobList) == 0 { 102 | return scheduledResult, err 103 | } 104 | jobListName := make([]string, 0) 105 | for _, job := range filterJobList { 106 | jobListName = append(jobListName, job.Name) 107 | } 108 | jobTemplate.Status.JobDependsOnList = jobListName 109 | // update 110 | if err := r.Status().Update(ctx, jobTemplate); err != nil { 111 | klog.Error(err, "update error!") 112 | return scheduledResult, err 113 | } 114 | klog.Info("end jobTemplate Reconcile..........") 115 | return scheduledResult, nil 116 | } 117 | 118 | // SetupWithManager sets up the controller with the Manager. 119 | func (r *JobTemplateReconciler) SetupWithManager(mgr ctrl.Manager) error { 120 | return ctrl.NewControllerManagedBy(mgr). 121 | For(&batchv1alpha1.JobTemplate{}). 122 | Watches(&source.Kind{Type: &v1alpha1.Job{}}, handler.Funcs{CreateFunc: jobCreateHandler}). 123 | Complete(r) 124 | } 125 | 126 | func jobCreateHandler(e event.CreateEvent, w workqueue.RateLimitingInterface) { 127 | if e.Object.GetAnnotations()[utils.CreateByJobTemplate] != "" { 128 | nameNamespace := strings.Split(e.Object.GetAnnotations()[utils.CreateByJobTemplate], ".") 129 | namespace, name := nameNamespace[0], nameNamespace[1] 130 | w.AddRateLimited(reconcile.Request{NamespacedName: types.NamespacedName{Name: name, Namespace: namespace}}) 131 | } 132 | } 133 | -------------------------------------------------------------------------------- /controllers/suite_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2021. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package controllers 18 | 19 | import ( 20 | "path/filepath" 21 | "testing" 22 | 23 | . "github.com/onsi/ginkgo" 24 | . "github.com/onsi/gomega" 25 | "k8s.io/client-go/kubernetes/scheme" 26 | "k8s.io/client-go/rest" 27 | "sigs.k8s.io/controller-runtime/pkg/client" 28 | "sigs.k8s.io/controller-runtime/pkg/envtest" 29 | "sigs.k8s.io/controller-runtime/pkg/envtest/printer" 30 | logf "sigs.k8s.io/controller-runtime/pkg/log" 31 | "sigs.k8s.io/controller-runtime/pkg/log/zap" 32 | 33 | batchv1alpha1 "jobflow/api/v1alpha1" 34 | //+kubebuilder:scaffold:imports 35 | ) 36 | 37 | // These tests use Ginkgo (BDD-style Go testing framework). Refer to 38 | // http://onsi.github.io/ginkgo/ to learn more about Ginkgo. 39 | 40 | var cfg *rest.Config 41 | var k8sClient client.Client 42 | var testEnv *envtest.Environment 43 | 44 | func TestAPIs(t *testing.T) { 45 | RegisterFailHandler(Fail) 46 | 47 | RunSpecsWithDefaultAndCustomReporters(t, 48 | "Controller Suite", 49 | []Reporter{printer.NewlineReporter{}}) 50 | } 51 | 52 | var _ = BeforeSuite(func() { 53 | logf.SetLogger(zap.New(zap.WriteTo(GinkgoWriter), zap.UseDevMode(true))) 54 | 55 | By("bootstrapping test environment") 56 | testEnv = &envtest.Environment{ 57 | CRDDirectoryPaths: []string{filepath.Join("..", "config", "crd", "bases")}, 58 | ErrorIfCRDPathMissing: true, 59 | } 60 | 61 | cfg, err := testEnv.Start() 62 | Expect(err).NotTo(HaveOccurred()) 63 | Expect(cfg).NotTo(BeNil()) 64 | 65 | err = batchv1alpha1.AddToScheme(scheme.Scheme) 66 | Expect(err).NotTo(HaveOccurred()) 67 | 68 | err = batchv1alpha1.AddToScheme(scheme.Scheme) 69 | Expect(err).NotTo(HaveOccurred()) 70 | 71 | //+kubebuilder:scaffold:scheme 72 | 73 | k8sClient, err = client.New(cfg, client.Options{Scheme: scheme.Scheme}) 74 | Expect(err).NotTo(HaveOccurred()) 75 | Expect(k8sClient).NotTo(BeNil()) 76 | 77 | }, 60) 78 | 79 | var _ = AfterSuite(func() { 80 | By("tearing down the test environment") 81 | err := testEnv.Stop() 82 | Expect(err).NotTo(HaveOccurred()) 83 | }) 84 | -------------------------------------------------------------------------------- /docs/community/roadmap.md: -------------------------------------------------------------------------------- 1 | # JobFlow Roadmap 2 | 3 | ## v0.0.1 4 | 5 | * create JobFlow and JobTemplate CRD 6 | * Support sequential start of vcjob 7 | * Support vcjob to depend on other vcjobs to start 8 | * Support the conversion of vcjob and JobTemplate to each other 9 | * Supports viewing of the running status of JobFlow 10 | 11 | ## Later (To be updated) 12 | 13 | * `if` statements 14 | * `switch` statements 15 | * for statements 16 | * Support job failure retry in JobFlow 17 | * Integration with volcano-scheduler 18 | * Support for scheduling plugins at JobFlow level -------------------------------------------------------------------------------- /docs/design/jobflow.md: -------------------------------------------------------------------------------- 1 | ## JobFlow 2 | 3 | ### Introduction 4 | 5 | JobFlow defines the running flow of a set of jobs. Fields in JobFlow define how jobs are orchestrated. 6 | 7 | ### Definition 8 | 9 | JobFlow mainly has the following fields: 10 | 11 | * spec.jobretainpolicy: After JobFlow runs, keep the generated job. Otherwise, delete it. 12 | * flows.name: Indicates the name of the vcjob. 13 | * flows.dependsOn.targets: Indicates the name of the vcjob that the above vcjob depends on, which can be one or multiple vcjobs 14 | 15 | [the sample file of JobFlow](../../example/JobFlow.yaml) -------------------------------------------------------------------------------- /docs/design/jobtemplate.md: -------------------------------------------------------------------------------- 1 | ## JobTemplate 2 | 3 | ### Introduction 4 | 5 | * JobTemplate is the template of vcjob, after JobTemplate is created, it will not be processed by vc-controller like vcjob, it will wait to be referenced by JobFlow. 6 | * JobTemplate can be referenced multiple times by JobFlow. 7 | * JobTemplate can be converted to and from vcjob. 8 | 9 | ### Definition 10 | 11 | The spec field of JobTemplate is exactly the same as that of vcjob. You can view [the sample file of JobTemplate](../../example/JobTemplate.yaml) -------------------------------------------------------------------------------- /docs/images/jobflow.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BoCloud/JobFlow/ddf4475afadc9da6a7e9149498121b447f838c49/docs/images/jobflow.gif -------------------------------------------------------------------------------- /example/JobFlow.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: flow.volcano.sh/v1alpha1 2 | kind: JobFlow 3 | metadata: 4 | name: test 5 | namespace: default 6 | spec: 7 | jobRetainPolicy: delete # After jobflow runs, keep the generated job. Otherwise, delete it. 8 | flows: 9 | - name: a 10 | - name: b 11 | dependsOn: 12 | targets: ['a'] 13 | - name: c 14 | dependsOn: 15 | targets: ['b'] 16 | - name: d 17 | dependsOn: 18 | targets: ['b'] 19 | - name: e 20 | dependsOn: 21 | targets: ['c','d'] 22 | -------------------------------------------------------------------------------- /example/JobTemplate.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: flow.volcano.sh/v1alpha1 2 | kind: JobTemplate 3 | metadata: 4 | name: a 5 | spec: 6 | minAvailable: 1 7 | schedulerName: volcano 8 | priorityClassName: high-priority 9 | policies: 10 | - event: PodEvicted 11 | action: RestartJob 12 | plugins: 13 | ssh: [] 14 | env: [] 15 | svc: [] 16 | maxRetry: 5 17 | queue: default 18 | tasks: 19 | - replicas: 1 20 | name: "default-nginx" 21 | template: 22 | metadata: 23 | name: web 24 | spec: 25 | containers: 26 | - image: nginx:1.14.2 27 | command: 28 | - sh 29 | - -c 30 | - sleep 10s 31 | imagePullPolicy: IfNotPresent 32 | name: nginx 33 | resources: 34 | requests: 35 | cpu: "1" 36 | restartPolicy: OnFailure 37 | --- 38 | apiVersion: flow.volcano.sh/v1alpha1 39 | kind: JobTemplate 40 | metadata: 41 | name: b 42 | spec: 43 | minAvailable: 1 44 | schedulerName: volcano 45 | priorityClassName: high-priority 46 | policies: 47 | - event: PodEvicted 48 | action: RestartJob 49 | plugins: 50 | ssh: [] 51 | env: [] 52 | svc: [] 53 | maxRetry: 5 54 | queue: default 55 | tasks: 56 | - replicas: 1 57 | name: "default-nginx" 58 | template: 59 | metadata: 60 | name: web 61 | spec: 62 | containers: 63 | - image: nginx:1.14.2 64 | command: 65 | - sh 66 | - -c 67 | - sleep 10s 68 | imagePullPolicy: IfNotPresent 69 | name: nginx 70 | resources: 71 | requests: 72 | cpu: "1" 73 | restartPolicy: OnFailure 74 | --- 75 | apiVersion: flow.volcano.sh/v1alpha1 76 | kind: JobTemplate 77 | metadata: 78 | name: c 79 | spec: 80 | minAvailable: 1 81 | schedulerName: volcano 82 | priorityClassName: high-priority 83 | policies: 84 | - event: PodEvicted 85 | action: RestartJob 86 | plugins: 87 | ssh: [] 88 | env: [] 89 | svc: [] 90 | maxRetry: 5 91 | queue: default 92 | tasks: 93 | - replicas: 1 94 | name: "default-nginx" 95 | template: 96 | metadata: 97 | name: web 98 | spec: 99 | containers: 100 | - image: nginx:1.14.2 101 | command: 102 | - sh 103 | - -c 104 | - sleep 10s 105 | imagePullPolicy: IfNotPresent 106 | name: nginx 107 | resources: 108 | requests: 109 | cpu: "1" 110 | restartPolicy: OnFailure 111 | --- 112 | apiVersion: flow.volcano.sh/v1alpha1 113 | kind: JobTemplate 114 | metadata: 115 | name: d 116 | spec: 117 | minAvailable: 1 118 | schedulerName: volcano 119 | priorityClassName: high-priority 120 | policies: 121 | - event: PodEvicted 122 | action: RestartJob 123 | plugins: 124 | ssh: [] 125 | env: [] 126 | svc: [] 127 | maxRetry: 5 128 | queue: default 129 | tasks: 130 | - replicas: 1 131 | name: "default-nginx" 132 | template: 133 | metadata: 134 | name: web 135 | spec: 136 | containers: 137 | - image: nginx:1.14.2 138 | command: 139 | - sh 140 | - -c 141 | - sleep 10s 142 | imagePullPolicy: IfNotPresent 143 | name: nginx 144 | resources: 145 | requests: 146 | cpu: "1" 147 | restartPolicy: OnFailure 148 | --- 149 | apiVersion: flow.volcano.sh/v1alpha1 150 | kind: JobTemplate 151 | metadata: 152 | name: e 153 | spec: 154 | minAvailable: 1 155 | schedulerName: volcano 156 | priorityClassName: high-priority 157 | policies: 158 | - event: PodEvicted 159 | action: RestartJob 160 | plugins: 161 | ssh: [] 162 | env: [] 163 | svc: [] 164 | maxRetry: 5 165 | queue: default 166 | tasks: 167 | - replicas: 1 168 | name: "default-nginx" 169 | template: 170 | metadata: 171 | name: web 172 | spec: 173 | containers: 174 | - image: nginx:1.14.2 175 | command: 176 | - sh 177 | - -c 178 | - sleep 10s 179 | imagePullPolicy: IfNotPresent 180 | name: nginx 181 | resources: 182 | requests: 183 | cpu: "1" 184 | restartPolicy: OnFailure -------------------------------------------------------------------------------- /example/README.md: -------------------------------------------------------------------------------- 1 | ## Fields definition 2 | 3 | ### JobTemplate 4 | 5 | `Jobtemplate` is abbreviated as `jt`, and the resource can be viewed through `kubectl get jt` 6 | 7 | jobtemplate and vcjob can be converted to each other through vcctl 8 | 9 | The difference between jobtemplate and vcjob is that jobtemplate will not be issued by the job controller, and jobflow can directly reference the name of the JobTemplate to implement the issuance of vcjob. 10 | 11 | ### JobFlow 12 | 13 | `jobflow` is abbreviated as `jf`, and the resource can be viewed through `kubectl get jf` 14 | 15 | jobflow aims to realize job-dependent operation between vcjobs in volcano. According to the dependency between vcjob, vcjob is issued. 16 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module jobflow 2 | 3 | go 1.16 4 | 5 | require ( 6 | github.com/eapache/queue v1.1.0 7 | github.com/fsnotify/fsnotify v1.5.1 // indirect 8 | github.com/hashicorp/go-multierror v1.0.0 9 | github.com/onsi/ginkgo v1.16.4 10 | github.com/onsi/ginkgo/v2 v2.0.0 11 | github.com/onsi/gomega v1.17.0 12 | golang.org/x/sys v0.0.0-20220406163625-3f8b81556e12 // indirect 13 | k8s.io/api v0.20.2 14 | k8s.io/apimachinery v0.20.2 15 | k8s.io/client-go v0.20.2 16 | k8s.io/klog v1.0.0 17 | k8s.io/kubernetes v1.19.6 18 | sigs.k8s.io/controller-runtime v0.8.3 19 | volcano.sh/apis v0.0.0-20210603070204-70005b2d502a 20 | 21 | ) 22 | 23 | replace ( 24 | k8s.io/api => k8s.io/api v0.19.6 25 | k8s.io/apiextensions-apiserver => k8s.io/apiextensions-apiserver v0.19.6 26 | k8s.io/apimachinery => k8s.io/apimachinery v0.19.6 27 | k8s.io/apiserver => k8s.io/apiserver v0.19.6 28 | k8s.io/cli-runtime => k8s.io/cli-runtime v0.19.6 29 | k8s.io/client-go => k8s.io/client-go v0.19.6 30 | k8s.io/cloud-provider => k8s.io/cloud-provider v0.19.6 31 | k8s.io/cluster-bootstrap => k8s.io/cluster-bootstrap v0.19.6 32 | k8s.io/code-generator => k8s.io/code-generator v0.19.6 33 | k8s.io/component-base => k8s.io/component-base v0.19.6 34 | k8s.io/cri-api => k8s.io/cri-api v0.19.6 35 | k8s.io/csi-translation-lib => k8s.io/csi-translation-lib v0.19.6 36 | k8s.io/kube-aggregator => k8s.io/kube-aggregator v0.19.6 37 | k8s.io/kube-controller-manager => k8s.io/kube-controller-manager v0.19.6 38 | k8s.io/kube-proxy => k8s.io/kube-proxy v0.19.6 39 | k8s.io/kube-scheduler => k8s.io/kube-scheduler v0.19.6 40 | k8s.io/kubectl => k8s.io/kubectl v0.19.6 41 | k8s.io/kubelet => k8s.io/kubelet v0.19.6 42 | k8s.io/legacy-cloud-providers => k8s.io/legacy-cloud-providers v0.19.6 43 | k8s.io/metrics => k8s.io/metrics v0.19.6 44 | k8s.io/sample-apiserver => k8s.io/sample-apiserver v0.19.6 45 | ) 46 | -------------------------------------------------------------------------------- /hack/boilerplate.go.txt: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2021. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ -------------------------------------------------------------------------------- /hack/run-e2e.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | export E2E_TYPE=${E2E_TYPE:-"ALL"} 4 | 5 | # Run e2e test 6 | 7 | go install -mod=mod github.com/onsi/ginkgo/v2/ginkgo 8 | 9 | case ${E2E_TYPE} in 10 | "ALL") 11 | echo "Running e2e..." 12 | KUBECONFIG=${KUBECONFIG} ginkgo -r --slow-spec-threshold='30s' --progress ./test/e2e/jobtemplate-controller/ 13 | KUBECONFIG=${KUBECONFIG} ginkgo -r --slow-spec-threshold='30s' --progress ./test/e2e/jobflow-controller/ 14 | KUBECONFIG=${KUBECONFIG} ginkgo -r --slow-spec-threshold='30s' --progress ./test/e2e/jobflow-admission/ 15 | KUBECONFIG=${KUBECONFIG} ginkgo -r --slow-spec-threshold='30s' --progress ./test/e2e/jobtemplate-admission/ 16 | ;; 17 | "JOBTEMPLATECONTROLLER") 18 | echo "Running jobtemplate controller e2e suite..." 19 | KUBECONFIG=${KUBECONFIG} ginkgo -r --slow-spec-threshold='30s' --progress ./test/e2e/jobtemplate-controller/ 20 | ;; 21 | "JOBFLOWCONTROLLER") 22 | echo "Running jobflow controller e2e suite..." 23 | KUBECONFIG=${KUBECONFIG} ginkgo -r --slow-spec-threshold='30s' --progress ./test/e2e/jobflow-controller/ 24 | ;; 25 | "JOBFLOWADMISSION") 26 | echo "Running jobflow admission e2e suite..." 27 | KUBECONFIG=${KUBECONFIG} ginkgo -r --slow-spec-threshold='30s' --progress ./test/e2e/jobflow-admission/ 28 | ;; 29 | "JOBTEMPLATEADMISSION") 30 | echo "Running jobtemplate admission e2e suite..." 31 | KUBECONFIG=${KUBECONFIG} ginkgo -r --slow-spec-threshold='30s' --progress ./test/e2e/jobtemplate-admission/ 32 | ;; 33 | esac 34 | 35 | -------------------------------------------------------------------------------- /main.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2021. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package main 18 | 19 | import ( 20 | "flag" 21 | "os" 22 | 23 | // Import all Kubernetes client auth plugins (e.g. Azure, GCP, OIDC, etc.) 24 | // to ensure that exec-entrypoint and run can make use of them. 25 | _ "k8s.io/client-go/plugin/pkg/client/auth" 26 | 27 | "k8s.io/apimachinery/pkg/runtime" 28 | utilruntime "k8s.io/apimachinery/pkg/util/runtime" 29 | clientgoscheme "k8s.io/client-go/kubernetes/scheme" 30 | ctrl "sigs.k8s.io/controller-runtime" 31 | "sigs.k8s.io/controller-runtime/pkg/healthz" 32 | "sigs.k8s.io/controller-runtime/pkg/log/zap" 33 | "volcano.sh/apis/pkg/apis/batch/v1alpha1" 34 | 35 | batchv1alpha1 "jobflow/api/v1alpha1" 36 | "jobflow/controllers" 37 | "jobflow/webhooks" 38 | _ "jobflow/webhooks/admission/jobflow/validate" 39 | _ "jobflow/webhooks/admission/template/validate" 40 | //+kubebuilder:scaffold:imports 41 | ) 42 | 43 | var ( 44 | scheme = runtime.NewScheme() 45 | setupLog = ctrl.Log.WithName("setup") 46 | ) 47 | 48 | func init() { 49 | utilruntime.Must(clientgoscheme.AddToScheme(scheme)) 50 | 51 | utilruntime.Must(batchv1alpha1.AddToScheme(scheme)) 52 | //+kubebuilder:scaffold:scheme 53 | 54 | utilruntime.Must(v1alpha1.AddToScheme(scheme)) 55 | } 56 | 57 | func main() { 58 | var metricsAddr string 59 | var enableLeaderElection bool 60 | var probeAddr string 61 | flag.StringVar(&metricsAddr, "metrics-bind-address", ":8080", "The address the metric endpoint binds to.") 62 | flag.StringVar(&probeAddr, "health-probe-bind-address", ":8081", "The address the probe endpoint binds to.") 63 | flag.BoolVar(&enableLeaderElection, "leader-elect", false, 64 | "Enable leader election for controller manager. "+ 65 | "Enabling this will ensure there is only one active controller manager.") 66 | opts := zap.Options{ 67 | Development: true, 68 | } 69 | opts.BindFlags(flag.CommandLine) 70 | flag.Parse() 71 | 72 | ctrl.SetLogger(zap.New(zap.UseFlagOptions(&opts))) 73 | 74 | mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{ 75 | Scheme: scheme, 76 | MetricsBindAddress: metricsAddr, 77 | Port: 9443, 78 | HealthProbeBindAddress: probeAddr, 79 | LeaderElection: enableLeaderElection, 80 | LeaderElectionID: "1b1c5f74.volcano.sh", 81 | }) 82 | if err != nil { 83 | setupLog.Error(err, "unable to start manager") 84 | os.Exit(1) 85 | } 86 | 87 | if err = (&controllers.JobFlowReconciler{ 88 | Client: mgr.GetClient(), 89 | Scheme: mgr.GetScheme(), 90 | Recorder: mgr.GetEventRecorderFor("containerset-controller"), 91 | }).SetupWithManager(mgr); err != nil { 92 | setupLog.Error(err, "unable to create controller", "controller", "JobFlow") 93 | os.Exit(1) 94 | } 95 | if err = (&controllers.JobTemplateReconciler{ 96 | Client: mgr.GetClient(), 97 | Scheme: mgr.GetScheme(), 98 | Recorder: mgr.GetEventRecorderFor("containerset-controller"), 99 | }).SetupWithManager(mgr); err != nil { 100 | setupLog.Error(err, "unable to create controller", "controller", "Job") 101 | os.Exit(1) 102 | } 103 | 104 | if err = webhooks.Run(); err != nil { 105 | setupLog.Error(err, "unable to create webhook", "webhook", "jobFlow or jobTemplate") 106 | os.Exit(1) 107 | } 108 | //+kubebuilder:scaffold:builder 109 | 110 | if err := mgr.AddHealthzCheck("healthz", healthz.Ping); err != nil { 111 | setupLog.Error(err, "unable to set up health check") 112 | os.Exit(1) 113 | } 114 | if err := mgr.AddReadyzCheck("readyz", healthz.Ping); err != nil { 115 | setupLog.Error(err, "unable to set up ready check") 116 | os.Exit(1) 117 | } 118 | 119 | setupLog.Info("starting manager") 120 | if err := mgr.Start(ctrl.SetupSignalHandler()); err != nil { 121 | setupLog.Error(err, "problem running manager") 122 | os.Exit(1) 123 | } 124 | } 125 | -------------------------------------------------------------------------------- /test/e2e/jobflow-admission/admission.go: -------------------------------------------------------------------------------- 1 | package jobflow_admission 2 | 3 | import ( 4 | . "github.com/onsi/ginkgo/v2" 5 | . "github.com/onsi/gomega" 6 | 7 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 8 | 9 | jobflowv1alpha1 "jobflow/api/v1alpha1" 10 | e2eutil "jobflow/test/e2e/util" 11 | ) 12 | 13 | var _ = Describe("JobFlow E2E Test: Test Admission service", func() { 14 | 15 | It("jobFlow validate check: duplicate job name check when create", func() { 16 | ctx := e2eutil.InitTestContext(e2eutil.Options{}) 17 | defer e2eutil.CleanupTestContext(ctx) 18 | 19 | jobFlow := &jobflowv1alpha1.JobFlow{ 20 | ObjectMeta: metav1.ObjectMeta{ 21 | Name: "test-jobflow", 22 | Namespace: ctx.Namespace, 23 | }, 24 | Spec: jobflowv1alpha1.JobFlowSpec{ 25 | Flows: []jobflowv1alpha1.Flow{ 26 | { 27 | Name: "job-a", 28 | DependsOn: &jobflowv1alpha1.DependsOn{ 29 | Targets: []string{}, 30 | }, 31 | }, 32 | { 33 | Name: "job-b", 34 | DependsOn: &jobflowv1alpha1.DependsOn{ 35 | Targets: []string{}, 36 | }, 37 | }, 38 | { 39 | Name: "job-b", 40 | DependsOn: &jobflowv1alpha1.DependsOn{ 41 | Targets: []string{}, 42 | }, 43 | }, 44 | }, 45 | JobRetainPolicy: jobflowv1alpha1.Retain, 46 | }, 47 | } 48 | 49 | _, err := e2eutil.CreateJobFlowInner(ctx, jobFlow) 50 | Expect(err).To(MatchError(ContainSubstring(`duplicated template name job-b`))) 51 | }) 52 | 53 | It("jobFlow validate check: targets name check when create", func() { 54 | ctx := e2eutil.InitTestContext(e2eutil.Options{}) 55 | defer e2eutil.CleanupTestContext(ctx) 56 | 57 | jobFlow := &jobflowv1alpha1.JobFlow{ 58 | ObjectMeta: metav1.ObjectMeta{ 59 | Name: "test-jobflow", 60 | Namespace: ctx.Namespace, 61 | }, 62 | Spec: jobflowv1alpha1.JobFlowSpec{ 63 | Flows: []jobflowv1alpha1.Flow{ 64 | { 65 | Name: "job-a", 66 | DependsOn: &jobflowv1alpha1.DependsOn{ 67 | Targets: []string{}, 68 | }, 69 | }, 70 | { 71 | Name: "job-b", 72 | DependsOn: &jobflowv1alpha1.DependsOn{ 73 | Targets: []string{"unKnow-job"}, 74 | }, 75 | }, 76 | }, 77 | JobRetainPolicy: jobflowv1alpha1.Retain, 78 | }, 79 | } 80 | 81 | _, err := e2eutil.CreateJobFlowInner(ctx, jobFlow) 82 | Expect(err).To(MatchError(ContainSubstring(`cannot find the template: unKnow-job`))) 83 | }) 84 | 85 | It("jobFlow validate check: dependencies check when create", func() { 86 | ctx := e2eutil.InitTestContext(e2eutil.Options{}) 87 | defer e2eutil.CleanupTestContext(ctx) 88 | 89 | jobFlow := &jobflowv1alpha1.JobFlow{ 90 | ObjectMeta: metav1.ObjectMeta{ 91 | Name: "test-jobflow", 92 | Namespace: ctx.Namespace, 93 | }, 94 | Spec: jobflowv1alpha1.JobFlowSpec{ 95 | Flows: []jobflowv1alpha1.Flow{ 96 | { 97 | Name: "job-a", 98 | DependsOn: &jobflowv1alpha1.DependsOn{ 99 | Targets: []string{"job-b", "job-c"}, 100 | }, 101 | }, 102 | { 103 | Name: "job-b", 104 | DependsOn: &jobflowv1alpha1.DependsOn{ 105 | Targets: []string{"job-a", "job-c"}, 106 | }, 107 | }, 108 | { 109 | Name: "job-c", 110 | DependsOn: &jobflowv1alpha1.DependsOn{ 111 | Targets: []string{"job-a", "job-b"}, 112 | }, 113 | }, 114 | }, 115 | JobRetainPolicy: jobflowv1alpha1.Retain, 116 | }, 117 | } 118 | 119 | _, err := e2eutil.CreateJobFlowInner(ctx, jobFlow) 120 | Expect(err).To(MatchError(ContainSubstring(`find bad dependency`))) 121 | }) 122 | 123 | }) 124 | -------------------------------------------------------------------------------- /test/e2e/jobflow-admission/e2e_test.go: -------------------------------------------------------------------------------- 1 | package jobflow_admission 2 | 3 | import ( 4 | "testing" 5 | 6 | . "github.com/onsi/ginkgo/v2" 7 | . "github.com/onsi/gomega" 8 | 9 | e2eutil "jobflow/test/e2e/util" 10 | ) 11 | 12 | func TestE2E(t *testing.T) { 13 | RegisterFailHandler(Fail) 14 | RunSpecs(t, "Volcano JobFlow Admission Test Suite") 15 | e2eutil.Cancel() 16 | } 17 | -------------------------------------------------------------------------------- /test/e2e/jobflow-admission/main_test.go: -------------------------------------------------------------------------------- 1 | package jobflow_admission 2 | 3 | import ( 4 | "os" 5 | "testing" 6 | 7 | e2eutil "jobflow/test/e2e/util" 8 | ) 9 | 10 | func TestMain(m *testing.M) { 11 | mgr := e2eutil.NewManager() 12 | e2eutil.JobFlowReconciler = e2eutil.NewJobFlowReconciler(mgr) 13 | e2eutil.JobTemplateReconciler = e2eutil.NewJobTemplateReconciler(mgr) 14 | e2eutil.StartMgr(mgr) 15 | e2eutil.InitKubeClient() 16 | os.Exit(m.Run()) 17 | } 18 | -------------------------------------------------------------------------------- /test/e2e/jobflow-controller/e2e_test.go: -------------------------------------------------------------------------------- 1 | package jobflow_controller 2 | 3 | import ( 4 | "testing" 5 | 6 | . "github.com/onsi/ginkgo/v2" 7 | . "github.com/onsi/gomega" 8 | "jobflow/test/e2e/util" 9 | ) 10 | 11 | func TestE2E(t *testing.T) { 12 | RegisterFailHandler(Fail) 13 | RunSpecs(t, "Volcano Job Seq Test Suite") 14 | util.Cancel() 15 | } 16 | -------------------------------------------------------------------------------- /test/e2e/jobflow-controller/jobflow.go: -------------------------------------------------------------------------------- 1 | package jobflow_controller 2 | 3 | import ( 4 | "context" 5 | "time" 6 | 7 | . "github.com/onsi/ginkgo/v2" 8 | . "github.com/onsi/gomega" 9 | "jobflow/test/e2e/util" 10 | "k8s.io/apimachinery/pkg/api/errors" 11 | "k8s.io/apimachinery/pkg/types" 12 | "k8s.io/apimachinery/pkg/util/wait" 13 | "volcano.sh/apis/pkg/apis/batch/v1alpha1" 14 | ) 15 | 16 | var _ = Describe("JobFlow E2E Test", func() { 17 | It("will create success and deploy by flow", func() { 18 | ctx := util.InitTestContext(util.Options{}) 19 | defer util.CleanupTestContext(ctx) 20 | 21 | //create jobtemplateA and jobtemplateB 22 | jobTemplateA := util.GetJobTemplateInstance("jobtemplate-a") 23 | jobTemplateB := util.GetJobTemplateInstance("jobtemplate-b") 24 | util.CreateJobTemplate(ctx, jobTemplateA) 25 | util.CreateJobTemplate(ctx, jobTemplateB) 26 | 27 | jobflow := util.GetFlowInstance("jobflowtest") 28 | 29 | jobFlowRes := util.CreateJobFlow(ctx, jobflow) 30 | err := wait.Poll(100*time.Millisecond, util.OneMinute, util.JobFlowExist(ctx, jobFlowRes)) 31 | Expect(err).NotTo(HaveOccurred(), "failed to wait for JobFlow created") 32 | 33 | err = wait.Poll(100*time.Millisecond, util.FiveMinute, util.VcJobExist(ctx, jobFlowRes, jobTemplateA)) 34 | Expect(err).NotTo(HaveOccurred(), "failed to wait for vcjob created") 35 | 36 | err = wait.Poll(100*time.Millisecond, util.FiveMinute, func() (done bool, err error) { 37 | jobB := types.NamespacedName{ 38 | Namespace: jobTemplateB.Namespace, 39 | Name: util.GetJobName(jobflow.Name, jobTemplateB.Name), 40 | } 41 | jobBGet := &v1alpha1.Job{} 42 | err = ctx.JobFlowReconciler.Get(context.TODO(), jobB, jobBGet) 43 | if err != nil { 44 | if errors.IsNotFound(err) { 45 | return false, nil 46 | } 47 | return false, err 48 | } 49 | jobA := types.NamespacedName{ 50 | Namespace: jobTemplateA.Namespace, 51 | Name: util.GetJobName(jobflow.Name, jobTemplateA.Name), 52 | } 53 | jobAGet := &v1alpha1.Job{} 54 | err = ctx.JobFlowReconciler.Get(context.TODO(), jobA, jobAGet) 55 | if err != nil { 56 | if errors.IsNotFound(err) { 57 | return false, nil 58 | } 59 | return false, err 60 | } 61 | if jobAGet.Status.State.Phase == v1alpha1.Completed && jobBGet.Status.State.Phase != v1alpha1.Completed { 62 | return true, nil 63 | } 64 | return false, nil 65 | }) 66 | Expect(err).NotTo(HaveOccurred(), "failed to wait for JobFlow success running") 67 | }) 68 | It("will delete success ", func() { 69 | ctx := util.InitTestContext(util.Options{}) 70 | defer util.CleanupTestContext(ctx) 71 | 72 | jobFlow := util.GetFlowInstance("jobflowtest") 73 | 74 | util.CreateJobFlow(ctx, jobFlow) 75 | 76 | jobFlowRes := util.DeleteJobFlow(ctx, jobFlow) 77 | err := wait.Poll(100*time.Millisecond, util.OneMinute, util.JobFlowNotExist(ctx, jobFlowRes)) 78 | Expect(err).NotTo(HaveOccurred(), "failed to wait for JobFlow created") 79 | }) 80 | It("will update status success ", func() { 81 | ctx := util.InitTestContext(util.Options{}) 82 | defer util.CleanupTestContext(ctx) 83 | 84 | jobFlow := util.GetFlowInstance("jobflowtest") 85 | 86 | jobFlowRes := util.CreateJobFlow(ctx, jobFlow) 87 | 88 | jobFlowRes.Status.PendingJobs = []string{"jobA"} 89 | 90 | jobFlowUpdateRes := util.UpdateJobFlowStatus(ctx, jobFlow) 91 | err := wait.Poll(100*time.Millisecond, util.OneMinute, func() (done bool, err error) { 92 | if jobFlowUpdateRes.Status.PendingJobs != nil && len(jobFlowUpdateRes.Status.PendingJobs) > 0 { 93 | return true, nil 94 | } 95 | return false, nil 96 | }) 97 | Expect(err).NotTo(HaveOccurred(), "failed to wait for JobFlowStatus updated") 98 | }) 99 | It("will delete all completed vcJobs success ", func() { 100 | ctx := util.InitTestContext(util.Options{}) 101 | defer util.CleanupTestContext(ctx) 102 | 103 | //create jobtemplateA and jobtemplateB 104 | jobTemplateA := util.GetJobTemplateInstance("jobtemplate-a") 105 | jobTemplateB := util.GetJobTemplateInstance("jobtemplate-b") 106 | util.CreateJobTemplate(ctx, jobTemplateA) 107 | util.CreateJobTemplate(ctx, jobTemplateB) 108 | 109 | jobflow := util.GetFlowInstanceRetainPolicyDelete("jobflowtest") 110 | 111 | jobFlowRes := util.CreateJobFlow(ctx, jobflow) 112 | err := wait.Poll(100*time.Millisecond, util.OneMinute, util.JobFlowExist(ctx, jobFlowRes)) 113 | Expect(err).NotTo(HaveOccurred(), "failed to wait for JobFlow created") 114 | 115 | err = wait.Poll(100*time.Millisecond, util.FiveMinute, util.VcJobExist(ctx, jobFlowRes, jobTemplateA)) 116 | Expect(err).NotTo(HaveOccurred(), "failed to wait for vcjob created") 117 | 118 | err = wait.Poll(100*time.Millisecond, util.FiveMinute, util.VcJobExist(ctx, jobFlowRes, jobTemplateB)) 119 | Expect(err).NotTo(HaveOccurred(), "failed to wait for vcjob created") 120 | 121 | err = wait.Poll(100*time.Millisecond, util.FiveMinute, util.VcJobNotExist(ctx, jobFlowRes, jobTemplateA)) 122 | Expect(err).NotTo(HaveOccurred(), "failed to wait for vcjob deleted") 123 | 124 | err = wait.Poll(100*time.Millisecond, util.FiveMinute, util.VcJobNotExist(ctx, jobFlowRes, jobTemplateB)) 125 | Expect(err).NotTo(HaveOccurred(), "failed to wait for vcjob deleted") 126 | }) 127 | }) 128 | -------------------------------------------------------------------------------- /test/e2e/jobflow-controller/main_test.go: -------------------------------------------------------------------------------- 1 | package jobflow_controller 2 | 3 | import ( 4 | "os" 5 | "testing" 6 | 7 | "jobflow/test/e2e/util" 8 | ) 9 | 10 | func TestMain(m *testing.M) { 11 | mgr := util.NewManager() 12 | util.JobFlowReconciler = util.NewJobFlowReconciler(mgr) 13 | util.JobTemplateReconciler = util.NewJobTemplateReconciler(mgr) 14 | util.StartMgr(mgr) 15 | util.InitKubeClient() 16 | os.Exit(m.Run()) 17 | } 18 | -------------------------------------------------------------------------------- /test/e2e/jobtemplate-admission/admission.go: -------------------------------------------------------------------------------- 1 | package jobtemplate_admission 2 | 3 | import ( 4 | . "github.com/onsi/ginkgo/v2" 5 | . "github.com/onsi/gomega" 6 | 7 | corev1 "k8s.io/api/core/v1" 8 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 9 | "volcano.sh/apis/pkg/apis/batch/v1alpha1" 10 | busv1alpha1 "volcano.sh/apis/pkg/apis/bus/v1alpha1" 11 | 12 | jobflowv1alpha1 "jobflow/api/v1alpha1" 13 | e2eutil "jobflow/test/e2e/util" 14 | ) 15 | 16 | var _ = Describe("JobTemplate E2E Test: Test Admission service", func() { 17 | 18 | It("jobTemplate validate check: duplicate task name check when create", func() { 19 | ctx := e2eutil.InitTestContext(e2eutil.Options{}) 20 | defer e2eutil.CleanupTestContext(ctx) 21 | 22 | jobTemplate := &jobflowv1alpha1.JobTemplate{ 23 | ObjectMeta: metav1.ObjectMeta{ 24 | Name: "test-job-template", 25 | Namespace: ctx.Namespace, 26 | }, 27 | Spec: v1alpha1.JobSpec{ 28 | SchedulerName: "volcano", 29 | MinAvailable: 1, 30 | Tasks: []v1alpha1.TaskSpec{ 31 | { 32 | Name: "task1", 33 | Replicas: 1, 34 | Template: corev1.PodTemplateSpec{ 35 | Spec: corev1.PodSpec{ 36 | Containers: []corev1.Container{ 37 | { 38 | Name: "nginx", 39 | Image: "nginx:1.14.2", 40 | }, 41 | }, 42 | }, 43 | }, 44 | }, 45 | { 46 | Name: "task1", 47 | Replicas: 1, 48 | Template: corev1.PodTemplateSpec{ 49 | Spec: corev1.PodSpec{ 50 | Containers: []corev1.Container{ 51 | { 52 | Name: "nginx", 53 | Image: "nginx:1.14.2", 54 | }, 55 | }, 56 | }, 57 | }, 58 | }, 59 | }, 60 | }, 61 | } 62 | 63 | _, err := e2eutil.CreateJobTemplateInner(ctx, jobTemplate) 64 | Expect(err).To(MatchError(ContainSubstring(`duplicated task name task1`))) 65 | }) 66 | 67 | It("jobTemplate validate check: minAvailable larger than replicas when create", func() { 68 | ctx := e2eutil.InitTestContext(e2eutil.Options{}) 69 | defer e2eutil.CleanupTestContext(ctx) 70 | 71 | jobTemplate := &jobflowv1alpha1.JobTemplate{ 72 | ObjectMeta: metav1.ObjectMeta{ 73 | Name: "test-job-template", 74 | Namespace: ctx.Namespace, 75 | }, 76 | Spec: v1alpha1.JobSpec{ 77 | SchedulerName: "volcano", 78 | MinAvailable: 2, 79 | Tasks: []v1alpha1.TaskSpec{ 80 | { 81 | Name: "task1", 82 | Replicas: 1, 83 | Template: corev1.PodTemplateSpec{ 84 | Spec: corev1.PodSpec{ 85 | Containers: []corev1.Container{ 86 | { 87 | Name: "nginx", 88 | Image: "nginx:1.14.2", 89 | }, 90 | }, 91 | }, 92 | }, 93 | }, 94 | }, 95 | }, 96 | } 97 | 98 | _, err := e2eutil.CreateJobTemplateInner(ctx, jobTemplate) 99 | Expect(err).To(MatchError(ContainSubstring(`job 'minAvailable' should not be greater than total replicas in tasks`))) 100 | }) 101 | 102 | It("jobTemplate validate check: no task specified in the jobTemplate when create", func() { 103 | ctx := e2eutil.InitTestContext(e2eutil.Options{}) 104 | defer e2eutil.CleanupTestContext(ctx) 105 | 106 | jobTemplate := &jobflowv1alpha1.JobTemplate{ 107 | ObjectMeta: metav1.ObjectMeta{ 108 | Name: "test-job-template", 109 | Namespace: ctx.Namespace, 110 | }, 111 | Spec: v1alpha1.JobSpec{ 112 | SchedulerName: "volcano", 113 | MinAvailable: 1, 114 | Tasks: []v1alpha1.TaskSpec{}, 115 | }, 116 | } 117 | 118 | _, err := e2eutil.CreateJobTemplateInner(ctx, jobTemplate) 119 | Expect(err).To(MatchError(ContainSubstring(`no task specified in job spec`))) 120 | }) 121 | 122 | It("jobTemplate validate check: invalid policy action when create", func() { 123 | ctx := e2eutil.InitTestContext(e2eutil.Options{}) 124 | defer e2eutil.CleanupTestContext(ctx) 125 | 126 | jobTemplate := &jobflowv1alpha1.JobTemplate{ 127 | ObjectMeta: metav1.ObjectMeta{ 128 | Name: "test-job-template", 129 | Namespace: ctx.Namespace, 130 | }, 131 | Spec: v1alpha1.JobSpec{ 132 | SchedulerName: "volcano", 133 | Tasks: []v1alpha1.TaskSpec{ 134 | { 135 | Name: "task1", 136 | Replicas: 1, 137 | Template: corev1.PodTemplateSpec{ 138 | Spec: corev1.PodSpec{ 139 | Containers: []corev1.Container{ 140 | { 141 | Name: "nginx", 142 | Image: "nginx:1.14.2", 143 | }, 144 | }, 145 | }, 146 | }, 147 | }, 148 | }, 149 | Policies: []v1alpha1.LifecyclePolicy{ 150 | { 151 | Event: busv1alpha1.PodEvictedEvent, 152 | Action: busv1alpha1.Action("someFakeAction"), 153 | }, 154 | }, 155 | }, 156 | } 157 | 158 | _, err := e2eutil.CreateJobTemplateInner(ctx, jobTemplate) 159 | Expect(err).To(MatchError(ContainSubstring(`invalid policy action`))) 160 | }) 161 | 162 | }) 163 | -------------------------------------------------------------------------------- /test/e2e/jobtemplate-admission/e2e_test.go: -------------------------------------------------------------------------------- 1 | package jobtemplate_admission 2 | 3 | import ( 4 | "testing" 5 | 6 | . "github.com/onsi/ginkgo/v2" 7 | . "github.com/onsi/gomega" 8 | 9 | e2eutil "jobflow/test/e2e/util" 10 | ) 11 | 12 | func TestE2E(t *testing.T) { 13 | RegisterFailHandler(Fail) 14 | RunSpecs(t, "Volcano JobTemplate Admission Test Suite") 15 | e2eutil.Cancel() 16 | } 17 | -------------------------------------------------------------------------------- /test/e2e/jobtemplate-admission/main_test.go: -------------------------------------------------------------------------------- 1 | package jobtemplate_admission 2 | 3 | import ( 4 | "os" 5 | "testing" 6 | 7 | e2eutil "jobflow/test/e2e/util" 8 | ) 9 | 10 | func TestMain(m *testing.M) { 11 | mgr := e2eutil.NewManager() 12 | e2eutil.JobFlowReconciler = e2eutil.NewJobFlowReconciler(mgr) 13 | e2eutil.JobTemplateReconciler = e2eutil.NewJobTemplateReconciler(mgr) 14 | e2eutil.StartMgr(mgr) 15 | e2eutil.InitKubeClient() 16 | os.Exit(m.Run()) 17 | } 18 | -------------------------------------------------------------------------------- /test/e2e/jobtemplate-controller/e2e_test.go: -------------------------------------------------------------------------------- 1 | package jobtemplate_controller 2 | 3 | import ( 4 | "testing" 5 | 6 | . "github.com/onsi/ginkgo/v2" 7 | . "github.com/onsi/gomega" 8 | "jobflow/test/e2e/util" 9 | ) 10 | 11 | func TestE2E(t *testing.T) { 12 | RegisterFailHandler(Fail) 13 | RunSpecs(t, "Volcano Job Seq Test Suite") 14 | util.Cancel() 15 | } 16 | -------------------------------------------------------------------------------- /test/e2e/jobtemplate-controller/jobtemplate.go: -------------------------------------------------------------------------------- 1 | package jobtemplate_controller 2 | 3 | import ( 4 | "time" 5 | 6 | . "github.com/onsi/ginkgo/v2" 7 | . "github.com/onsi/gomega" 8 | "jobflow/test/e2e/util" 9 | "k8s.io/apimachinery/pkg/util/wait" 10 | ) 11 | 12 | var _ = Describe("JobTemplate E2E Test", func() { 13 | It("will create success ", func() { 14 | context := util.InitTestContext(util.Options{}) 15 | defer util.CleanupTestContext(context) 16 | 17 | jobTemplate := util.GetJobTemplateInstance("jobtemplatetest") 18 | 19 | jobTemplateRes := util.CreateJobTemplate(context, jobTemplate) 20 | err := wait.Poll(100*time.Millisecond, util.OneMinute, util.JobTemplateExist(context, jobTemplateRes)) 21 | Expect(err).NotTo(HaveOccurred(), "failed to wait for JobTemplate created") 22 | }) 23 | It("will delete success", func() { 24 | context := util.InitTestContext(util.Options{}) 25 | defer util.CleanupTestContext(context) 26 | 27 | jobTemplate := util.GetJobTemplateInstance("jobtemplatetest") 28 | 29 | util.CreateJobTemplate(context, jobTemplate) 30 | 31 | jobTemplateRes := util.DeleteJobTemplate(context, jobTemplate) 32 | err := wait.Poll(100*time.Millisecond, util.OneMinute, util.JobTemplateNotExist(context, jobTemplateRes)) 33 | Expect(err).NotTo(HaveOccurred(), "failed to wait for JobTemplate deleted") 34 | }) 35 | It("will update status success", func() { 36 | context := util.InitTestContext(util.Options{}) 37 | defer util.CleanupTestContext(context) 38 | 39 | jobTemplate := util.GetJobTemplateInstance("jobtemplatetest") 40 | 41 | jobTemplateRes := util.CreateJobTemplate(context, jobTemplate) 42 | 43 | jobTemplateRes.Status.JobDependsOnList = []string{"jobflowtest"} 44 | 45 | jobTemplateUpdateRes := util.UpdateJobTemplateStatus(context, jobTemplateRes) 46 | err := wait.Poll(100*time.Millisecond, util.OneMinute, func() (done bool, err error) { 47 | if jobTemplateUpdateRes.Status.JobDependsOnList != nil && len(jobTemplateUpdateRes.Status.JobDependsOnList) > 0 { 48 | return true, nil 49 | } 50 | return false, nil 51 | }) 52 | Expect(err).NotTo(HaveOccurred(), "failed to wait for JobTemplateStatus updated") 53 | }) 54 | }) 55 | -------------------------------------------------------------------------------- /test/e2e/jobtemplate-controller/main_test.go: -------------------------------------------------------------------------------- 1 | package jobtemplate_controller 2 | 3 | import ( 4 | "os" 5 | "testing" 6 | 7 | "jobflow/test/e2e/util" 8 | ) 9 | 10 | func TestMain(m *testing.M) { 11 | mgr := util.NewManager() 12 | util.JobTemplateReconciler = util.NewJobTemplateReconciler(mgr) 13 | util.StartMgr(mgr) 14 | util.InitKubeClient() 15 | os.Exit(m.Run()) 16 | } 17 | -------------------------------------------------------------------------------- /test/e2e/util/util.go: -------------------------------------------------------------------------------- 1 | package util 2 | 3 | import ( 4 | "context" 5 | "math/rand" 6 | "os" 7 | "path/filepath" 8 | "time" 9 | 10 | . "github.com/onsi/ginkgo/v2" 11 | . "github.com/onsi/gomega" 12 | flowv1alpha1 "jobflow/api/v1alpha1" 13 | "jobflow/controllers" 14 | corev1 "k8s.io/api/core/v1" 15 | v1 "k8s.io/api/core/v1" 16 | "k8s.io/apimachinery/pkg/api/errors" 17 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 18 | "k8s.io/apimachinery/pkg/runtime" 19 | "k8s.io/apimachinery/pkg/types" 20 | utilruntime "k8s.io/apimachinery/pkg/util/runtime" 21 | "k8s.io/apimachinery/pkg/util/wait" 22 | "k8s.io/client-go/kubernetes" 23 | clientgoscheme "k8s.io/client-go/kubernetes/scheme" 24 | "k8s.io/client-go/tools/clientcmd" 25 | ctrl "sigs.k8s.io/controller-runtime" 26 | "volcano.sh/apis/pkg/apis/batch/v1alpha1" 27 | busv1alpha1 "volcano.sh/apis/pkg/apis/bus/v1alpha1" 28 | ) 29 | 30 | var ( 31 | FiveMinute = 5 * time.Minute 32 | OneMinute = 1 * time.Minute 33 | ) 34 | 35 | var KubeClient *kubernetes.Clientset 36 | 37 | var JobFlowReconciler *controllers.JobFlowReconciler 38 | 39 | var Cancel context.CancelFunc 40 | 41 | func StartMgr(mgr ctrl.Manager) { 42 | ctx, cancel := context.WithCancel(context.Background()) 43 | go func() { 44 | if err := mgr.Start(ctx); err != nil { 45 | os.Exit(1) 46 | } 47 | }() 48 | Cancel = cancel 49 | } 50 | 51 | func NewJobFlowReconciler(mgr ctrl.Manager) *controllers.JobFlowReconciler { 52 | jobFlowReconciler := &controllers.JobFlowReconciler{ 53 | Client: mgr.GetClient(), 54 | Scheme: mgr.GetScheme(), 55 | Recorder: mgr.GetEventRecorderFor("containerset-controller"), 56 | } 57 | return jobFlowReconciler 58 | } 59 | 60 | var JobTemplateReconciler *controllers.JobTemplateReconciler 61 | 62 | func NewJobTemplateReconciler(mgr ctrl.Manager) *controllers.JobTemplateReconciler { 63 | jobTemplateReconciler := &controllers.JobTemplateReconciler{ 64 | Client: mgr.GetClient(), 65 | Scheme: mgr.GetScheme(), 66 | Recorder: mgr.GetEventRecorderFor("containerset-controller"), 67 | } 68 | return jobTemplateReconciler 69 | } 70 | 71 | func InitKubeClient() { 72 | home := HomeDir() 73 | configPath := KubeconfigPath(home) 74 | config, _ := clientcmd.BuildConfigFromFlags(MasterURL(), configPath) 75 | KubeClient = kubernetes.NewForConfigOrDie(config) 76 | } 77 | 78 | func NewManager() ctrl.Manager { 79 | scheme := runtime.NewScheme() 80 | setupLog := ctrl.Log.WithName("setup") 81 | 82 | utilruntime.Must(clientgoscheme.AddToScheme(scheme)) 83 | 84 | utilruntime.Must(flowv1alpha1.AddToScheme(scheme)) 85 | //+kubebuilder:scaffold:scheme 86 | 87 | utilruntime.Must(v1alpha1.AddToScheme(scheme)) 88 | 89 | mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{ 90 | Scheme: scheme, 91 | MetricsBindAddress: ":8080", 92 | Port: 9443, 93 | HealthProbeBindAddress: ":8081", 94 | LeaderElection: false, 95 | LeaderElectionID: "1b1c5f74.volcano.sh", 96 | }) 97 | if err != nil { 98 | setupLog.Error(err, "unable to start manager") 99 | os.Exit(1) 100 | } 101 | return mgr 102 | } 103 | 104 | type Options struct { 105 | Namespace string 106 | } 107 | 108 | type TestContext struct { 109 | Namespace string 110 | JobFlowReconciler *controllers.JobFlowReconciler 111 | JobTemplateReconciler *controllers.JobTemplateReconciler 112 | KubeClient *kubernetes.Clientset 113 | } 114 | 115 | func InitTestContext(o Options) *TestContext { 116 | By("Initializing test context") 117 | 118 | if o.Namespace == "" { 119 | o.Namespace = GenRandomStr(8) 120 | } 121 | ctx := &TestContext{ 122 | Namespace: o.Namespace, 123 | JobFlowReconciler: JobFlowReconciler, 124 | JobTemplateReconciler: JobTemplateReconciler, 125 | KubeClient: KubeClient, 126 | } 127 | 128 | _, err := ctx.KubeClient.CoreV1().Namespaces().Create(context.TODO(), 129 | &v1.Namespace{ 130 | ObjectMeta: metav1.ObjectMeta{ 131 | Name: ctx.Namespace, 132 | }, 133 | }, 134 | metav1.CreateOptions{}, 135 | ) 136 | Expect(err).NotTo(HaveOccurred(), "failed to create namespace") 137 | 138 | return ctx 139 | } 140 | 141 | func CleanupTestContext(ctx *TestContext) { 142 | By("Cleaning up test context") 143 | 144 | foreground := metav1.DeletePropagationForeground 145 | err := ctx.KubeClient.CoreV1().Namespaces().Delete(context.TODO(), ctx.Namespace, metav1.DeleteOptions{ 146 | PropagationPolicy: &foreground, 147 | }) 148 | Expect(err).NotTo(HaveOccurred(), "failed to delete namespace") 149 | 150 | // Wait for namespace deleted. 151 | err = wait.Poll(100*time.Millisecond, FiveMinute, NamespaceNotExist(ctx)) 152 | Expect(err).NotTo(HaveOccurred(), "failed to wait for namespace deleted") 153 | } 154 | 155 | func NamespaceNotExist(ctx *TestContext) wait.ConditionFunc { 156 | return NamespaceNotExistWithName(ctx, ctx.Namespace) 157 | } 158 | 159 | func NamespaceNotExistWithName(ctx *TestContext, name string) wait.ConditionFunc { 160 | return func() (bool, error) { 161 | _, err := ctx.KubeClient.CoreV1().Namespaces().Get(context.TODO(), name, metav1.GetOptions{}) 162 | if err != nil && errors.IsNotFound(err) { 163 | return true, nil 164 | } 165 | return false, nil 166 | } 167 | } 168 | 169 | func GenRandomStr(l int) string { 170 | str := "0123456789abcdefghijklmnopqrstuvwxyz" 171 | bytes := []byte(str) 172 | var result []byte 173 | r := rand.New(rand.NewSource(time.Now().UnixNano())) 174 | for i := 0; i < l; i++ { 175 | result = append(result, bytes[r.Intn(len(bytes))]) 176 | } 177 | return string(result) 178 | } 179 | 180 | func HomeDir() string { 181 | if h := os.Getenv("HOME"); h != "" { 182 | return h 183 | } 184 | return os.Getenv("USERPROFILE") // windows 185 | } 186 | 187 | func MasterURL() string { 188 | if m := os.Getenv("MASTER"); m != "" { 189 | return m 190 | } 191 | return "" 192 | } 193 | 194 | func KubeconfigPath(home string) string { 195 | if m := os.Getenv("KUBECONFIG"); m != "" { 196 | return m 197 | } 198 | return filepath.Join(home, ".kube", "config") // default kubeconfig path is $HOME/.kube/config 199 | } 200 | 201 | func CreateJobTemplate(context *TestContext, jobTemplate *flowv1alpha1.JobTemplate) *flowv1alpha1.JobTemplate { 202 | jobTemplateRes, err := CreateJobTemplateInner(context, jobTemplate) 203 | Expect(err).NotTo(HaveOccurred(), "failed to create jobTemplate %s in namespace %s", jobTemplate.Name, jobTemplate.Namespace) 204 | return jobTemplateRes 205 | } 206 | 207 | func CreateJobTemplateInner(ctx *TestContext, jobTemplate *flowv1alpha1.JobTemplate) (*flowv1alpha1.JobTemplate, error) { 208 | jobTemplate.Namespace = ctx.Namespace 209 | err := ctx.JobTemplateReconciler.Create(context.TODO(), jobTemplate) 210 | return jobTemplate, err 211 | } 212 | 213 | func DeleteJobTemplate(context *TestContext, jobTemplate *flowv1alpha1.JobTemplate) *flowv1alpha1.JobTemplate { 214 | jobTemplateRes, err := DeleteJobTemplateInner(context, jobTemplate) 215 | Expect(err).NotTo(HaveOccurred(), "failed to delete jobTemplate %s in namespace %s", jobTemplate.Name, jobTemplate.Namespace) 216 | return jobTemplateRes 217 | } 218 | 219 | func DeleteJobTemplateInner(ctx *TestContext, jobTemplate *flowv1alpha1.JobTemplate) (*flowv1alpha1.JobTemplate, error) { 220 | jobTemplate.Namespace = ctx.Namespace 221 | err := ctx.JobTemplateReconciler.Delete(context.TODO(), jobTemplate) 222 | return jobTemplate, err 223 | } 224 | 225 | func UpdateJobTemplateStatus(context *TestContext, jobTemplate *flowv1alpha1.JobTemplate) *flowv1alpha1.JobTemplate { 226 | jobTemplateRes, err := UpdateJobTemplateStatusInner(context, jobTemplate) 227 | Expect(err).NotTo(HaveOccurred(), "failed to update jobTemplate %s in namespace %s", jobTemplate.Name, jobTemplate.Namespace) 228 | return jobTemplateRes 229 | } 230 | 231 | func UpdateJobTemplateStatusInner(ctx *TestContext, jobTemplate *flowv1alpha1.JobTemplate) (*flowv1alpha1.JobTemplate, error) { 232 | jobTemplate.Namespace = ctx.Namespace 233 | err := ctx.JobTemplateReconciler.Status().Update(context.TODO(), jobTemplate) 234 | return jobTemplate, err 235 | } 236 | 237 | func JobTemplateExist(ctx *TestContext, jobTemplate *flowv1alpha1.JobTemplate) wait.ConditionFunc { 238 | return func() (bool, error) { 239 | key := types.NamespacedName{ 240 | Namespace: jobTemplate.Namespace, 241 | Name: jobTemplate.Name, 242 | } 243 | jobTemplateGet := &flowv1alpha1.JobTemplate{} 244 | err := ctx.JobTemplateReconciler.Get(context.TODO(), key, jobTemplateGet) 245 | if err != nil { 246 | if errors.IsNotFound(err) { 247 | return false, nil 248 | } 249 | return false, err 250 | } 251 | return true, nil 252 | } 253 | } 254 | 255 | func JobTemplateNotExist(ctx *TestContext, jobTemplate *flowv1alpha1.JobTemplate) wait.ConditionFunc { 256 | return func() (bool, error) { 257 | key := types.NamespacedName{ 258 | Namespace: jobTemplate.Namespace, 259 | Name: jobTemplate.Name, 260 | } 261 | jobTemplateGet := &flowv1alpha1.JobTemplate{} 262 | err := ctx.JobTemplateReconciler.Get(context.TODO(), key, jobTemplateGet) 263 | if err != nil { 264 | if errors.IsNotFound(err) { 265 | return true, nil 266 | } 267 | return false, err 268 | } 269 | return false, nil 270 | } 271 | } 272 | 273 | func GetJobTemplateInstance(jobTemplateName string) *flowv1alpha1.JobTemplate { 274 | jobTemplate := &flowv1alpha1.JobTemplate{ 275 | ObjectMeta: metav1.ObjectMeta{ 276 | Name: jobTemplateName, 277 | }, 278 | Spec: v1alpha1.JobSpec{ 279 | SchedulerName: "volcano", 280 | MinAvailable: 1, 281 | Volumes: nil, 282 | Tasks: []v1alpha1.TaskSpec{ 283 | { 284 | Name: "tasktest", 285 | Replicas: 1, 286 | MinAvailable: nil, 287 | Template: corev1.PodTemplateSpec{ 288 | ObjectMeta: metav1.ObjectMeta{}, 289 | Spec: corev1.PodSpec{ 290 | Volumes: nil, 291 | Containers: []corev1.Container{ 292 | { 293 | Name: "nginx", 294 | Image: "nginx:1.14.2", 295 | Command: []string{ 296 | "sh", 297 | "-c", 298 | "sleep 10s", 299 | }, 300 | }, 301 | }, 302 | RestartPolicy: corev1.RestartPolicyNever, 303 | }, 304 | }, 305 | Policies: []v1alpha1.LifecyclePolicy{ 306 | { 307 | Event: busv1alpha1.TaskCompletedEvent, 308 | Action: busv1alpha1.CompleteJobAction, 309 | }, 310 | }, 311 | TopologyPolicy: "", 312 | MaxRetry: 0, 313 | }, 314 | }, 315 | Policies: []v1alpha1.LifecyclePolicy{ 316 | { 317 | Event: busv1alpha1.PodEvictedEvent, 318 | Action: busv1alpha1.RestartJobAction, 319 | }, 320 | }, 321 | Plugins: nil, 322 | RunningEstimate: nil, 323 | Queue: "default", 324 | MaxRetry: 0, 325 | TTLSecondsAfterFinished: nil, 326 | PriorityClassName: "", 327 | MinSuccess: nil, 328 | }, 329 | Status: flowv1alpha1.JobTemplateStatus{}, 330 | } 331 | return jobTemplate 332 | } 333 | 334 | func CreateJobFlow(context *TestContext, jobFlow *flowv1alpha1.JobFlow) *flowv1alpha1.JobFlow { 335 | jobFlowRes, err := CreateJobFlowInner(context, jobFlow) 336 | Expect(err).NotTo(HaveOccurred(), "failed to create jobFlow %s in namespace %s", jobFlow.Name, jobFlow.Namespace) 337 | return jobFlowRes 338 | } 339 | 340 | func CreateJobFlowInner(ctx *TestContext, jobFlow *flowv1alpha1.JobFlow) (*flowv1alpha1.JobFlow, error) { 341 | jobFlow.Namespace = ctx.Namespace 342 | err := ctx.JobFlowReconciler.Create(context.TODO(), jobFlow) 343 | return jobFlow, err 344 | } 345 | 346 | func DeleteJobFlow(context *TestContext, jobFlow *flowv1alpha1.JobFlow) *flowv1alpha1.JobFlow { 347 | jobFlowRes, err := DeleteJobFlowInner(context, jobFlow) 348 | Expect(err).NotTo(HaveOccurred(), "failed to create jobFlow %s in namespace %s", jobFlow.Name, jobFlow.Namespace) 349 | return jobFlowRes 350 | } 351 | 352 | func DeleteJobFlowInner(ctx *TestContext, jobFlow *flowv1alpha1.JobFlow) (*flowv1alpha1.JobFlow, error) { 353 | jobFlow.Namespace = ctx.Namespace 354 | err := ctx.JobFlowReconciler.Delete(context.TODO(), jobFlow) 355 | return jobFlow, err 356 | } 357 | 358 | func UpdateJobFlowStatus(ctx *TestContext, jobFlow *flowv1alpha1.JobFlow) *flowv1alpha1.JobFlow { 359 | jobTemplateRes, err := UpdateJobFlowStatusInner(ctx, jobFlow) 360 | Expect(err).NotTo(HaveOccurred(), "failed to update jobFlow %s in namespace %s", jobFlow.Name, jobFlow.Namespace) 361 | return jobTemplateRes 362 | } 363 | 364 | func UpdateJobFlowStatusInner(ctx *TestContext, jobFlow *flowv1alpha1.JobFlow) (*flowv1alpha1.JobFlow, error) { 365 | jobFlow.Namespace = ctx.Namespace 366 | err := ctx.JobFlowReconciler.Status().Update(context.TODO(), jobFlow) 367 | return jobFlow, err 368 | } 369 | 370 | func JobFlowExist(ctx *TestContext, jobFlow *flowv1alpha1.JobFlow) wait.ConditionFunc { 371 | return func() (bool, error) { 372 | key := types.NamespacedName{ 373 | Namespace: jobFlow.Namespace, 374 | Name: jobFlow.Name, 375 | } 376 | jobFlowGet := &flowv1alpha1.JobFlow{} 377 | err := ctx.JobFlowReconciler.Get(context.TODO(), key, jobFlowGet) 378 | if err != nil { 379 | if errors.IsNotFound(err) { 380 | return false, nil 381 | } 382 | return false, err 383 | } 384 | return true, nil 385 | } 386 | } 387 | 388 | func JobFlowNotExist(ctx *TestContext, jobFlow *flowv1alpha1.JobFlow) wait.ConditionFunc { 389 | return func() (bool, error) { 390 | key := types.NamespacedName{ 391 | Namespace: jobFlow.Namespace, 392 | Name: jobFlow.Name, 393 | } 394 | jobTemplateGet := &flowv1alpha1.JobTemplate{} 395 | err := ctx.JobFlowReconciler.Get(context.TODO(), key, jobTemplateGet) 396 | if err != nil { 397 | if errors.IsNotFound(err) { 398 | return true, nil 399 | } 400 | return false, err 401 | } 402 | return false, nil 403 | } 404 | } 405 | 406 | func GetFlowInstance(jobFlowName string) *flowv1alpha1.JobFlow { 407 | jobflow := &flowv1alpha1.JobFlow{ 408 | TypeMeta: metav1.TypeMeta{}, 409 | ObjectMeta: metav1.ObjectMeta{ 410 | Name: jobFlowName, 411 | }, 412 | Spec: flowv1alpha1.JobFlowSpec{ 413 | Flows: []flowv1alpha1.Flow{ 414 | { 415 | Name: "jobtemplate-a", 416 | DependsOn: nil, 417 | }, 418 | { 419 | Name: "jobtemplate-b", 420 | DependsOn: &flowv1alpha1.DependsOn{ 421 | Targets: []string{"jobtemplate-a"}, 422 | Probe: nil, 423 | }, 424 | }, 425 | }, 426 | JobRetainPolicy: flowv1alpha1.Retain, 427 | }, 428 | Status: flowv1alpha1.JobFlowStatus{}, 429 | } 430 | return jobflow 431 | } 432 | 433 | func GetFlowInstanceRetainPolicyDelete(jobFlowName string) *flowv1alpha1.JobFlow { 434 | jobflow := &flowv1alpha1.JobFlow{ 435 | TypeMeta: metav1.TypeMeta{}, 436 | ObjectMeta: metav1.ObjectMeta{ 437 | Name: jobFlowName, 438 | }, 439 | Spec: flowv1alpha1.JobFlowSpec{ 440 | Flows: []flowv1alpha1.Flow{ 441 | { 442 | Name: "jobtemplate-a", 443 | DependsOn: nil, 444 | }, 445 | { 446 | Name: "jobtemplate-b", 447 | DependsOn: &flowv1alpha1.DependsOn{ 448 | Targets: []string{"jobtemplate-a"}, 449 | Probe: nil, 450 | }, 451 | }, 452 | }, 453 | JobRetainPolicy: flowv1alpha1.Delete, 454 | }, 455 | Status: flowv1alpha1.JobFlowStatus{}, 456 | } 457 | return jobflow 458 | } 459 | 460 | func VcJobExist(ctx *TestContext, jobFlow *flowv1alpha1.JobFlow, jobTemplate *flowv1alpha1.JobTemplate) wait.ConditionFunc { 461 | return func() (bool, error) { 462 | key := types.NamespacedName{ 463 | Namespace: jobTemplate.Namespace, 464 | Name: GetJobName(jobFlow.Name, jobTemplate.Name), 465 | } 466 | jobGet := &v1alpha1.Job{} 467 | err := ctx.JobFlowReconciler.Get(context.TODO(), key, jobGet) 468 | if err != nil { 469 | if errors.IsNotFound(err) { 470 | return false, nil 471 | } 472 | return false, err 473 | } 474 | return true, nil 475 | } 476 | } 477 | 478 | func VcJobNotExist(ctx *TestContext, jobFlow *flowv1alpha1.JobFlow, jobTemplate *flowv1alpha1.JobTemplate) wait.ConditionFunc { 479 | return func() (bool, error) { 480 | key := types.NamespacedName{ 481 | Namespace: jobTemplate.Namespace, 482 | Name: GetJobName(jobFlow.Name, jobTemplate.Name), 483 | } 484 | jobGet := &v1alpha1.Job{} 485 | err := ctx.JobFlowReconciler.Get(context.TODO(), key, jobGet) 486 | if err != nil { 487 | if errors.IsNotFound(err) { 488 | return true, nil 489 | } 490 | return false, err 491 | } 492 | return false, nil 493 | } 494 | } 495 | 496 | func GetJobName(jobFlowName string, jobTemplateName string) string { 497 | return jobFlowName + "-" + jobTemplateName 498 | } 499 | -------------------------------------------------------------------------------- /utils/utils.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | const CreateByJobTemplate = "volcano.sh/createByJobTemplate" 4 | 5 | func GetConnectionOfJobAndJobTemplate(namespace, name string) string { 6 | return namespace + "." + name 7 | } 8 | -------------------------------------------------------------------------------- /utils/utils_test.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | import "testing" 4 | 5 | func TestGetConnectionOfJobAndJobTemplate(t *testing.T) { 6 | type args struct { 7 | namespace string 8 | name string 9 | } 10 | tests := []struct { 11 | name string 12 | args args 13 | want string 14 | }{ 15 | { 16 | name: "TestGetConnectionOfJobAndJobTemplate", 17 | args: args{ 18 | namespace: "default", 19 | name: "flow", 20 | }, 21 | want: "default.flow", 22 | }, 23 | } 24 | for _, tt := range tests { 25 | t.Run(tt.name, func(t *testing.T) { 26 | if got := GetConnectionOfJobAndJobTemplate(tt.args.namespace, tt.args.name); got != tt.want { 27 | t.Errorf("GetConnectionOfJobAndJobTemplate() = %v, want %v", got, tt.want) 28 | } 29 | }) 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /utils/validate_dag.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | import ( 4 | "fmt" 5 | "github.com/eapache/queue" 6 | ) 7 | 8 | type Vertex struct { 9 | Key string 10 | Parents []*Vertex 11 | Children []*Vertex 12 | Value interface{} 13 | } 14 | 15 | type DAG struct { 16 | Vertexes []*Vertex 17 | } 18 | 19 | func (dag *DAG) AddVertex(v *Vertex) { 20 | dag.Vertexes = append(dag.Vertexes, v) 21 | } 22 | 23 | func (dag *DAG) AddEdge(from, to *Vertex) { 24 | from.Children = append(from.Children, to) 25 | 26 | to.Parents = append(from.Parents, from) 27 | } 28 | 29 | func (dag *DAG) BFS(root *Vertex) error { 30 | q := queue.New() 31 | 32 | visitMap := make(map[string]bool) 33 | visitMap[root.Key] = true 34 | 35 | q.Add(root) 36 | 37 | for { 38 | if q.Length() == 0 { 39 | break 40 | } 41 | current := q.Remove().(*Vertex) 42 | 43 | for _, v := range current.Children { 44 | if v.Key == root.Key { 45 | return fmt.Errorf("find bad dependency, please check the dependencies of your templates") 46 | } 47 | if _, ok := visitMap[v.Key]; !ok { 48 | visitMap[v.Key] = true 49 | q.Add(v) 50 | } 51 | } 52 | } 53 | 54 | return nil 55 | } 56 | -------------------------------------------------------------------------------- /utils/validate_util.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2018 The Volcano Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package utils 18 | 19 | import ( 20 | "fmt" 21 | 22 | "github.com/hashicorp/go-multierror" 23 | 24 | "k8s.io/apimachinery/pkg/util/validation/field" 25 | "k8s.io/kubernetes/pkg/apis/core/validation" 26 | 27 | batchv1alpha1 "volcano.sh/apis/pkg/apis/batch/v1alpha1" 28 | busv1alpha1 "volcano.sh/apis/pkg/apis/bus/v1alpha1" 29 | ) 30 | 31 | // policyEventMap defines all policy events and whether to allow external use. 32 | var policyEventMap = map[busv1alpha1.Event]bool{ 33 | busv1alpha1.AnyEvent: true, 34 | busv1alpha1.PodFailedEvent: true, 35 | busv1alpha1.PodEvictedEvent: true, 36 | busv1alpha1.JobUnknownEvent: true, 37 | busv1alpha1.TaskCompletedEvent: true, 38 | busv1alpha1.TaskFailedEvent: true, 39 | busv1alpha1.OutOfSyncEvent: false, 40 | busv1alpha1.CommandIssuedEvent: false, 41 | busv1alpha1.JobUpdatedEvent: true, 42 | } 43 | 44 | // policyActionMap defines all policy actions and whether to allow external use. 45 | var policyActionMap = map[busv1alpha1.Action]bool{ 46 | busv1alpha1.AbortJobAction: true, 47 | busv1alpha1.RestartJobAction: true, 48 | busv1alpha1.RestartTaskAction: true, 49 | busv1alpha1.TerminateJobAction: true, 50 | busv1alpha1.CompleteJobAction: true, 51 | busv1alpha1.ResumeJobAction: true, 52 | busv1alpha1.SyncJobAction: false, 53 | busv1alpha1.EnqueueAction: false, 54 | busv1alpha1.SyncQueueAction: false, 55 | busv1alpha1.OpenQueueAction: false, 56 | busv1alpha1.CloseQueueAction: false, 57 | } 58 | 59 | func ValidatePolicies(policies []batchv1alpha1.LifecyclePolicy, fldPath *field.Path) error { 60 | var err error 61 | policyEvents := map[busv1alpha1.Event]struct{}{} 62 | exitCodes := map[int32]struct{}{} 63 | 64 | for _, policy := range policies { 65 | if (policy.Event != "" || len(policy.Events) != 0) && policy.ExitCode != nil { 66 | err = multierror.Append(err, fmt.Errorf("must not specify event and exitCode simultaneously")) 67 | break 68 | } 69 | 70 | if policy.Event == "" && len(policy.Events) == 0 && policy.ExitCode == nil { 71 | err = multierror.Append(err, fmt.Errorf("either event and exitCode should be specified")) 72 | break 73 | } 74 | 75 | if len(policy.Event) != 0 || len(policy.Events) != 0 { 76 | bFlag := false 77 | policyEventsList := getEventList(policy) 78 | for _, event := range policyEventsList { 79 | if allow, ok := policyEventMap[event]; !ok || !allow { 80 | err = multierror.Append(err, field.Invalid(fldPath, event, "invalid policy event")) 81 | bFlag = true 82 | break 83 | } 84 | 85 | if allow, ok := policyActionMap[policy.Action]; !ok || !allow { 86 | err = multierror.Append(err, field.Invalid(fldPath, policy.Action, "invalid policy action")) 87 | bFlag = true 88 | break 89 | } 90 | if _, found := policyEvents[event]; found { 91 | err = multierror.Append(err, fmt.Errorf("duplicate event %v across different policy", event)) 92 | bFlag = true 93 | break 94 | } else { 95 | policyEvents[event] = struct{}{} 96 | } 97 | } 98 | if bFlag { 99 | break 100 | } 101 | } else { 102 | if *policy.ExitCode == 0 { 103 | err = multierror.Append(err, fmt.Errorf("0 is not a valid error code")) 104 | break 105 | } 106 | if _, found := exitCodes[*policy.ExitCode]; found { 107 | err = multierror.Append(err, fmt.Errorf("duplicate exitCode %v", *policy.ExitCode)) 108 | break 109 | } else { 110 | exitCodes[*policy.ExitCode] = struct{}{} 111 | } 112 | } 113 | } 114 | 115 | if _, found := policyEvents[busv1alpha1.AnyEvent]; found && len(policyEvents) > 1 { 116 | err = multierror.Append(err, fmt.Errorf("if there's * here, no other policy should be here")) 117 | } 118 | 119 | return err 120 | } 121 | 122 | func getEventList(policy batchv1alpha1.LifecyclePolicy) []busv1alpha1.Event { 123 | policyEventsList := policy.Events 124 | if len(policy.Event) > 0 { 125 | policyEventsList = append(policyEventsList, policy.Event) 126 | } 127 | uniquePolicyEventlist := removeDuplicates(policyEventsList) 128 | return uniquePolicyEventlist 129 | } 130 | 131 | func removeDuplicates(eventList []busv1alpha1.Event) []busv1alpha1.Event { 132 | keys := make(map[busv1alpha1.Event]bool) 133 | list := []busv1alpha1.Event{} 134 | for _, val := range eventList { 135 | if _, value := keys[val]; !value { 136 | keys[val] = true 137 | list = append(list, val) 138 | } 139 | } 140 | return list 141 | } 142 | 143 | func GetValidEvents() []busv1alpha1.Event { 144 | var events []busv1alpha1.Event 145 | for e, allow := range policyEventMap { 146 | if allow { 147 | events = append(events, e) 148 | } 149 | } 150 | 151 | return events 152 | } 153 | 154 | func GetValidActions() []busv1alpha1.Action { 155 | var actions []busv1alpha1.Action 156 | for a, allow := range policyActionMap { 157 | if allow { 158 | actions = append(actions, a) 159 | } 160 | } 161 | 162 | return actions 163 | } 164 | 165 | // validateIO validates IO configuration. 166 | func ValidateIO(volumes []batchv1alpha1.VolumeSpec) error { 167 | volumeMap := map[string]bool{} 168 | for _, volume := range volumes { 169 | if len(volume.MountPath) == 0 { 170 | return fmt.Errorf(" mountPath is required;") 171 | } 172 | if _, found := volumeMap[volume.MountPath]; found { 173 | return fmt.Errorf(" duplicated mountPath: %s;", volume.MountPath) 174 | } 175 | if volume.VolumeClaim == nil && volume.VolumeClaimName == "" { 176 | return fmt.Errorf(" either VolumeClaim or VolumeClaimName must be specified;") 177 | } 178 | if len(volume.VolumeClaimName) != 0 { 179 | if volume.VolumeClaim != nil { 180 | return fmt.Errorf("conflict: If you want to use an existing PVC, just specify VolumeClaimName." + 181 | "If you want to create a new PVC, you do not need to specify VolumeClaimName") 182 | } 183 | if errMsgs := validation.ValidatePersistentVolumeName(volume.VolumeClaimName, false); len(errMsgs) > 0 { 184 | return fmt.Errorf("invalid VolumeClaimName %s : %v", volume.VolumeClaimName, errMsgs) 185 | } 186 | } 187 | 188 | volumeMap[volume.MountPath] = true 189 | } 190 | return nil 191 | } 192 | -------------------------------------------------------------------------------- /webhooks/admission/jobflow/validate/validate_jobflow.go: -------------------------------------------------------------------------------- 1 | package validate 2 | 3 | import ( 4 | "fmt" 5 | 6 | "k8s.io/api/admission/v1beta1" 7 | whv1 "k8s.io/api/admissionregistration/v1" 8 | "k8s.io/klog" 9 | 10 | jobflowv1alpha1 "jobflow/api/v1alpha1" 11 | "jobflow/utils" 12 | "jobflow/webhooks/router" 13 | "jobflow/webhooks/schema" 14 | ) 15 | 16 | func init() { 17 | _ = router.RegisterAdmission(service) 18 | } 19 | 20 | var sideEffectsNone = whv1.SideEffectClassNone 21 | var service = &router.AdmissionService{ 22 | Path: "/jobflows/validate", 23 | Func: AdmitJobFlows, 24 | 25 | ValidatingConfig: &whv1.ValidatingWebhookConfiguration{ 26 | Webhooks: []whv1.ValidatingWebhook{{ 27 | Name: "validatejobflow.volcano.sh", 28 | Rules: []whv1.RuleWithOperations{ 29 | { 30 | Operations: []whv1.OperationType{whv1.Create}, 31 | Rule: whv1.Rule{ 32 | APIGroups: []string{"flow.volcano.sh"}, 33 | APIVersions: []string{"v1alpha1"}, 34 | Resources: []string{"jobflows"}, 35 | }, 36 | }, 37 | }, 38 | SideEffects: &sideEffectsNone, 39 | AdmissionReviewVersions: []string{"v1beta1"}, 40 | }}, 41 | }, 42 | } 43 | 44 | // AdmitJobFlows is to admit jobFlows and return response. 45 | func AdmitJobFlows(ar v1beta1.AdmissionReview) error { 46 | klog.V(3).Infof("admitting jobflows -- %s", ar.Request.Operation) 47 | 48 | jobFlow, err := schema.DecodeJobFlow(ar.Request.Object, ar.Request.Resource) 49 | if err != nil { 50 | return err 51 | } 52 | 53 | switch ar.Request.Operation { 54 | case v1beta1.Create: 55 | err = validateJobFlowCreate(jobFlow) 56 | default: 57 | err = fmt.Errorf("only support 'CREATE' operation") 58 | } 59 | 60 | return err 61 | } 62 | 63 | func validateJobFlowCreate(jobFlow *jobflowv1alpha1.JobFlow) error { 64 | flows := jobFlow.Spec.Flows 65 | var msg string 66 | templateNames := map[string][]string{} 67 | vertexMap := make(map[string]*utils.Vertex) 68 | dag := &utils.DAG{} 69 | var duplicatedTemplate = false 70 | for _, template := range flows { 71 | if _, found := templateNames[template.Name]; found { 72 | // duplicate task name 73 | msg += fmt.Sprintf(" duplicated template name %s;", template.Name) 74 | duplicatedTemplate = true 75 | break 76 | } else { 77 | if template.DependsOn == nil || template.DependsOn.Targets == nil { 78 | template.DependsOn = new(jobflowv1alpha1.DependsOn) 79 | } 80 | templateNames[template.Name] = template.DependsOn.Targets 81 | vertexMap[template.Name] = &utils.Vertex{Key: template.Name} 82 | } 83 | } 84 | // Skip closed-loop detection if there are duplicate templates 85 | if !duplicatedTemplate { 86 | // Build dag through dependencies 87 | for current, parents := range templateNames { 88 | if vertexMap == nil { 89 | break 90 | } 91 | if parents != nil && len(parents) > 0 { 92 | for _, parent := range parents { 93 | if _, found := vertexMap[parent]; !found { 94 | msg += fmt.Sprintf("cannot find the template: %s ", parent) 95 | vertexMap = nil 96 | break 97 | } 98 | dag.AddEdge(vertexMap[parent], vertexMap[current]) 99 | } 100 | } 101 | } 102 | // Check if there is a closed loop 103 | for k := range vertexMap { 104 | if err := dag.BFS(vertexMap[k]); err != nil { 105 | msg += fmt.Sprintf("%v;", err) 106 | break 107 | } 108 | } 109 | } 110 | 111 | if msg != "" { 112 | return fmt.Errorf("failed to create jobFlow for: %s", msg) 113 | } 114 | 115 | return nil 116 | } 117 | -------------------------------------------------------------------------------- /webhooks/admission/jobflow/validate/validate_jobflow_test.go: -------------------------------------------------------------------------------- 1 | package validate 2 | 3 | import ( 4 | "jobflow/webhooks/util" 5 | "strings" 6 | "testing" 7 | 8 | "k8s.io/api/admission/v1beta1" 9 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 10 | 11 | jobflowv1alpha1 "jobflow/api/v1alpha1" 12 | ) 13 | 14 | func TestValidateJobCreate(t *testing.T) { 15 | namespace := "test" 16 | 17 | testCases := []struct { 18 | Name string 19 | JobFlow jobflowv1alpha1.JobFlow 20 | ExpectErr bool 21 | reviewResponse v1beta1.AdmissionResponse 22 | ret string 23 | }{ 24 | { 25 | Name: "validate valid-jobFlow", 26 | JobFlow: jobflowv1alpha1.JobFlow{ 27 | ObjectMeta: metav1.ObjectMeta{ 28 | Name: "valid-jobFlow", 29 | Namespace: namespace, 30 | }, 31 | Spec: jobflowv1alpha1.JobFlowSpec{ 32 | Flows: []jobflowv1alpha1.Flow{ 33 | { 34 | Name: "job-a", 35 | DependsOn: &jobflowv1alpha1.DependsOn{ 36 | Targets: []string{}, 37 | }, 38 | }, 39 | { 40 | Name: "job-b", 41 | DependsOn: &jobflowv1alpha1.DependsOn{ 42 | Targets: []string{"job-a"}, 43 | }, 44 | }, 45 | { 46 | Name: "job-c", 47 | DependsOn: &jobflowv1alpha1.DependsOn{ 48 | Targets: []string{"job-b"}, 49 | }, 50 | }, 51 | }, 52 | JobRetainPolicy: "succeed", 53 | }, 54 | }, 55 | reviewResponse: v1beta1.AdmissionResponse{Allowed: true}, 56 | ret: "", 57 | ExpectErr: false, 58 | }, 59 | // duplicate jobTemplate name 60 | { 61 | Name: "duplicate-job-jobFlow", 62 | JobFlow: jobflowv1alpha1.JobFlow{ 63 | ObjectMeta: metav1.ObjectMeta{ 64 | Name: "duplicate-job-jobFlow", 65 | Namespace: namespace, 66 | }, 67 | Spec: jobflowv1alpha1.JobFlowSpec{ 68 | Flows: []jobflowv1alpha1.Flow{ 69 | { 70 | Name: "job-a", 71 | DependsOn: &jobflowv1alpha1.DependsOn{ 72 | Targets: []string{}, 73 | }, 74 | }, 75 | { 76 | Name: "job-b", 77 | DependsOn: &jobflowv1alpha1.DependsOn{ 78 | Targets: []string{}, 79 | }, 80 | }, 81 | { 82 | Name: "job-b", 83 | DependsOn: &jobflowv1alpha1.DependsOn{ 84 | Targets: []string{}, 85 | }, 86 | }, 87 | }, 88 | JobRetainPolicy: "succeed", 89 | }, 90 | }, 91 | reviewResponse: v1beta1.AdmissionResponse{Allowed: true}, 92 | ret: "duplicated template name job-b", 93 | ExpectErr: true, 94 | }, 95 | // dependsOn illegal 96 | { 97 | Name: "dependsOn-illegal-jobFlow", 98 | JobFlow: jobflowv1alpha1.JobFlow{ 99 | ObjectMeta: metav1.ObjectMeta{ 100 | Name: "dependsOn-illegal-jobFlow", 101 | Namespace: namespace, 102 | }, 103 | Spec: jobflowv1alpha1.JobFlowSpec{ 104 | Flows: []jobflowv1alpha1.Flow{ 105 | { 106 | Name: "job-a", 107 | DependsOn: &jobflowv1alpha1.DependsOn{ 108 | Targets: []string{}, 109 | }, 110 | }, 111 | { 112 | Name: "job-b", 113 | DependsOn: &jobflowv1alpha1.DependsOn{ 114 | Targets: []string{"job-m"}, 115 | }, 116 | }, 117 | }, 118 | JobRetainPolicy: "succeed", 119 | }, 120 | }, 121 | reviewResponse: v1beta1.AdmissionResponse{Allowed: true}, 122 | ret: "cannot find the template: job-m", 123 | ExpectErr: true, 124 | }, 125 | // bad-dag-jobFlow 126 | { 127 | Name: "bad-dag-jobFlow", 128 | JobFlow: jobflowv1alpha1.JobFlow{ 129 | ObjectMeta: metav1.ObjectMeta{ 130 | Name: "bad-dag-jobFlow", 131 | Namespace: namespace, 132 | }, 133 | Spec: jobflowv1alpha1.JobFlowSpec{ 134 | Flows: []jobflowv1alpha1.Flow{ 135 | { 136 | Name: "job-a", 137 | DependsOn: &jobflowv1alpha1.DependsOn{ 138 | Targets: []string{"job-c", "job-b"}, 139 | }, 140 | }, 141 | { 142 | Name: "job-b", 143 | DependsOn: &jobflowv1alpha1.DependsOn{ 144 | Targets: []string{"job-a", "job-c"}, 145 | }, 146 | }, 147 | { 148 | Name: "job-c", 149 | DependsOn: &jobflowv1alpha1.DependsOn{ 150 | Targets: []string{"job-b", "job-a"}, 151 | }, 152 | }, 153 | }, 154 | JobRetainPolicy: "succeed", 155 | }, 156 | }, 157 | reviewResponse: v1beta1.AdmissionResponse{Allowed: true}, 158 | ret: "find bad dependency, please check the dependencies of your templates", 159 | ExpectErr: true, 160 | }, 161 | } 162 | 163 | for _, testCase := range testCases { 164 | t.Run(testCase.Name, func(t *testing.T) { 165 | ret := validateJobFlowCreate(&testCase.JobFlow) 166 | testCase.reviewResponse = *util.ToAdmissionResponse(ret) 167 | //fmt.Printf("test-case name:%s, ret:%v testCase.reviewResponse:%v \n", testCase.Name, ret,testCase.reviewResponse) 168 | if testCase.ExpectErr == true && ret == nil { 169 | t.Errorf("Expect error msg :%s, but got nil.", testCase.ret) 170 | } 171 | if testCase.ExpectErr == true && testCase.reviewResponse.Allowed != false { 172 | t.Errorf("Expect Allowed as false but got true.") 173 | } 174 | if testCase.ExpectErr == true && ret != nil && !strings.Contains(ret.Error(), testCase.ret) { 175 | t.Errorf("Expect error msg :%s, but got diff error %v", testCase.ret, ret) 176 | } 177 | 178 | if testCase.ExpectErr == false && ret != nil { 179 | t.Errorf("Expect no error, but got error %v", ret) 180 | } 181 | if testCase.ExpectErr == false && testCase.reviewResponse.Allowed != true { 182 | t.Errorf("Expect Allowed as true but got false. %v", testCase.reviewResponse) 183 | } 184 | }) 185 | } 186 | } 187 | -------------------------------------------------------------------------------- /webhooks/admission/template/validate/validate_template.go: -------------------------------------------------------------------------------- 1 | package validate 2 | 3 | import ( 4 | "fmt" 5 | 6 | "k8s.io/api/admission/v1beta1" 7 | whv1 "k8s.io/api/admissionregistration/v1" 8 | v1 "k8s.io/api/core/v1" 9 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 10 | "k8s.io/apimachinery/pkg/util/validation" 11 | "k8s.io/apimachinery/pkg/util/validation/field" 12 | "k8s.io/klog" 13 | k8score "k8s.io/kubernetes/pkg/apis/core" 14 | k8scorev1 "k8s.io/kubernetes/pkg/apis/core/v1" 15 | v1qos "k8s.io/kubernetes/pkg/apis/core/v1/helper/qos" 16 | k8scorevalid "k8s.io/kubernetes/pkg/apis/core/validation" 17 | "volcano.sh/apis/pkg/apis/batch/v1alpha1" 18 | 19 | jobflowv1alpha1 "jobflow/api/v1alpha1" 20 | "jobflow/utils" 21 | "jobflow/webhooks/router" 22 | "jobflow/webhooks/schema" 23 | ) 24 | 25 | func init() { 26 | _ = router.RegisterAdmission(service) 27 | } 28 | 29 | var sideEffectsNone = whv1.SideEffectClassNone 30 | var service = &router.AdmissionService{ 31 | Path: "/jobtemplates/validate", 32 | Func: AdmitJobTemplates, 33 | 34 | ValidatingConfig: &whv1.ValidatingWebhookConfiguration{ 35 | Webhooks: []whv1.ValidatingWebhook{{ 36 | Name: "validatetemplate.volcano.sh", 37 | Rules: []whv1.RuleWithOperations{ 38 | { 39 | Operations: []whv1.OperationType{whv1.Create}, 40 | Rule: whv1.Rule{ 41 | APIGroups: []string{"flow.volcano.sh"}, 42 | APIVersions: []string{"v1alpha1"}, 43 | Resources: []string{"jobtemplates"}, 44 | }, 45 | }, 46 | }, 47 | SideEffects: &sideEffectsNone, 48 | AdmissionReviewVersions: []string{"v1beta1"}, 49 | }}, 50 | }, 51 | } 52 | 53 | // AdmitJobFlows is to admit jobFlows and return response. 54 | func AdmitJobTemplates(ar v1beta1.AdmissionReview) error { 55 | klog.V(3).Infof("admitting jobtemplates -- %s", ar.Request.Operation) 56 | 57 | jobTemplate, err := schema.DecodeJobTemplate(ar.Request.Object, ar.Request.Resource) 58 | if err != nil { 59 | return err 60 | } 61 | 62 | switch ar.Request.Operation { 63 | case v1beta1.Create: 64 | err = validateJobTemplateCreate(jobTemplate) 65 | default: 66 | err = fmt.Errorf("only support 'CREATE' operation") 67 | } 68 | 69 | return err 70 | } 71 | 72 | func validateJobTemplateCreate(job *jobflowv1alpha1.JobTemplate) error { 73 | klog.V(3).Infof("validate create %s", job.Name) 74 | var msg string 75 | taskNames := map[string]string{} 76 | var totalReplicas int32 77 | 78 | if job.Spec.MinAvailable < 0 { 79 | return fmt.Errorf("job 'minAvailable' must be >= 0") 80 | } 81 | 82 | if job.Spec.MaxRetry < 0 { 83 | return fmt.Errorf("'maxRetry' cannot be less than zero") 84 | } 85 | 86 | if job.Spec.TTLSecondsAfterFinished != nil && *job.Spec.TTLSecondsAfterFinished < 0 { 87 | return fmt.Errorf("'ttlSecondsAfterFinished' cannot be less than zero") 88 | } 89 | 90 | if len(job.Spec.Tasks) == 0 { 91 | return fmt.Errorf("no task specified in job spec") 92 | } 93 | 94 | for index, task := range job.Spec.Tasks { 95 | if task.Replicas < 0 { 96 | msg += fmt.Sprintf(" 'replicas' < 0 in task: %s;", task.Name) 97 | } 98 | 99 | if task.MinAvailable != nil && *task.MinAvailable > task.Replicas { 100 | msg += fmt.Sprintf(" 'minAvailable' is greater than 'replicas' in task: %s, job: %s", task.Name, job.Name) 101 | } 102 | 103 | // count replicas 104 | totalReplicas += task.Replicas 105 | 106 | // validate task name 107 | if errMsgs := validation.IsDNS1123Label(task.Name); len(errMsgs) > 0 { 108 | msg += fmt.Sprintf(" %v;", errMsgs) 109 | } 110 | 111 | // duplicate task name 112 | if _, found := taskNames[task.Name]; found { 113 | msg += fmt.Sprintf(" duplicated task name %s;", task.Name) 114 | break 115 | } else { 116 | taskNames[task.Name] = task.Name 117 | } 118 | 119 | podName := makePodName(job.Name, task.Name, index) 120 | if err := validateK8sPodNameLength(podName); err != nil { 121 | msg += err.Error() 122 | } 123 | if err := validateTaskTemplate(task, job, index); err != nil { 124 | msg += err.Error() 125 | } 126 | } 127 | 128 | if err := validateJobName(job); err != nil { 129 | msg += err.Error() 130 | } 131 | 132 | if totalReplicas < job.Spec.MinAvailable { 133 | msg += "job 'minAvailable' should not be greater than total replicas in tasks;" 134 | } 135 | 136 | if err := utils.ValidatePolicies(job.Spec.Policies, field.NewPath("spec.policies")); err != nil { 137 | msg = msg + err.Error() + fmt.Sprintf(" valid events are %v, valid actions are %v;", 138 | utils.GetValidEvents(), utils.GetValidActions()) 139 | } 140 | 141 | if err := utils.ValidateIO(job.Spec.Volumes); err != nil { 142 | msg += err.Error() 143 | } 144 | 145 | if msg != "" { 146 | return fmt.Errorf(msg) 147 | } 148 | 149 | return nil 150 | } 151 | 152 | func validateTaskTemplate(task v1alpha1.TaskSpec, job *jobflowv1alpha1.JobTemplate, index int) error { 153 | var v1PodTemplate v1.PodTemplate 154 | v1PodTemplate.Template = *task.Template.DeepCopy() 155 | k8scorev1.SetObjectDefaults_PodTemplate(&v1PodTemplate) 156 | 157 | var coreTemplateSpec k8score.PodTemplateSpec 158 | if err := k8scorev1.Convert_v1_PodTemplateSpec_To_core_PodTemplateSpec(&v1PodTemplate.Template, &coreTemplateSpec, nil); err != nil { 159 | return fmt.Errorf("failed to convert v1_PodTemplateSpec to core_PodTemplateSpec") 160 | } 161 | 162 | // Skip verify container SecurityContex.Privileged as it depends on 163 | // the kube-apiserver `allow-privileged` flag. 164 | for i, container := range coreTemplateSpec.Spec.Containers { 165 | if container.SecurityContext != nil && container.SecurityContext.Privileged != nil { 166 | coreTemplateSpec.Spec.Containers[i].SecurityContext.Privileged = nil 167 | } 168 | } 169 | 170 | corePodTemplate := k8score.PodTemplate{ 171 | ObjectMeta: metav1.ObjectMeta{ 172 | Name: task.Name, 173 | Namespace: job.Namespace, 174 | }, 175 | Template: coreTemplateSpec, 176 | } 177 | 178 | if allErrs := k8scorevalid.ValidatePodTemplate(&corePodTemplate); len(allErrs) > 0 { 179 | msg := fmt.Sprintf("spec.task[%d].", index) 180 | for index := range allErrs { 181 | msg += allErrs[index].Error() + ". " 182 | } 183 | return fmt.Errorf(msg) 184 | } 185 | 186 | err := validateTaskTopoPolicy(task, index) 187 | if err != nil { 188 | return err 189 | } 190 | 191 | return nil 192 | } 193 | 194 | // MakePodName creates pod name. 195 | func makePodName(jobName string, taskName string, index int) string { 196 | return fmt.Sprintf("%s-%s-%d", jobName, taskName, index) 197 | } 198 | func validateK8sPodNameLength(podName string) error { 199 | if errMsgs := validation.IsQualifiedName(podName); len(errMsgs) > 0 { 200 | return fmt.Errorf(" create pod with name %s validate failed %v", podName, errMsgs) 201 | } 202 | return nil 203 | } 204 | func validateJobName(job *jobflowv1alpha1.JobTemplate) error { 205 | if errMsgs := validation.IsQualifiedName(job.Name); len(errMsgs) > 0 { 206 | return fmt.Errorf(" create job with name %s validate failed %v", job.Name, errMsgs) 207 | } 208 | return nil 209 | } 210 | 211 | func validateTaskTopoPolicy(task v1alpha1.TaskSpec, index int) error { 212 | if task.TopologyPolicy == "" || task.TopologyPolicy == v1alpha1.None { 213 | return nil 214 | } 215 | 216 | template := task.Template.DeepCopy() 217 | 218 | for id, container := range template.Spec.Containers { 219 | if len(container.Resources.Requests) == 0 { 220 | template.Spec.Containers[id].Resources.Requests = container.Resources.Limits.DeepCopy() 221 | } 222 | } 223 | 224 | for id, container := range template.Spec.InitContainers { 225 | if len(container.Resources.Requests) == 0 { 226 | template.Spec.InitContainers[id].Resources.Requests = container.Resources.Limits.DeepCopy() 227 | } 228 | } 229 | 230 | pod := &v1.Pod{ 231 | Spec: template.Spec, 232 | } 233 | 234 | if v1qos.GetPodQOS(pod) != v1.PodQOSGuaranteed { 235 | return fmt.Errorf("spec.task[%d] isn't Guaranteed pod, kind=%v", index, v1qos.GetPodQOS(pod)) 236 | } 237 | 238 | for id, container := range append(template.Spec.Containers, template.Spec.InitContainers...) { 239 | requestNum := guaranteedCPUs(container) 240 | if requestNum == 0 { 241 | return fmt.Errorf("the cpu request isn't an integer in spec.task[%d] container[%d].", 242 | index, id) 243 | } 244 | } 245 | 246 | return nil 247 | } 248 | 249 | func guaranteedCPUs(container v1.Container) int { 250 | cpuQuantity := container.Resources.Requests[v1.ResourceCPU] 251 | if cpuQuantity.Value()*1000 != cpuQuantity.MilliValue() { 252 | return 0 253 | } 254 | 255 | return int(cpuQuantity.Value()) 256 | } 257 | -------------------------------------------------------------------------------- /webhooks/router/admission.go: -------------------------------------------------------------------------------- 1 | package router 2 | 3 | import ( 4 | "fmt" 5 | "net/http" 6 | "sync" 7 | ) 8 | 9 | type AdmissionHandler func(w http.ResponseWriter, r *http.Request) 10 | 11 | var admissionMap = make(map[string]*AdmissionService) 12 | var admissionMutex sync.Mutex 13 | 14 | func RegisterAdmission(service *AdmissionService) error { 15 | admissionMutex.Lock() 16 | defer admissionMutex.Unlock() 17 | 18 | if _, found := admissionMap[service.Path]; found { 19 | return fmt.Errorf("duplicated admission service for %s", service.Path) 20 | } 21 | 22 | // Also register handler to the service. 23 | service.Handler = func(w http.ResponseWriter, r *http.Request) { 24 | Serve(w, r, service.Func) 25 | } 26 | 27 | admissionMap[service.Path] = service 28 | 29 | return nil 30 | } 31 | 32 | func ForEachAdmission(admissions []string, handler func(*AdmissionService)) { 33 | for _, admission := range admissions { 34 | if service, found := admissionMap[admission]; found { 35 | handler(service) 36 | } 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /webhooks/router/interface.go: -------------------------------------------------------------------------------- 1 | package router 2 | 3 | import ( 4 | "k8s.io/api/admission/v1beta1" 5 | whv1 "k8s.io/api/admissionregistration/v1" 6 | ) 7 | 8 | // The AdmitFunc returns response. 9 | type AdmitFunc func(v1beta1.AdmissionReview) error 10 | 11 | type AdmissionService struct { 12 | Path string 13 | Func AdmitFunc 14 | Handler AdmissionHandler 15 | 16 | ValidatingConfig *whv1.ValidatingWebhookConfiguration 17 | MutatingConfig *whv1.MutatingWebhookConfiguration 18 | } 19 | -------------------------------------------------------------------------------- /webhooks/router/server.go: -------------------------------------------------------------------------------- 1 | package router 2 | 3 | import ( 4 | "encoding/json" 5 | "io" 6 | "io/ioutil" 7 | "net/http" 8 | 9 | "k8s.io/api/admission/v1beta1" 10 | "k8s.io/apimachinery/pkg/runtime" 11 | "k8s.io/klog" 12 | 13 | "jobflow/webhooks/schema" 14 | "jobflow/webhooks/util" 15 | ) 16 | 17 | // CONTENTTYPE http content-type. 18 | var CONTENTTYPE = "Content-Type" 19 | 20 | // APPLICATIONJSON json content. 21 | var APPLICATIONJSON = "application/json" 22 | 23 | // Serve the http request. 24 | func Serve(w io.Writer, r *http.Request, admit AdmitFunc) { 25 | var body []byte 26 | if r.Body != nil { 27 | if data, err := ioutil.ReadAll(r.Body); err == nil { 28 | body = data 29 | } 30 | } 31 | 32 | // verify the content type is accurate 33 | contentType := r.Header.Get(CONTENTTYPE) 34 | if contentType != APPLICATIONJSON { 35 | klog.Errorf("contentType=%s, expect application/json", contentType) 36 | return 37 | } 38 | 39 | var reviewResponse *v1beta1.AdmissionResponse 40 | ar := v1beta1.AdmissionReview{} 41 | deserializer := schema.Codecs.UniversalDeserializer() 42 | if _, _, err := deserializer.Decode(body, nil, &ar); err != nil { 43 | reviewResponse = util.ToAdmissionResponse(err) 44 | } else { 45 | err = admit(ar) 46 | reviewResponse = util.ToAdmissionResponse(err) 47 | } 48 | klog.V(3).Infof("sending response: %v", reviewResponse) 49 | 50 | response := createResponse(reviewResponse, &ar) 51 | resp, err := json.Marshal(response) 52 | if err != nil { 53 | klog.Error(err) 54 | } 55 | if _, err := w.Write(resp); err != nil { 56 | klog.Error(err) 57 | } 58 | } 59 | 60 | func createResponse(reviewResponse *v1beta1.AdmissionResponse, ar *v1beta1.AdmissionReview) v1beta1.AdmissionReview { 61 | response := v1beta1.AdmissionReview{} 62 | if reviewResponse != nil { 63 | response.Response = reviewResponse 64 | response.Response.UID = ar.Request.UID 65 | } 66 | // reset the Object and OldObject, they are not needed in a response. 67 | ar.Request.Object = runtime.RawExtension{} 68 | ar.Request.OldObject = runtime.RawExtension{} 69 | 70 | return response 71 | } 72 | -------------------------------------------------------------------------------- /webhooks/schema/schema.go: -------------------------------------------------------------------------------- 1 | package schema 2 | 3 | import ( 4 | "fmt" 5 | "k8s.io/api/admission/v1beta1" 6 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 7 | "k8s.io/apimachinery/pkg/runtime" 8 | "k8s.io/apimachinery/pkg/runtime/serializer" 9 | "k8s.io/klog" 10 | corev1 "k8s.io/kubernetes/pkg/apis/core/v1" 11 | 12 | jobflowv1alpha1 "jobflow/api/v1alpha1" 13 | ) 14 | 15 | func init() { 16 | addToScheme(scheme) 17 | } 18 | 19 | var scheme = runtime.NewScheme() 20 | 21 | // Codecs is for retrieving serializers for the supported wire formats 22 | // and conversion wrappers to define preferred internal and external versions. 23 | var Codecs = serializer.NewCodecFactory(scheme) 24 | 25 | func addToScheme(scheme *runtime.Scheme) { 26 | corev1.AddToScheme(scheme) 27 | v1beta1.AddToScheme(scheme) 28 | } 29 | 30 | // DecodeJob decodes the jobFlow using deserializer from the raw object. 31 | func DecodeJobFlow(object runtime.RawExtension, resource metav1.GroupVersionResource) (*jobflowv1alpha1.JobFlow, error) { 32 | jobFlowResource := metav1.GroupVersionResource{Group: jobflowv1alpha1.GroupVersion.Group, Version: jobflowv1alpha1.GroupVersion.Version, Resource: "jobflows"} 33 | raw := object.Raw 34 | jobFlow := jobflowv1alpha1.JobFlow{} 35 | 36 | if resource != jobFlowResource { 37 | err := fmt.Errorf("expect resource to be %s", jobFlowResource) 38 | return &jobFlow, err 39 | } 40 | 41 | deserializer := Codecs.UniversalDeserializer() 42 | if _, _, err := deserializer.Decode(raw, nil, &jobFlow); err != nil { 43 | return &jobFlow, err 44 | } 45 | klog.V(3).Infof("the jobFlow struct is %+v", jobFlow) 46 | 47 | return &jobFlow, nil 48 | } 49 | 50 | // DecodeJob decodes the jobTemplate using deserializer from the raw object. 51 | func DecodeJobTemplate(object runtime.RawExtension, resource metav1.GroupVersionResource) (*jobflowv1alpha1.JobTemplate, error) { 52 | jobTemplateResource := metav1.GroupVersionResource{Group: jobflowv1alpha1.GroupVersion.Group, Version: jobflowv1alpha1.GroupVersion.Version, Resource: "jobtemplates"} 53 | raw := object.Raw 54 | jobTemplate := jobflowv1alpha1.JobTemplate{} 55 | 56 | if resource != jobTemplateResource { 57 | err := fmt.Errorf("expect resource to be %s", jobTemplateResource) 58 | return &jobTemplate, err 59 | } 60 | 61 | deserializer := Codecs.UniversalDeserializer() 62 | if _, _, err := deserializer.Decode(raw, nil, &jobTemplate); err != nil { 63 | return &jobTemplate, err 64 | } 65 | klog.V(3).Infof("the jobTemplate struct is %+v", jobTemplate) 66 | 67 | return &jobTemplate, nil 68 | } 69 | -------------------------------------------------------------------------------- /webhooks/server.go: -------------------------------------------------------------------------------- 1 | package webhooks 2 | 3 | import ( 4 | "context" 5 | "crypto/tls" 6 | "fmt" 7 | "io/ioutil" 8 | "net/http" 9 | "regexp" 10 | "strconv" 11 | "strings" 12 | 13 | whv1 "k8s.io/api/admissionregistration/v1" 14 | apierrors "k8s.io/apimachinery/pkg/api/errors" 15 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 16 | "k8s.io/client-go/kubernetes" 17 | restclient "k8s.io/client-go/rest" 18 | "k8s.io/klog" 19 | 20 | "jobflow/webhooks/router" 21 | ) 22 | 23 | const ( 24 | defaultQPS = 50.0 25 | defaultBurst = 100 26 | defaultPort = 8725 27 | webhookServiceName = "jobflow-webhook-service" 28 | webhookServiceNamespace = "kube-system" 29 | defaultEnabledAdmission = "/jobflows/validate,/jobtemplates/validate" 30 | caCertFilePath = "/tmp/k8s-webhook-server/serving-certs/ca.crt" 31 | tlsCertFilePath = "/tmp/k8s-webhook-server/serving-certs/tls.crt" 32 | tlsKeyFilePath = "/tmp/k8s-webhook-server/serving-certs/tls.key" 33 | ) 34 | 35 | func Run() error { 36 | caBundle, err := ioutil.ReadFile(caCertFilePath) 37 | if err != nil { 38 | return fmt.Errorf("unable to read cacert file (%s): %v", caCertFilePath, err) 39 | } 40 | restConfig, err := restclient.InClusterConfig() 41 | if err != nil { 42 | return fmt.Errorf("unable to build k8s config: %v", err) 43 | } 44 | restConfig.QPS = defaultQPS 45 | restConfig.Burst = defaultBurst 46 | kubeClient, err := kubernetes.NewForConfig(restConfig) 47 | if err != nil { 48 | return fmt.Errorf("unable to get k8s client: %v", err) 49 | } 50 | admissions := strings.Split(strings.TrimSpace(defaultEnabledAdmission), ",") 51 | router.ForEachAdmission(admissions, func(service *router.AdmissionService) { 52 | klog.V(3).Infof("Registered '%s' as webhook.", service.Path) 53 | http.HandleFunc(service.Path, service.Handler) 54 | klog.V(3).Infof("Registered configuration for webhook <%s>", service.Path) 55 | registerWebhookConfig(kubeClient, service, caBundle) 56 | }) 57 | 58 | server := &http.Server{ 59 | Addr: ":" + strconv.Itoa(defaultPort), 60 | TLSConfig: configTLS(), 61 | } 62 | go func() { 63 | err = server.ListenAndServeTLS("", "") 64 | if err != nil && err != http.ErrServerClosed { 65 | klog.Fatalf("ListenAndServeTLS for admission webhook failed: %v", err) 66 | } 67 | 68 | klog.Info("Volcano Webhook manager started.") 69 | }() 70 | 71 | return nil 72 | } 73 | 74 | func registerWebhookConfig(kubeClient *kubernetes.Clientset, service *router.AdmissionService, caBundle []byte) { 75 | clientConfig := whv1.WebhookClientConfig{ 76 | CABundle: caBundle, 77 | Service: &whv1.ServiceReference{ 78 | Name: webhookServiceName, 79 | Namespace: webhookServiceNamespace, 80 | Path: &service.Path, 81 | }, 82 | } 83 | 84 | if service.MutatingConfig != nil { 85 | for i := range service.MutatingConfig.Webhooks { 86 | service.MutatingConfig.Webhooks[i].ClientConfig = clientConfig 87 | } 88 | 89 | service.MutatingConfig.ObjectMeta.Name = webhookConfigName(service.Path) 90 | 91 | if err := registerMutateWebhook(kubeClient, service.MutatingConfig); err != nil { 92 | klog.Errorf("Failed to register mutating admission webhook (%s): %v", 93 | service.Path, err) 94 | } else { 95 | fmt.Printf("Registered mutating webhook for path <%s>.", service.Path) 96 | klog.V(3).Infof("Registered mutating webhook for path <%s>.", service.Path) 97 | } 98 | } 99 | if service.ValidatingConfig != nil { 100 | for i := range service.ValidatingConfig.Webhooks { 101 | service.ValidatingConfig.Webhooks[i].ClientConfig = clientConfig 102 | } 103 | 104 | service.ValidatingConfig.ObjectMeta.Name = webhookConfigName(service.Path) 105 | 106 | if err := registerValidateWebhook(kubeClient, service.ValidatingConfig); err != nil { 107 | klog.Errorf("Failed to register validating admission webhook (%s): %v", 108 | service.Path, err) 109 | } else { 110 | klog.V(3).Infof("Registered validating webhook for path <%s>.", service.Path) 111 | } 112 | } 113 | } 114 | 115 | func registerMutateWebhook(clientset *kubernetes.Clientset, hook *whv1.MutatingWebhookConfiguration) error { 116 | client := clientset.AdmissionregistrationV1().MutatingWebhookConfigurations() 117 | existing, err := client.Get(context.TODO(), hook.Name, metav1.GetOptions{}) 118 | if err != nil && !apierrors.IsNotFound(err) { 119 | return err 120 | } 121 | if err == nil && existing != nil { 122 | klog.V(4).Infof("Updating MutatingWebhookConfiguration %v", hook) 123 | existing.Webhooks = hook.Webhooks 124 | if _, err := client.Update(context.TODO(), existing, metav1.UpdateOptions{}); err != nil { 125 | return err 126 | } 127 | } else { 128 | klog.V(4).Infof("Creating MutatingWebhookConfiguration %v", hook) 129 | if _, err := client.Create(context.TODO(), hook, metav1.CreateOptions{}); err != nil { 130 | return err 131 | } 132 | } 133 | 134 | return nil 135 | } 136 | 137 | func registerValidateWebhook(clientset *kubernetes.Clientset, hook *whv1.ValidatingWebhookConfiguration) error { 138 | client := clientset.AdmissionregistrationV1().ValidatingWebhookConfigurations() 139 | 140 | existing, err := client.Get(context.TODO(), hook.Name, metav1.GetOptions{}) 141 | if err != nil && !apierrors.IsNotFound(err) { 142 | return err 143 | } 144 | if err == nil && existing != nil { 145 | existing.Webhooks = hook.Webhooks 146 | klog.V(4).Infof("Updating ValidatingWebhookConfiguration %v", hook) 147 | if _, err := client.Update(context.TODO(), existing, metav1.UpdateOptions{}); err != nil { 148 | return err 149 | } 150 | } else { 151 | klog.V(4).Infof("Creating ValidatingWebhookConfiguration %v", hook) 152 | if _, err := client.Create(context.TODO(), hook, metav1.CreateOptions{}); err != nil { 153 | return err 154 | } 155 | } 156 | 157 | return nil 158 | } 159 | 160 | func webhookConfigName(path string) string { 161 | name := "webhook" 162 | re := regexp.MustCompile(`-+`) 163 | raw := strings.Join([]string{name, strings.ReplaceAll(path, "/", "-")}, "-") 164 | return re.ReplaceAllString(raw, "-") 165 | } 166 | 167 | func configTLS() *tls.Config { 168 | sCert, err := tls.LoadX509KeyPair(tlsCertFilePath, tlsKeyFilePath) 169 | if err != nil { 170 | klog.Fatal(err) 171 | } 172 | 173 | return &tls.Config{ 174 | Certificates: []tls.Certificate{sCert}, 175 | } 176 | } 177 | -------------------------------------------------------------------------------- /webhooks/util/util.go: -------------------------------------------------------------------------------- 1 | package util 2 | 3 | import ( 4 | "k8s.io/api/admission/v1beta1" 5 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 6 | "k8s.io/klog" 7 | ) 8 | 9 | // ToAdmissionResponse updates the admission response with the input error. 10 | func ToAdmissionResponse(err error) *v1beta1.AdmissionResponse { 11 | if err != nil { 12 | klog.Error(err) 13 | return &v1beta1.AdmissionResponse{ 14 | Allowed: false, 15 | Result: &metav1.Status{ 16 | Message: err.Error(), 17 | }, 18 | } 19 | } else { 20 | return &v1beta1.AdmissionResponse{ 21 | Allowed: true, 22 | } 23 | } 24 | } 25 | --------------------------------------------------------------------------------