├── .dockerignore ├── .gitee └── PULL_REQUEST_TEMPLATE.md ├── .github ├── ISSUE_TEMPLATE │ ├── RFC.md │ ├── bug-report.md │ └── task-tracking.md └── PULL_REQUEST_TEMPLATE.md ├── .gitignore ├── Dockerfile ├── LICENSE ├── Makefile ├── OWNERS ├── PROJECT ├── README.md ├── cmd └── manager │ └── main.go ├── config ├── crd │ ├── bases │ │ └── mindspore.gitee.com_msjobs.yaml │ ├── kustomization.yaml │ ├── kustomizeconfig.yaml │ └── patches │ │ ├── cainjection_in_msjobs.yaml │ │ └── webhook_in_msjobs.yaml ├── default │ ├── kustomization.yaml │ ├── manager_auth_proxy_patch.yaml │ └── manager_config_patch.yaml ├── manager │ ├── controller_manager_config.yaml │ ├── kustomization.yaml │ └── manager.yaml ├── prometheus │ ├── kustomization.yaml │ └── monitor.yaml ├── rbac │ ├── auth_proxy_client_clusterrole.yaml │ ├── auth_proxy_role.yaml │ ├── auth_proxy_role_binding.yaml │ ├── auth_proxy_service.yaml │ ├── kustomization.yaml │ ├── leader_election_role.yaml │ ├── leader_election_role_binding.yaml │ ├── msjob_editor_role.yaml │ ├── msjob_viewer_role.yaml │ ├── role.yaml │ ├── role_binding.yaml │ └── service_account.yaml └── samples │ ├── ms_wide_deep_dataparallel.yaml │ ├── ms_wide_deep_ps_distribute.yaml │ ├── ms_wide_deep_ps_standalone.yaml │ └── ms_wide_deep_standalone.yaml ├── deploy └── v1 │ └── ms-operator.yaml ├── go.mod ├── go.sum ├── hack ├── boilerplate.go.txt └── update-codegen.sh └── pkg ├── apis └── v1 │ ├── common.go │ ├── constants.go │ ├── defaults.go │ ├── doc.go │ ├── groupversion_info.go │ ├── msjob_types.go │ ├── zz_generated.deepcopy.go │ └── zz_generated.defaults.go └── controllers └── v1 ├── msjob_controller.go ├── suite_test.go └── utils.go /.dockerignore: -------------------------------------------------------------------------------- 1 | # More info: https://docs.docker.com/engine/reference/builder/#dockerignore-file 2 | # Ignore build and test binaries. 3 | bin/ 4 | testbin/ 5 | -------------------------------------------------------------------------------- /.gitee/PULL_REQUEST_TEMPLATE.md: -------------------------------------------------------------------------------- 1 | 5 | 6 | **What type of PR is this?** 7 | > Uncomment only one ` /kind <>` line, hit enter to put that in a new line, and remove leading whitespaces from that line: 8 | > 9 | > /kind bug 10 | > /kind task 11 | > /kind feature 12 | 13 | 14 | **What does this PR do / why do we need it**: 15 | 16 | 17 | **Which issue(s) this PR fixes**: 18 | 22 | Fixes # 23 | 24 | **Special notes for your reviewers**: 25 | 26 | 27 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/RFC.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: RFC 3 | about: Use this template for the new feature or enhancement 4 | labels: kind/feature or kind/enhancement 5 | 6 | --- 7 | 8 | ## Background 9 | - Describe the status of the problem you wish to solve 10 | - Attach the relevant issue if have 11 | 12 | ## Introduction 13 | - Describe the general solution, design and/or pseudo-code 14 | 15 | ## Trail 16 | | No. | Task Description | Related Issue(URL) | 17 | | --- | ---------------- | ------------------ | 18 | | 1 | | | 19 | | 2 | | | 20 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/bug-report.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Bug Report 3 | about: Use this template for reporting a bug 4 | labels: kind/bug 5 | 6 | --- 7 | 8 | 12 | 13 | ## Environment 14 | ### Hardware Environment(`Ascend`/`GPU`/`CPU`): 15 | > Uncomment only one ` /device <>` line, hit enter to put that in a new line, and remove leading whitespaces from that line: 16 | > 17 | > `/device ascend`
18 | > `/device gpu`
19 | > `/device cpu`
20 | 21 | ### Software Environment: 22 | - **MindSpore version (source or binary)**: 23 | - **Python version (e.g., Python 3.7.5)**: 24 | - **OS platform and distribution (e.g., Linux Ubuntu 16.04)**: 25 | - **GCC/Compiler version (if compiled from source)**: 26 | 27 | ## Describe the current behavior 28 | 29 | 30 | ## Describe the expected behavior 31 | 32 | 33 | ## Steps to reproduce the issue 34 | 1. 35 | 2. 36 | 3. 37 | 38 | ## Related log / screenshot 39 | 40 | 41 | ## Special notes for this issue 42 | 43 | 44 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/task-tracking.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Task 3 | about: Use this template for task tracking 4 | labels: kind/task 5 | 6 | --- 7 | 8 | ## Task Description 9 | 10 | 11 | ## Task Goal 12 | 13 | 14 | ## Sub Task 15 | | No. | Task Description | Issue ID | 16 | | --- | ---------------- | -------- | 17 | | 1 | | | 18 | | 2 | | | 19 | 20 | -------------------------------------------------------------------------------- /.github/PULL_REQUEST_TEMPLATE.md: -------------------------------------------------------------------------------- 1 | 5 | 6 | **What type of PR is this?** 7 | > Uncomment only one ` /kind <>` line, hit enter to put that in a new line, and remove leading whitespaces from that line: 8 | > 9 | > `/kind bug`
10 | > `/kind task`
11 | > `/kind feature`
12 | 13 | **What does this PR do / why do we need it**: 14 | 15 | 16 | **Which issue(s) this PR fixes**: 17 | 21 | Fixes # 22 | 23 | **Special notes for your reviewers**: 24 | 25 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Binareis for programs and plugins 2 | bin 3 | testbin/* 4 | vendor/ 5 | # Other temporary files 6 | .DS_Store 7 | 8 | # Files created by Gogland IDE 9 | .idea/ 10 | 11 | # IDEs 12 | .vscode/ 13 | __debug_bin -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | # Build the manager binary 2 | FROM golang:1.17 as builder 3 | 4 | WORKDIR /workspace 5 | # Copy the Go Modules manifests 6 | COPY go.mod go.mod 7 | COPY go.sum go.sum 8 | # cache deps before building and copying source so that we don't need to re-download as much 9 | # and so that source changes don't invalidate our downloaded layer 10 | RUN go env -w GOPROXY=https://goproxy.cn,direct 11 | RUN go mod download 12 | 13 | # Copy the go source 14 | COPY cmd/manager/main.go cmd/manager/main.go 15 | COPY pkg/apis/ pkg/apis/ 16 | COPY pkg/controllers/v1/ pkg/controllers/v1/ 17 | 18 | # Build 19 | RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -a -o manager cmd/manager/main.go 20 | 21 | # Use distroless as minimal base image to package the manager binary 22 | # Refer to https://github.com/GoogleContainerTools/distroless for more details 23 | FROM gcr.io/distroless/static:nonroot 24 | WORKDIR / 25 | COPY --from=builder /workspace/manager . 26 | USER 65532:65532 27 | 28 | ENTRYPOINT ["/manager"] 29 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright [yyyy] [name of copyright owner] 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | 2 | # Image URL to use all building/pushing image targets 3 | IMG ?= controller:latest 4 | # ENVTEST_K8S_VERSION refers to the version of kubebuilder assets to be downloaded by envtest binary. 5 | ENVTEST_K8S_VERSION = 1.23 6 | 7 | # Get the currently used golang install path (in GOPATH/bin, unless GOBIN is set) 8 | ifeq (,$(shell go env GOBIN)) 9 | GOBIN=$(shell go env GOPATH)/bin 10 | else 11 | GOBIN=$(shell go env GOBIN) 12 | endif 13 | 14 | # Setting SHELL to bash allows bash commands to be executed by recipes. 15 | # This is a requirement for 'setup-envtest.sh' in the test target. 16 | # Options are set to exit when a recipe line exits non-zero or a piped command fails. 17 | SHELL = /usr/bin/env bash -o pipefail 18 | .SHELLFLAGS = -ec 19 | 20 | .PHONY: all 21 | all: build 22 | 23 | ##@ General 24 | 25 | # The help target prints out all targets with their descriptions organized 26 | # beneath their categories. The categories are represented by '##@' and the 27 | # target descriptions by '##'. The awk commands is responsible for reading the 28 | # entire set of makefiles included in this invocation, looking for lines of the 29 | # file as xyz: ## something, and then pretty-format the target and help. Then, 30 | # if there's a line with ##@ something, that gets pretty-printed as a category. 31 | # More info on the usage of ANSI control characters for terminal formatting: 32 | # https://en.wikipedia.org/wiki/ANSI_escape_code#SGR_parameters 33 | # More info on the awk command: 34 | # http://linuxcommand.org/lc3_adv_awk.php 35 | 36 | .PHONY: help 37 | help: ## Display this help. 38 | @awk 'BEGIN {FS = ":.*##"; printf "\nUsage:\n make \033[36m\033[0m\n"} /^[a-zA-Z_0-9-]+:.*?##/ { printf " \033[36m%-15s\033[0m %s\n", $$1, $$2 } /^##@/ { printf "\n\033[1m%s\033[0m\n", substr($$0, 5) } ' $(MAKEFILE_LIST) 39 | 40 | ##@ Development 41 | 42 | .PHONY: manifests 43 | manifests: controller-gen ## Generate WebhookConfiguration, ClusterRole and CustomResourceDefinition objects. 44 | $(CONTROLLER_GEN) rbac:roleName=manager-role crd webhook paths="./..." output:crd:artifacts:config=config/crd/bases 45 | 46 | .PHONY: generate 47 | generate: controller-gen ## Generate code containing DeepCopy, DeepCopyInto, and DeepCopyObject method implementations. 48 | $(CONTROLLER_GEN) object:headerFile="hack/boilerplate.go.txt" paths="./..." 49 | 50 | .PHONY: fmt 51 | fmt: ## Run go fmt against code. 52 | go fmt ./... 53 | 54 | .PHONY: vet 55 | vet: ## Run go vet against code. 56 | go vet ./... 57 | 58 | .PHONY: test 59 | test: manifests generate fmt vet envtest ## Run tests. 60 | KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) -p path)" go test ./... -coverprofile cover.out 61 | 62 | ##@ Build 63 | 64 | .PHONY: build 65 | build: generate fmt vet ## Build manager binary. 66 | go build -o bin/manager cmd/manager/main.go 67 | 68 | .PHONY: run 69 | run: manifests generate fmt vet ## Run a controller from your host. 70 | go run ./cmd/manager/main.go 71 | 72 | .PHONY: docker-build 73 | docker-build: ## Build docker image with the manager. 74 | docker build -t ${IMG} . 75 | 76 | .PHONY: docker-push 77 | docker-push: ## Push docker image with the manager. 78 | docker push ${IMG} 79 | 80 | ##@ Deployment 81 | 82 | ifndef ignore-not-found 83 | ignore-not-found = false 84 | endif 85 | 86 | .PHONY: install 87 | install: manifests kustomize ## Install CRDs into the K8s cluster specified in ~/.kube/config. 88 | $(KUSTOMIZE) build config/crd | kubectl apply -f - 89 | 90 | .PHONY: uninstall 91 | uninstall: manifests kustomize ## Uninstall CRDs from the K8s cluster specified in ~/.kube/config. Call with ignore-not-found=true to ignore resource not found errors during deletion. 92 | $(KUSTOMIZE) build config/crd | kubectl delete --ignore-not-found=$(ignore-not-found) -f - 93 | 94 | .PHONY: deploy 95 | deploy: manifests kustomize ## Deploy controller to the K8s cluster specified in ~/.kube/config. 96 | cd config/manager && $(KUSTOMIZE) edit set image controller=${IMG} 97 | $(KUSTOMIZE) build config/default | kubectl apply -f - 98 | 99 | .PHONY: undeploy 100 | undeploy: ## Undeploy controller from the K8s cluster specified in ~/.kube/config. Call with ignore-not-found=true to ignore resource not found errors during deletion. 101 | $(KUSTOMIZE) build config/default | kubectl delete --ignore-not-found=$(ignore-not-found) -f - 102 | 103 | CONTROLLER_GEN = $(shell pwd)/bin/controller-gen 104 | .PHONY: controller-gen 105 | controller-gen: ## Download controller-gen locally if necessary. 106 | $(call go-get-tool,$(CONTROLLER_GEN),sigs.k8s.io/controller-tools/cmd/controller-gen@v0.8.0) 107 | 108 | KUSTOMIZE = $(shell pwd)/bin/kustomize 109 | .PHONY: kustomize 110 | kustomize: ## Download kustomize locally if necessary. 111 | $(call go-get-tool,$(KUSTOMIZE),sigs.k8s.io/kustomize/kustomize/v3@v3.8.7) 112 | 113 | ENVTEST = $(shell pwd)/bin/setup-envtest 114 | .PHONY: envtest 115 | envtest: ## Download envtest-setup locally if necessary. 116 | $(call go-get-tool,$(ENVTEST),sigs.k8s.io/controller-runtime/tools/setup-envtest@latest) 117 | 118 | # go-get-tool will 'go get' any package $2 and install it to $1. 119 | PROJECT_DIR := $(shell dirname $(abspath $(lastword $(MAKEFILE_LIST)))) 120 | define go-get-tool 121 | @[ -f $(1) ] || { \ 122 | set -e ;\ 123 | TMP_DIR=$$(mktemp -d) ;\ 124 | cd $$TMP_DIR ;\ 125 | go mod init tmp ;\ 126 | echo "Downloading $(2)" ;\ 127 | GOBIN=$(PROJECT_DIR)/bin go get $(2) ;\ 128 | rm -rf $$TMP_DIR ;\ 129 | } 130 | endef 131 | -------------------------------------------------------------------------------- /OWNERS: -------------------------------------------------------------------------------- 1 | approvers: 2 | - cristoval 3 | - fangzehua -------------------------------------------------------------------------------- /PROJECT: -------------------------------------------------------------------------------- 1 | domain: gitee.com 2 | layout: 3 | - go.kubebuilder.io/v3 4 | projectName: ms-operator 5 | repo: ms-operator 6 | resources: 7 | - api: 8 | crdVersion: v1 9 | namespaced: true 10 | controller: true 11 | domain: gitee.com 12 | group: mindspore 13 | kind: MSJob 14 | path: ms-operator/api/v1 15 | version: v1 16 | version: "3" 17 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # MindSpore Operator 2 | MindSpore Operator 是MindSpore在Kubernetes上进行分布式训练的插件。CRD（Custom Resource Definition）中定义了Scheduler、PS、Worker三种角色，用户只需配置yaml文件，即可轻松实现分布式训练。 3 | 4 | # 安装 5 | 安装方法可以有以下几种 6 | ## 1. 使用yaml直接安装 7 | ``` 8 | kubectl apply -f deploy/v1/ms-operator.yaml 9 | ``` 10 | 安装后： 11 | 使用`kubectl get pods --all-namespaces`,即可看到namespace为ms-operator-system的部署任务。 12 | 使用`kubectl describe pod ms-operator-controller-manager-xxx-xxx -n ms-operator-system`，可查看pod的详细信息。 13 | ## 2. 使用make deploy安装 14 | ``` 15 | make deploy IMG=swr.cn-south-1.myhuaweicloud.com/mindspore/ms-operator:latest 16 | ``` 17 | 18 | ## 3. 本地调试环境 19 | ``` 20 | make run 21 | ``` 22 | 23 | # Samples 24 | 当前ms-operator支持普通单Worker训练、PS模式的单Worker训练以及自动并行（例如数据并行、模型并行等）的Scheduler、Worker启动。 25 | 26 | 在`config/samples/`中有运行样例。 27 | 以数据并行的Scheduler、Worker启动为例，其中数据集和网络脚本需提前准备： 28 | ``` 29 | kubectl apply -f config/samples/ms_wide_deep_dataparallel.yaml 30 | ``` 31 | 使用`kubectl get all -o wide`即可看到集群中启动的Scheduler和Worker，以及Scheduler对应的Service。 32 | # 开发指南 33 | ## 核心代码： 34 | `pkg/apis/v1/msjob_types.go`中为MSJob的CRD定义。 35 | `pkg/controllers/v1/msjob_controller.go`中为MSJob controller的核心逻辑。 36 | ## 镜像制作、上传 37 | ``` 38 | make docker-build IMG=swr.cn-south-1.myhuaweicloud.com/mindspore/ms-operator:latest 39 | docker push swr.cn-south-1.myhuaweicloud.com/mindspore/ms-operator:latest 40 | ``` 41 | ## 常见问题 42 | - 镜像构建过程中若发现gcr.io/distroless/static无法拉取，可参考[issue](https://github.com/anjia0532/gcr.io_mirror/issues/169) 43 | - 安装部署过程中发现gcr.io/kubebuilder/kube-rbac-proxy无法拉取，参考[issue](https://github.com/anjia0532/gcr.io_mirror/issues/153) 44 | 45 | -------------------------------------------------------------------------------- /cmd/manager/main.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2022 Huawei Technologies Co., Ltd. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package main 18 | 19 | import ( 20 | "flag" 21 | "os" 22 | 23 | // Import all Kubernetes client auth plugins (e.g. Azure, GCP, OIDC, etc.) 24 | // to ensure that exec-entrypoint and run can make use of them. 25 | _ "k8s.io/client-go/plugin/pkg/client/auth" 26 | 27 | "k8s.io/apimachinery/pkg/runtime" 28 | utilruntime "k8s.io/apimachinery/pkg/util/runtime" 29 | clientgoscheme "k8s.io/client-go/kubernetes/scheme" 30 | ctrl "sigs.k8s.io/controller-runtime" 31 | "sigs.k8s.io/controller-runtime/pkg/healthz" 32 | "sigs.k8s.io/controller-runtime/pkg/log/zap" 33 | 34 | mindsporev1 "ms-operator/pkg/apis/v1" 35 | "ms-operator/pkg/controllers/v1" 36 | //+kubebuilder:scaffold:imports 37 | ) 38 | 39 | var ( 40 | scheme = runtime.NewScheme() 41 | setupLog = ctrl.Log.WithName("setup") 42 | ) 43 | 44 | func init() { 45 | utilruntime.Must(clientgoscheme.AddToScheme(scheme)) 46 | 47 | utilruntime.Must(mindsporev1.AddToScheme(scheme)) 48 | //+kubebuilder:scaffold:scheme 49 | } 50 | 51 | func main() { 52 | var metricsAddr string 53 | var enableLeaderElection bool 54 | var enableGangScheduling bool 55 | var gangSchedulerName string 56 | var probeAddr string 57 | flag.StringVar(&metricsAddr, "metrics-bind-address", ":8080", "The address the metric endpoint binds to.") 58 | flag.StringVar(&probeAddr, "health-probe-bind-address", ":8081", "The address the probe endpoint binds to.") 59 | flag.BoolVar(&enableGangScheduling, "enable-gang-scheduling", false, "Set true to enable gang scheduling") 60 | flag.StringVar(&gangSchedulerName, "gang-scheduler-name", "volcano", "The scheduler to gang-schedule kubeflow jobs, defaults to volcano") 61 | flag.BoolVar(&enableLeaderElection, "leader-elect", false, 62 | "Enable leader election for controller manager. "+ 63 | "Enabling this will ensure there is only one active controller manager.") 64 | opts := zap.Options{ 65 | Development: true, 66 | } 67 | opts.BindFlags(flag.CommandLine) 68 | flag.Parse() 69 | 70 | ctrl.SetLogger(zap.New(zap.UseFlagOptions(&opts))) 71 | 72 | mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{ 73 | Scheme: scheme, 74 | MetricsBindAddress: metricsAddr, 75 | Port: 9443, 76 | HealthProbeBindAddress: probeAddr, 77 | LeaderElection: enableLeaderElection, 78 | LeaderElectionID: "4725f7c4.gitee.com", 79 | }) 80 | if err != nil { 81 | setupLog.Error(err, "unable to start manager") 82 | os.Exit(1) 83 | } 84 | 85 | if err = controllers.NewReconciler(mgr, enableGangScheduling).SetupWithManager(mgr); err != nil { 86 | setupLog.Error(err, "unable to create controller", "controller", "MSJob") 87 | os.Exit(1) 88 | } 89 | //+kubebuilder:scaffold:builder 90 | 91 | if err := mgr.AddHealthzCheck("healthz", healthz.Ping); err != nil { 92 | setupLog.Error(err, "unable to set up health check") 93 | os.Exit(1) 94 | } 95 | if err := mgr.AddReadyzCheck("readyz", healthz.Ping); err != nil { 96 | setupLog.Error(err, "unable to set up ready check") 97 | os.Exit(1) 98 | } 99 | 100 | setupLog.Info("starting manager") 101 | if err := mgr.Start(ctrl.SetupSignalHandler()); err != nil { 102 | setupLog.Error(err, "problem running manager") 103 | os.Exit(1) 104 | } 105 | } 106 | -------------------------------------------------------------------------------- /config/crd/kustomization.yaml: -------------------------------------------------------------------------------- 1 | # This kustomization.yaml is not intended to be run by itself, 2 | # since it depends on service name and namespace that are out of this kustomize package. 3 | # It should be run by config/default 4 | resources: 5 | - bases/mindspore.gitee.com_msjobs.yaml 6 | #+kubebuilder:scaffold:crdkustomizeresource 7 | 8 | patchesStrategicMerge: 9 | # [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix. 10 | # patches here are for enabling the conversion webhook for each CRD 11 | #- patches/webhook_in_msjobs.yaml 12 | #+kubebuilder:scaffold:crdkustomizewebhookpatch 13 | 14 | # [CERTMANAGER] To enable cert-manager, uncomment all the sections with [CERTMANAGER] prefix. 15 | # patches here are for enabling the CA injection for each CRD 16 | #- patches/cainjection_in_msjobs.yaml 17 | #+kubebuilder:scaffold:crdkustomizecainjectionpatch 18 | 19 | # the following config is for teaching kustomize how to do kustomization for CRDs. 20 | configurations: 21 | - kustomizeconfig.yaml 22 | -------------------------------------------------------------------------------- /config/crd/kustomizeconfig.yaml: -------------------------------------------------------------------------------- 1 | # This file is for teaching kustomize how to substitute name and namespace reference in CRD 2 | nameReference: 3 | - kind: Service 4 | version: v1 5 | fieldSpecs: 6 | - kind: CustomResourceDefinition 7 | version: v1 8 | group: apiextensions.k8s.io 9 | path: spec/conversion/webhook/clientConfig/service/name 10 | 11 | namespace: 12 | - kind: CustomResourceDefinition 13 | version: v1 14 | group: apiextensions.k8s.io 15 | path: spec/conversion/webhook/clientConfig/service/namespace 16 | create: false 17 | 18 | varReference: 19 | - path: metadata/annotations 20 | -------------------------------------------------------------------------------- /config/crd/patches/cainjection_in_msjobs.yaml: -------------------------------------------------------------------------------- 1 | # The following patch adds a directive for certmanager to inject CA into the CRD 2 | apiVersion: apiextensions.k8s.io/v1 3 | kind: CustomResourceDefinition 4 | metadata: 5 | annotations: 6 | cert-manager.io/inject-ca-from: $(CERTIFICATE_NAMESPACE)/$(CERTIFICATE_NAME) 7 | name: msjobs.mindspore.gitee.com 8 | -------------------------------------------------------------------------------- /config/crd/patches/webhook_in_msjobs.yaml: -------------------------------------------------------------------------------- 1 | # The following patch enables a conversion webhook for the CRD 2 | apiVersion: apiextensions.k8s.io/v1 3 | kind: CustomResourceDefinition 4 | metadata: 5 | name: msjobs.mindspore.gitee.com 6 | spec: 7 | conversion: 8 | strategy: Webhook 9 | webhook: 10 | clientConfig: 11 | service: 12 | namespace: system 13 | name: webhook-service 14 | path: /convert 15 | conversionReviewVersions: 16 | - v1 17 | -------------------------------------------------------------------------------- /config/default/kustomization.yaml: -------------------------------------------------------------------------------- 1 | # Adds namespace to all resources. 2 | namespace: ms-operator-system 3 | 4 | # Value of this field is prepended to the 5 | # names of all resources, e.g. a deployment named 6 | # "wordpress" becomes "alices-wordpress". 7 | # Note that it should also match with the prefix (text before '-') of the namespace 8 | # field above. 9 | namePrefix: ms-operator- 10 | 11 | # Labels to add to all resources and selectors. 12 | #commonLabels: 13 | # someName: someValue 14 | 15 | bases: 16 | - ../crd 17 | - ../rbac 18 | - ../manager 19 | # [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix including the one in 20 | # crd/kustomization.yaml 21 | #- ../webhook 22 | # [CERTMANAGER] To enable cert-manager, uncomment all sections with 'CERTMANAGER'. 'WEBHOOK' components are required. 23 | #- ../certmanager 24 | # [PROMETHEUS] To enable prometheus monitor, uncomment all sections with 'PROMETHEUS'. 25 | #- ../prometheus 26 | 27 | patchesStrategicMerge: 28 | # Protect the /metrics endpoint by putting it behind auth. 29 | # If you want your controller-manager to expose the /metrics 30 | # endpoint w/o any authn/z, please comment the following line. 31 | - manager_auth_proxy_patch.yaml 32 | 33 | # Mount the controller config file for loading manager configurations 34 | # through a ComponentConfig type 35 | #- manager_config_patch.yaml 36 | 37 | # [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix including the one in 38 | # crd/kustomization.yaml 39 | #- manager_webhook_patch.yaml 40 | 41 | # [CERTMANAGER] To enable cert-manager, uncomment all sections with 'CERTMANAGER'. 42 | # Uncomment 'CERTMANAGER' sections in crd/kustomization.yaml to enable the CA injection in the admission webhooks. 43 | # 'CERTMANAGER' needs to be enabled to use ca injection 44 | #- webhookcainjection_patch.yaml 45 | 46 | # the following config is for teaching kustomize how to do var substitution 47 | vars: 48 | # [CERTMANAGER] To enable cert-manager, uncomment all sections with 'CERTMANAGER' prefix. 49 | #- name: CERTIFICATE_NAMESPACE # namespace of the certificate CR 50 | # objref: 51 | # kind: Certificate 52 | # group: cert-manager.io 53 | # version: v1 54 | # name: serving-cert # this name should match the one in certificate.yaml 55 | # fieldref: 56 | # fieldpath: metadata.namespace 57 | #- name: CERTIFICATE_NAME 58 | # objref: 59 | # kind: Certificate 60 | # group: cert-manager.io 61 | # version: v1 62 | # name: serving-cert # this name should match the one in certificate.yaml 63 | #- name: SERVICE_NAMESPACE # namespace of the service 64 | # objref: 65 | # kind: Service 66 | # version: v1 67 | # name: webhook-service 68 | # fieldref: 69 | # fieldpath: metadata.namespace 70 | #- name: SERVICE_NAME 71 | # objref: 72 | # kind: Service 73 | # version: v1 74 | # name: webhook-service 75 | -------------------------------------------------------------------------------- /config/default/manager_auth_proxy_patch.yaml: -------------------------------------------------------------------------------- 1 | # This patch inject a sidecar container which is a HTTP proxy for the 2 | # controller manager, it performs RBAC authorization against the Kubernetes API using SubjectAccessReviews. 3 | apiVersion: apps/v1 4 | kind: Deployment 5 | metadata: 6 | name: controller-manager 7 | namespace: system 8 | spec: 9 | template: 10 | spec: 11 | containers: 12 | - name: kube-rbac-proxy 13 | image: gcr.io/kubebuilder/kube-rbac-proxy:v0.8.0 14 | args: 15 | - "--secure-listen-address=0.0.0.0:8443" 16 | - "--upstream=http://127.0.0.1:8080/" 17 | - "--logtostderr=true" 18 | - "--v=0" 19 | ports: 20 | - containerPort: 8443 21 | protocol: TCP 22 | name: https 23 | resources: 24 | limits: 25 | cpu: 500m 26 | memory: 128Mi 27 | requests: 28 | cpu: 5m 29 | memory: 64Mi 30 | - name: manager 31 | args: 32 | - "--health-probe-bind-address=:8081" 33 | - "--metrics-bind-address=127.0.0.1:8080" 34 | - "--leader-elect" 35 | -------------------------------------------------------------------------------- /config/default/manager_config_patch.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: apps/v1 2 | kind: Deployment 3 | metadata: 4 | name: controller-manager 5 | namespace: system 6 | spec: 7 | template: 8 | spec: 9 | containers: 10 | - name: manager 11 | args: 12 | - "--config=controller_manager_config.yaml" 13 | volumeMounts: 14 | - name: manager-config 15 | mountPath: /controller_manager_config.yaml 16 | subPath: controller_manager_config.yaml 17 | volumes: 18 | - name: manager-config 19 | configMap: 20 | name: manager-config 21 | -------------------------------------------------------------------------------- /config/manager/controller_manager_config.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: controller-runtime.sigs.k8s.io/v1alpha1 2 | kind: ControllerManagerConfig 3 | health: 4 | healthProbeBindAddress: :8081 5 | metrics: 6 | bindAddress: 127.0.0.1:8080 7 | webhook: 8 | port: 9443 9 | leaderElection: 10 | leaderElect: true 11 | resourceName: 4725f7c4.gitee.com 12 | -------------------------------------------------------------------------------- /config/manager/kustomization.yaml: -------------------------------------------------------------------------------- 1 | resources: 2 | - manager.yaml 3 | 4 | generatorOptions: 5 | disableNameSuffixHash: true 6 | 7 | configMapGenerator: 8 | - files: 9 | - controller_manager_config.yaml 10 | name: manager-config 11 | apiVersion: kustomize.config.k8s.io/v1beta1 12 | kind: Kustomization 13 | images: 14 | - name: controller 15 | newName: swr.cn-south-1.myhuaweicloud.com/mindspore/ms-operator 16 | newTag: latest 17 | - name: controller=IMG=mindspore/ms-operator 18 | newTag: latest 19 | -------------------------------------------------------------------------------- /config/manager/manager.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Namespace 3 | metadata: 4 | labels: 5 | control-plane: controller-manager 6 | name: system 7 | --- 8 | apiVersion: apps/v1 9 | kind: Deployment 10 | metadata: 11 | name: controller-manager 12 | namespace: system 13 | labels: 14 | control-plane: controller-manager 15 | spec: 16 | selector: 17 | matchLabels: 18 | control-plane: controller-manager 19 | replicas: 1 20 | template: 21 | metadata: 22 | annotations: 23 | kubectl.kubernetes.io/default-container: manager 24 | labels: 25 | control-plane: controller-manager 26 | spec: 27 | securityContext: 28 | runAsNonRoot: true 29 | containers: 30 | - command: 31 | - /manager 32 | args: 33 | - --leader-elect 34 | image: controller:latest 35 | name: manager 36 | securityContext: 37 | allowPrivilegeEscalation: false 38 | livenessProbe: 39 | httpGet: 40 | path: /healthz 41 | port: 8081 42 | initialDelaySeconds: 15 43 | periodSeconds: 20 44 | readinessProbe: 45 | httpGet: 46 | path: /readyz 47 | port: 8081 48 | initialDelaySeconds: 5 49 | periodSeconds: 10 50 | # TODO(user): Configure the resources accordingly based on the project requirements. 51 | # More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ 52 | resources: 53 | limits: 54 | cpu: 500m 55 | memory: 128Mi 56 | requests: 57 | cpu: 10m 58 | memory: 64Mi 59 | serviceAccountName: controller-manager 60 | terminationGracePeriodSeconds: 10 61 | -------------------------------------------------------------------------------- /config/prometheus/kustomization.yaml: -------------------------------------------------------------------------------- 1 | resources: 2 | - monitor.yaml 3 | -------------------------------------------------------------------------------- /config/prometheus/monitor.yaml: -------------------------------------------------------------------------------- 1 | 2 | # Prometheus Monitor Service (Metrics) 3 | apiVersion: monitoring.coreos.com/v1 4 | kind: ServiceMonitor 5 | metadata: 6 | labels: 7 | control-plane: controller-manager 8 | name: controller-manager-metrics-monitor 9 | namespace: system 10 | spec: 11 | endpoints: 12 | - path: /metrics 13 | port: https 14 | scheme: https 15 | bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token 16 | tlsConfig: 17 | insecureSkipVerify: true 18 | selector: 19 | matchLabels: 20 | control-plane: controller-manager 21 | -------------------------------------------------------------------------------- /config/rbac/auth_proxy_client_clusterrole.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: rbac.authorization.k8s.io/v1 2 | kind: ClusterRole 3 | metadata: 4 | name: metrics-reader 5 | rules: 6 | - nonResourceURLs: 7 | - "/metrics" 8 | verbs: 9 | - get 10 | -------------------------------------------------------------------------------- /config/rbac/auth_proxy_role.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: rbac.authorization.k8s.io/v1 2 | kind: ClusterRole 3 | metadata: 4 | name: proxy-role 5 | rules: 6 | - apiGroups: 7 | - authentication.k8s.io 8 | resources: 9 | - tokenreviews 10 | verbs: 11 | - create 12 | - apiGroups: 13 | - authorization.k8s.io 14 | resources: 15 | - subjectaccessreviews 16 | verbs: 17 | - create 18 | -------------------------------------------------------------------------------- /config/rbac/auth_proxy_role_binding.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: rbac.authorization.k8s.io/v1 2 | kind: ClusterRoleBinding 3 | metadata: 4 | name: proxy-rolebinding 5 | roleRef: 6 | apiGroup: rbac.authorization.k8s.io 7 | kind: ClusterRole 8 | name: proxy-role 9 | subjects: 10 | - kind: ServiceAccount 11 | name: controller-manager 12 | namespace: system 13 | -------------------------------------------------------------------------------- /config/rbac/auth_proxy_service.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Service 3 | metadata: 4 | labels: 5 | control-plane: controller-manager 6 | name: controller-manager-metrics-service 7 | namespace: system 8 | spec: 9 | ports: 10 | - name: https 11 | port: 8443 12 | protocol: TCP 13 | targetPort: https 14 | selector: 15 | control-plane: controller-manager 16 | -------------------------------------------------------------------------------- /config/rbac/kustomization.yaml: -------------------------------------------------------------------------------- 1 | resources: 2 | # All RBAC will be applied under this service account in 3 | # the deployment namespace. You may comment out this resource 4 | # if your manager will use a service account that exists at 5 | # runtime. Be sure to update RoleBinding and ClusterRoleBinding 6 | # subjects if changing service account names. 7 | - service_account.yaml 8 | - role.yaml 9 | - role_binding.yaml 10 | - leader_election_role.yaml 11 | - leader_election_role_binding.yaml 12 | # Comment the following 4 lines if you want to disable 13 | # the auth proxy (https://github.com/brancz/kube-rbac-proxy) 14 | # which protects your /metrics endpoint. 15 | - auth_proxy_service.yaml 16 | - auth_proxy_role.yaml 17 | - auth_proxy_role_binding.yaml 18 | - auth_proxy_client_clusterrole.yaml 19 | -------------------------------------------------------------------------------- /config/rbac/leader_election_role.yaml: -------------------------------------------------------------------------------- 1 | # permissions to do leader election. 2 | apiVersion: rbac.authorization.k8s.io/v1 3 | kind: Role 4 | metadata: 5 | name: leader-election-role 6 | rules: 7 | - apiGroups: 8 | - "" 9 | resources: 10 | - configmaps 11 | verbs: 12 | - get 13 | - list 14 | - watch 15 | - create 16 | - update 17 | - patch 18 | - delete 19 | - apiGroups: 20 | - coordination.k8s.io 21 | resources: 22 | - leases 23 | verbs: 24 | - get 25 | - list 26 | - watch 27 | - create 28 | - update 29 | - patch 30 | - delete 31 | - apiGroups: 32 | - "" 33 | resources: 34 | - events 35 | verbs: 36 | - create 37 | - patch 38 | -------------------------------------------------------------------------------- /config/rbac/leader_election_role_binding.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: rbac.authorization.k8s.io/v1 2 | kind: RoleBinding 3 | metadata: 4 | name: leader-election-rolebinding 5 | roleRef: 6 | apiGroup: rbac.authorization.k8s.io 7 | kind: Role 8 | name: leader-election-role 9 | subjects: 10 | - kind: ServiceAccount 11 | name: controller-manager 12 | namespace: system 13 | -------------------------------------------------------------------------------- /config/rbac/msjob_editor_role.yaml: -------------------------------------------------------------------------------- 1 | # permissions for end users to edit msjobs. 2 | apiVersion: rbac.authorization.k8s.io/v1 3 | kind: ClusterRole 4 | metadata: 5 | name: msjob-editor-role 6 | rules: 7 | - apiGroups: 8 | - mindspore.gitee.com 9 | resources: 10 | - msjobs 11 | verbs: 12 | - create 13 | - delete 14 | - get 15 | - list 16 | - patch 17 | - update 18 | - watch 19 | - apiGroups: 20 | - mindspore.gitee.com 21 | resources: 22 | - msjobs/status 23 | verbs: 24 | - get 25 | -------------------------------------------------------------------------------- /config/rbac/msjob_viewer_role.yaml: -------------------------------------------------------------------------------- 1 | # permissions for end users to view msjobs. 2 | apiVersion: rbac.authorization.k8s.io/v1 3 | kind: ClusterRole 4 | metadata: 5 | name: msjob-viewer-role 6 | rules: 7 | - apiGroups: 8 | - mindspore.gitee.com 9 | resources: 10 | - msjobs 11 | verbs: 12 | - get 13 | - list 14 | - watch 15 | - apiGroups: 16 | - mindspore.gitee.com 17 | resources: 18 | - msjobs/status 19 | verbs: 20 | - get 21 | -------------------------------------------------------------------------------- /config/rbac/role.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: rbac.authorization.k8s.io/v1 3 | kind: ClusterRole 4 | metadata: 5 | creationTimestamp: null 6 | name: manager-role 7 | rules: 8 | - apiGroups: 9 | - "" 10 | resources: 11 | - pods 12 | verbs: 13 | - create 14 | - delete 15 | - get 16 | - list 17 | - patch 18 | - update 19 | - watch 20 | - apiGroups: 21 | - "" 22 | resources: 23 | - services 24 | verbs: 25 | - create 26 | - delete 27 | - get 28 | - list 29 | - watch 30 | - apiGroups: 31 | - mindspore.gitee.com 32 | resources: 33 | - msjobs 34 | verbs: 35 | - create 36 | - delete 37 | - get 38 | - list 39 | - patch 40 | - update 41 | - watch 42 | - apiGroups: 43 | - mindspore.gitee.com 44 | resources: 45 | - msjobs/finalizers 46 | verbs: 47 | - update 48 | - apiGroups: 49 | - mindspore.gitee.com 50 | resources: 51 | - msjobs/status 52 | verbs: 53 | - get 54 | - patch 55 | - update 56 | -------------------------------------------------------------------------------- /config/rbac/role_binding.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: rbac.authorization.k8s.io/v1 2 | kind: ClusterRoleBinding 3 | metadata: 4 | name: manager-rolebinding 5 | roleRef: 6 | apiGroup: rbac.authorization.k8s.io 7 | kind: ClusterRole 8 | name: manager-role 9 | subjects: 10 | - kind: ServiceAccount 11 | name: controller-manager 12 | namespace: system 13 | -------------------------------------------------------------------------------- /config/rbac/service_account.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: ServiceAccount 3 | metadata: 4 | name: controller-manager 5 | namespace: system 6 | -------------------------------------------------------------------------------- /config/samples/ms_wide_deep_dataparallel.yaml: -------------------------------------------------------------------------------- 1 | # wide&deep dataparallel mode with scheduler and workers 2 | # network : https://gitee.com/mindspore/models/tree/master/official/recommend/wide_and_deep 3 | apiVersion: mindspore.gitee.com/v1 4 | kind: MSJob 5 | metadata: 6 | name: ms-widedeep-dataparallel 7 | spec: 8 | runPolicy: 9 | cleanPodPolicy: None 10 | successPolicy: AllWorkers 11 | msReplicaSpecs: 12 | Scheduler: 13 | replicas: 1 14 | restartPolicy: Never 15 | template: 16 | spec: 17 | volumes: 18 | - name: script-data 19 | hostPath: 20 | path: /home/fzh/wide_and_deep/ 21 | containers: 22 | - name: mindspore 23 | image: swr.cn-south-1.myhuaweicloud.com/mindspore-ci/mindspore-gpu:1.7.0-20220327121541 24 | imagePullPolicy: IfNotPresent 25 | command: 26 | - /bin/bash 27 | - -c 28 | - python -s /home/fzh/wide_and_deep/train_and_eval_distribute.py --device_target="GPU" --epochs=1 --data_path=/home/fzh/wide_and_deep/criteo_mindrecord --batch_size=16000 29 | volumeMounts: 30 | - mountPath: /home/fzh/wide_and_deep/ 31 | name: script-data 32 | env: 33 | - name: GLOG_v 34 | value: "1" 35 | Worker: 36 | replicas: 4 37 | restartPolicy: Never 38 | template: 39 | spec: 40 | volumes: 41 | - name: script-data 42 | hostPath: 43 | path: /home/fzh/wide_and_deep/ 44 | containers: 45 | - name: mindspore 46 | image: swr.cn-south-1.myhuaweicloud.com/mindspore-ci/mindspore-gpu:1.7.0-20220327121541 47 | imagePullPolicy: IfNotPresent 48 | command: 49 | - /bin/bash 50 | - -c 51 | - python -s /home/fzh/wide_and_deep/train_and_eval_distribute.py --device_target="GPU" --epochs=1 --data_path=/home/fzh/wide_and_deep/criteo_mindrecord --batch_size=16000 52 | volumeMounts: 53 | - mountPath: /home/fzh/wide_and_deep/ 54 | name: script-data 55 | env: 56 | - name: GLOG_v 57 | value: "1" 58 | resources: 59 | limits: 60 | nvidia.com/gpu: 1 -------------------------------------------------------------------------------- /config/samples/ms_wide_deep_ps_distribute.yaml: -------------------------------------------------------------------------------- 1 | # wide&deep ps mode distribute training. 2 | # network : https://gitee.com/mindspore/models/tree/master/official/recommend/wide_and_deep 3 | apiVersion: mindspore.gitee.com/v1 4 | kind: MSJob 5 | metadata: 6 | name: ms-widedeep-ps-distribute 7 | spec: 8 | runPolicy: 9 | cleanPodPolicy: None 10 | successPolicy: AllWorkers 11 | msReplicaSpecs: 12 | Scheduler: 13 | replicas: 1 14 | restartPolicy: Never 15 | template: 16 | spec: 17 | volumes: 18 | - name: script-data 19 | hostPath: 20 | path: /home/fzh/wide_and_deep/ 21 | containers: 22 | - name: mindspore 23 | image: swr.cn-south-1.myhuaweicloud.com/mindspore-ci/mindspore-gpu:1.7.0-20220327121541 24 | imagePullPolicy: IfNotPresent 25 | command: 26 | - /bin/bash 27 | - -c 28 | - python -s /home/fzh/wide_and_deep/train_and_eval_parameter_server_distribute.py --device_target="GPU" --epochs=1 --data_path=/home/fzh/wide_and_deep/criteo_mindrecord --parameter_server=1 --vocab_cache_size=300000 29 | volumeMounts: 30 | - mountPath: /home/fzh/wide_and_deep/ 31 | name: script-data 32 | env: 33 | - name: GLOG_v 34 | value: "1" 35 | PS: 36 | replicas: 2 37 | restartPolicy: Never 38 | template: 39 | spec: 40 | volumes: 41 | - name: script-data 42 | hostPath: 43 | path: /home/fzh/wide_and_deep/ 44 | containers: 45 | - name: mindspore 46 | image: swr.cn-south-1.myhuaweicloud.com/mindspore-ci/mindspore-gpu:1.7.0-20220327121541 47 | imagePullPolicy: IfNotPresent 48 | command: 49 | - /bin/bash 50 | - -c 51 | - python -s /home/fzh/wide_and_deep/train_and_eval_parameter_server_distribute.py --device_target="GPU" --epochs=1 --data_path=/home/fzh/wide_and_deep/criteo_mindrecord --parameter_server=1 --vocab_cache_size=300000 52 | volumeMounts: 53 | - mountPath: /home/fzh/wide_and_deep/ 54 | name: script-data 55 | Worker: 56 | replicas: 4 57 | restartPolicy: Never 58 | template: 59 | spec: 60 | volumes: 61 | - name: script-data 62 | hostPath: 63 | path: /home/fzh/wide_and_deep/ 64 | containers: 65 | - name: mindspore 66 | image: swr.cn-south-1.myhuaweicloud.com/mindspore-ci/mindspore-gpu:1.7.0-20220327121541 67 | imagePullPolicy: IfNotPresent 68 | command: 69 | - /bin/bash 70 | - -c 71 | - python -s /home/fzh/wide_and_deep/train_and_eval_parameter_server_distribute.py --device_target="GPU" --epochs=1 --data_path=/home/fzh/wide_and_deep/criteo_mindrecord --parameter_server=1 --vocab_cache_size=300000 --sparse=False 72 | volumeMounts: 73 | - mountPath: /home/fzh/wide_and_deep/ 74 | name: script-data 75 | resources: 76 | limits: 77 | nvidia.com/gpu: 1 78 | -------------------------------------------------------------------------------- /config/samples/ms_wide_deep_ps_standalone.yaml: -------------------------------------------------------------------------------- 1 | # wide&deep for ps mode which only has one worker training. 2 | # network : https://gitee.com/mindspore/models/tree/master/official/recommend/wide_and_deep 3 | apiVersion: mindspore.gitee.com/v1 4 | kind: MSJob 5 | metadata: 6 | name: ms-widedeep-ps-standalone 7 | spec: 8 | runPolicy: 9 | cleanPodPolicy: None 10 | successPolicy: AllWorkers 11 | msReplicaSpecs: 12 | Scheduler: 13 | replicas: 1 14 | restartPolicy: Never 15 | template: 16 | spec: 17 | volumes: 18 | - name: script-data 19 | hostPath: 20 | path: /home/fzh/wide_and_deep/ 21 | containers: 22 | - name: mindspore 23 | image: swr.cn-south-1.myhuaweicloud.com/mindspore-ci/mindspore-gpu:1.7.0-20220327121541 24 | imagePullPolicy: IfNotPresent 25 | command: 26 | - /bin/bash 27 | - -c 28 | - python -s /home/fzh/wide_and_deep/train_and_eval_parameter_server_standalone.py --device_target="GPU" --epochs=1 --data_path=/home/fzh/wide_and_deep/criteo_mindrecord --parameter_server=1 --vocab_cache_size=200000 29 | volumeMounts: 30 | - mountPath: /home/fzh/wide_and_deep/ 31 | name: script-data 32 | env: 33 | - name: GLOG_v 34 | value: "1" 35 | PS: 36 | replicas: 2 37 | restartPolicy: Never 38 | template: 39 | spec: 40 | volumes: 41 | - name: script-data 42 | hostPath: 43 | path: /home/fzh/wide_and_deep/ 44 | containers: 45 | - name: mindspore 46 | image: swr.cn-south-1.myhuaweicloud.com/mindspore-ci/mindspore-gpu:1.7.0-20220327121541 47 | imagePullPolicy: IfNotPresent 48 | command: 49 | - /bin/bash 50 | - -c 51 | - python -s /home/fzh/wide_and_deep/train_and_eval_parameter_server_standalone.py --device_target="GPU" --epochs=1 --data_path=/home/fzh/wide_and_deep/criteo_mindrecord --parameter_server=1 --vocab_cache_size=200000 52 | volumeMounts: 53 | - mountPath: /home/fzh/wide_and_deep/ 54 | name: script-data 55 | Worker: 56 | replicas: 1 57 | restartPolicy: Never 58 | template: 59 | spec: 60 | volumes: 61 | - name: script-data 62 | hostPath: 63 | path: /home/fzh/wide_and_deep/ 64 | containers: 65 | - name: mindspore 66 | image: swr.cn-south-1.myhuaweicloud.com/mindspore-ci/mindspore-gpu:1.7.0-20220327121541 67 | imagePullPolicy: IfNotPresent 68 | command: 69 | - /bin/bash 70 | - -c 71 | - python -s /home/fzh/wide_and_deep/train_and_eval_parameter_server_standalone.py --device_target="GPU" --epochs=1 --data_path=/home/fzh/wide_and_deep/criteo_mindrecord --parameter_server=1 --vocab_cache_size=200000 72 | volumeMounts: 73 | - mountPath: /home/fzh/wide_and_deep/ 74 | name: script-data 75 | -------------------------------------------------------------------------------- /config/samples/ms_wide_deep_standalone.yaml: -------------------------------------------------------------------------------- 1 | # wide&deep for only one worker training, just like in one machine which has 1 gpu. 2 | # network : https://gitee.com/mindspore/models/tree/master/official/recommend/wide_and_deep 3 | apiVersion: mindspore.gitee.com/v1 4 | kind: MSJob 5 | metadata: 6 | name: ms-widedeep-standalone 7 | spec: 8 | msReplicaSpecs: 9 | Worker: 10 | replicas: 1 11 | restartPolicy: Never 12 | template: 13 | spec: 14 | volumes: 15 | - name: script-data 16 | hostPath: 17 | path: /home/fzh/wide_and_deep/ 18 | containers: 19 | - name: mindspore 20 | image: swr.cn-south-1.myhuaweicloud.com/mindspore-ci/mindspore-gpu:1.7.0-20220327121541 21 | imagePullPolicy: IfNotPresent 22 | command: 23 | - /bin/bash 24 | - -c 25 | - python -s /home/fzh/wide_and_deep/train_and_eval.py --device_target="GPU" --data_path=/home/fzh/wide_and_deep/criteo_mindrecord --batch_size=16000 --epochs=1 26 | volumeMounts: 27 | - mountPath: /home/fzh/wide_and_deep/ 28 | name: script-data -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module ms-operator 2 | 3 | go 1.17 4 | 5 | require ( 6 | github.com/go-logr/logr v0.3.0 7 | github.com/kubeflow/common v0.4.1 8 | github.com/kubeflow/training-operator v1.4.0 9 | github.com/onsi/ginkgo v1.14.1 10 | github.com/onsi/gomega v1.10.2 11 | github.com/prometheus/client_golang v1.10.0 12 | github.com/prometheus/common v0.18.0 13 | github.com/sirupsen/logrus v1.6.0 14 | k8s.io/api v0.19.9 15 | k8s.io/apimachinery v0.19.9 16 | k8s.io/client-go v0.19.9 17 | sigs.k8s.io/controller-runtime v0.7.2 18 | volcano.sh/apis v1.2.0-k8s1.19.6 19 | ) 20 | 21 | require ( 22 | cloud.google.com/go v0.51.0 // indirect 23 | github.com/Azure/go-autorest/autorest v0.9.6 // indirect 24 | github.com/Azure/go-autorest/autorest/adal v0.8.2 // indirect 25 | github.com/Azure/go-autorest/autorest/date v0.2.0 // indirect 26 | github.com/Azure/go-autorest/logger v0.1.0 // indirect 27 | github.com/Azure/go-autorest/tracing v0.5.0 // indirect 28 | github.com/PuerkitoBio/purell v1.1.1 // indirect 29 | github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 // indirect 30 | github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 // indirect 31 | github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d // indirect 32 | github.com/beorn7/perks v1.0.1 // indirect 33 | github.com/cespare/xxhash/v2 v2.1.1 // indirect 34 | github.com/davecgh/go-spew v1.1.1 // indirect 35 | github.com/dgrijalva/jwt-go v3.2.0+incompatible // indirect 36 | github.com/emicklei/go-restful v2.9.5+incompatible // indirect 37 | github.com/evanphx/json-patch v4.9.0+incompatible // indirect 38 | github.com/fsnotify/fsnotify v1.4.9 // indirect 39 | github.com/go-logr/zapr v0.2.0 // indirect 40 | github.com/go-openapi/jsonpointer v0.19.5 // indirect 41 | github.com/go-openapi/jsonreference v0.19.5 // indirect 42 | github.com/go-openapi/spec v0.20.3 // indirect 43 | github.com/go-openapi/swag v0.19.14 // indirect 44 | github.com/gogo/protobuf v1.3.2 // indirect 45 | github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7 // indirect 46 | github.com/golang/protobuf v1.4.3 // indirect 47 | github.com/google/go-cmp v0.5.4 // indirect 48 | github.com/google/gofuzz v1.1.0 // indirect 49 | github.com/google/uuid v1.1.1 // indirect 50 | github.com/googleapis/gnostic v0.5.1 // indirect 51 | github.com/hashicorp/golang-lru v0.5.4 // indirect 52 | github.com/imdario/mergo v0.3.10 // indirect 53 | github.com/josharian/intern v1.0.0 // indirect 54 | github.com/json-iterator/go v1.1.10 // indirect 55 | github.com/konsorten/go-windows-terminal-sequences v1.0.3 // indirect 56 | github.com/mailru/easyjson v0.7.6 // indirect 57 | github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 // indirect 58 | github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect 59 | github.com/modern-go/reflect2 v1.0.1 // indirect 60 | github.com/nxadm/tail v1.4.4 // indirect 61 | github.com/pkg/errors v0.9.1 // indirect 62 | github.com/prometheus/client_model v0.2.0 // indirect 63 | github.com/prometheus/procfs v0.6.0 // indirect 64 | github.com/spf13/pflag v1.0.5 // indirect 65 | go.uber.org/atomic v1.6.0 // indirect 66 | go.uber.org/multierr v1.5.0 // indirect 67 | go.uber.org/zap v1.15.0 // indirect 68 | golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 // indirect 69 | golang.org/x/net v0.0.0-20220225172249-27dd8689420f // indirect 70 | golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6 // indirect 71 | golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e // indirect 72 | golang.org/x/text v0.3.7 // indirect 73 | golang.org/x/time v0.0.0-20200630173020-3af7569d3a1e // indirect 74 | golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect 75 | gomodules.xyz/jsonpatch/v2 v2.1.0 // indirect 76 | google.golang.org/appengine v1.6.6 // indirect 77 | google.golang.org/protobuf v1.24.0 // indirect 78 | gopkg.in/alecthomas/kingpin.v2 v2.2.6 // indirect 79 | gopkg.in/inf.v0 v0.9.1 // indirect 80 | gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect 81 | gopkg.in/yaml.v2 v2.4.0 // indirect 82 | gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776 // indirect 83 | honnef.co/go/tools v0.2.2 // indirect 84 | k8s.io/apiextensions-apiserver v0.19.2 // indirect 85 | k8s.io/code-generator v0.19.9 // indirect 86 | k8s.io/component-base v0.19.9 // indirect 87 | k8s.io/klog/v2 v2.2.0 // indirect 88 | k8s.io/kube-openapi v0.0.0-20200805222855-6aeccd4b50c6 // indirect 89 | k8s.io/utils v0.0.0-20200912215256-4140de9c8800 // indirect 90 | sigs.k8s.io/structured-merge-diff/v4 v4.0.1 // indirect 91 | sigs.k8s.io/yaml v1.2.0 // indirect 92 | ) 93 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= 2 | cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= 3 | cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= 4 | cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU= 5 | cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= 6 | cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc= 7 | cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0= 8 | cloud.google.com/go v0.51.0 h1:PvKAVQWCtlGUSlZkGW3QLelKaWq7KYv/MW1EboG8bfM= 9 | cloud.google.com/go v0.51.0/go.mod h1:hWtGJ6gnXH+KgDv+V0zFGDvpi07n3z8ZNj3T1RW0Gcw= 10 | cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= 11 | cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= 12 | cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= 13 | cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= 14 | dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= 15 | github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8= 16 | github.com/Azure/go-autorest/autorest v0.9.0/go.mod h1:xyHB1BMZT0cuDHU7I0+g046+BFDTQ8rEZB0s4Yfa6bI= 17 | github.com/Azure/go-autorest/autorest v0.9.6 h1:5YWtOnckcudzIw8lPPBcWOnmIFWMtHci1ZWAZulMSx0= 18 | github.com/Azure/go-autorest/autorest v0.9.6/go.mod h1:/FALq9T/kS7b5J5qsQ+RSTUdAmGFqi0vUdVNNx8q630= 19 | github.com/Azure/go-autorest/autorest/adal v0.5.0/go.mod h1:8Z9fGy2MpX0PvDjB1pEgQTmVqjGhiHBW7RJJEciWzS0= 20 | github.com/Azure/go-autorest/autorest/adal v0.8.2 h1:O1X4oexUxnZCaEUGsvMnr8ZGj8HI37tNezwY4npRqA0= 21 | github.com/Azure/go-autorest/autorest/adal v0.8.2/go.mod h1:ZjhuQClTqx435SRJ2iMlOxPYt3d2C/T/7TiQCVZSn3Q= 22 | github.com/Azure/go-autorest/autorest/date v0.1.0/go.mod h1:plvfp3oPSKwf2DNjlBjWF/7vwR+cUD/ELuzDCXwHUVA= 23 | github.com/Azure/go-autorest/autorest/date v0.2.0 h1:yW+Zlqf26583pE43KhfnhFcdmSWlm5Ew6bxipnr/tbM= 24 | github.com/Azure/go-autorest/autorest/date v0.2.0/go.mod h1:vcORJHLJEh643/Ioh9+vPmf1Ij9AEBM5FuBIXLmIy0g= 25 | github.com/Azure/go-autorest/autorest/mocks v0.1.0/go.mod h1:OTyCOPRA2IgIlWxVYxBee2F5Gr4kF2zd2J5cFRaIDN0= 26 | github.com/Azure/go-autorest/autorest/mocks v0.2.0/go.mod h1:OTyCOPRA2IgIlWxVYxBee2F5Gr4kF2zd2J5cFRaIDN0= 27 | github.com/Azure/go-autorest/autorest/mocks v0.3.0 h1:qJumjCaCudz+OcqE9/XtEPfvtOjOmKaui4EOpFI6zZc= 28 | github.com/Azure/go-autorest/autorest/mocks v0.3.0/go.mod h1:a8FDP3DYzQ4RYfVAxAN3SVSiiO77gL2j2ronKKP0syM= 29 | github.com/Azure/go-autorest/logger v0.1.0 h1:ruG4BSDXONFRrZZJ2GUXDiUyVpayPmb1GnWeHDdaNKY= 30 | github.com/Azure/go-autorest/logger v0.1.0/go.mod h1:oExouG+K6PryycPJfVSxi/koC6LSNgds39diKLz7Vrc= 31 | github.com/Azure/go-autorest/tracing v0.5.0 h1:TRn4WjSnkcSy5AEG3pnbtFSwNtwzjr4VYyQflFE619k= 32 | github.com/Azure/go-autorest/tracing v0.5.0/go.mod h1:r/s2XiOKccPW3HrqB+W0TQzfbtp2fGCgRFtBroKn4Dk= 33 | github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= 34 | github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= 35 | github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= 36 | github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible/go.mod h1:r7JcOSlj0wfOMncg0iLm8Leh48TZaKVeNIfJntJ2wa0= 37 | github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ= 38 | github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= 39 | github.com/PuerkitoBio/purell v1.0.0/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= 40 | github.com/PuerkitoBio/purell v1.1.0/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= 41 | github.com/PuerkitoBio/purell v1.1.1 h1:WEQqlqaGbrPkxLJWfBwQmfEAE1Z7ONdDLqrN38tNFfI= 42 | github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= 43 | github.com/PuerkitoBio/urlesc v0.0.0-20160726150825-5bd2802263f2/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= 44 | github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 h1:d+Bc7a5rLufV/sSk/8dngufqelfh6jnri85riMAaF/M= 45 | github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= 46 | github.com/Shopify/sarama v1.19.0/go.mod h1:FVkBWblsNy7DGZRfXLU0O9RCGt5g3g3yEuWXgklEdEo= 47 | github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI= 48 | github.com/VividCortex/gohistogram v1.0.0/go.mod h1:Pf5mBqqDxYaXu3hDrrU+w6nw50o/4+TcAqDqk/vUH7g= 49 | github.com/afex/hystrix-go v0.0.0-20180502004556-fa1af6a1f4f5/go.mod h1:SkGFH1ia65gfNATL8TAiHDNxPzPdmEL5uirI2Uyuz6c= 50 | github.com/agnivade/levenshtein v1.0.1/go.mod h1:CURSv5d9Uaml+FovSIICkLbAUZ9S4RqaHDIsdSBg7lM= 51 | github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= 52 | github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 h1:JYp7IbQjafoB+tBA3gMyHYHrpOtNuDiK/uB5uXxq5wM= 53 | github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= 54 | github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= 55 | github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= 56 | github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d h1:UQZhZ2O0vMHr2cI+DC1Mbh0TJxzA3RcLoMsFw+aXw7E= 57 | github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= 58 | github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883/go.mod h1:rCTlJbsFo29Kk6CurOXKm700vrz8f0KW0JNfpkRJY/8= 59 | github.com/apache/thrift v0.12.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= 60 | github.com/apache/thrift v0.13.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= 61 | github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= 62 | github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= 63 | github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= 64 | github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= 65 | github.com/aryann/difflib v0.0.0-20170710044230-e206f873d14a/go.mod h1:DAHtR1m6lCRdSC2Tm3DSWRPvIPr6xNKyeHdqDQSQT+A= 66 | github.com/asaskevich/govalidator v0.0.0-20180720115003-f9ffefc3facf/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= 67 | github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= 68 | github.com/aws/aws-lambda-go v1.13.3/go.mod h1:4UKl9IzQMoD+QF79YdCuzCwp8VbmG4VAQwij/eHl5CU= 69 | github.com/aws/aws-sdk-go v1.27.0/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= 70 | github.com/aws/aws-sdk-go-v2 v0.18.0/go.mod h1:JWVYvqSMppoMJC0x5wdwiImzgXTI9FuZwxzkQq9wy+g= 71 | github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= 72 | github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= 73 | github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= 74 | github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= 75 | github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= 76 | github.com/blang/semver v3.5.0+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk= 77 | github.com/casbin/casbin/v2 v2.1.2/go.mod h1:YcPU1XXisHhLzuxH9coDNf2FbKpjGlbCg3n9yuLkIJQ= 78 | github.com/cenkalti/backoff v2.2.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM= 79 | github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= 80 | github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= 81 | github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= 82 | github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY= 83 | github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= 84 | github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= 85 | github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= 86 | github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= 87 | github.com/clbanning/x2j v0.0.0-20191024224557-825249438eec/go.mod h1:jMjuTZXRI4dUb/I5gc9Hdhagfvm9+RyrPryS/auMzxE= 88 | github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= 89 | github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8= 90 | github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI= 91 | github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= 92 | github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= 93 | github.com/coreos/go-oidc v2.1.0+incompatible/go.mod h1:CgnwVTmzoESiwO9qyAFEMiHoZ1nMCKZlZ9V6mm3/LKc= 94 | github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= 95 | github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= 96 | github.com/coreos/go-systemd v0.0.0-20180511133405-39ca1b05acc7/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= 97 | github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= 98 | github.com/coreos/pkg v0.0.0-20160727233714-3ac0863d7acf/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= 99 | github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= 100 | github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= 101 | github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= 102 | github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= 103 | github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= 104 | github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 105 | github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= 106 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 107 | github.com/dgrijalva/jwt-go v3.2.0+incompatible h1:7qlOGliEKZXTDg6OTjfoBKDXWrumCAMpl/TFQ4/5kLM= 108 | github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= 109 | github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= 110 | github.com/docker/go-units v0.3.3/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= 111 | github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= 112 | github.com/docker/spdystream v0.0.0-20160310174837-449fdfce4d96/go.mod h1:Qh8CwZgvJUkLughtfhJv5dyTYa91l1fOUCrgjqmcifM= 113 | github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE= 114 | github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= 115 | github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= 116 | github.com/eapache/go-resiliency v1.1.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5mFgVsvEsIPBvNs= 117 | github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1:+020luEh2TKB4/GOp8oxxtq0Daoen/Cii55CzbTV6DU= 118 | github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I= 119 | github.com/edsrzf/mmap-go v1.0.0/go.mod h1:YO35OhQPt3KJa3ryjFM5Bs14WD66h8eGKpfaBNrHW5M= 120 | github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc= 121 | github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= 122 | github.com/emicklei/go-restful v2.9.5+incompatible h1:spTtZBk5DYEvbxMVutUuTyh1Ao2r4iyvLdACqsl/Ljk= 123 | github.com/emicklei/go-restful v2.9.5+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= 124 | github.com/envoyproxy/go-control-plane v0.6.9/go.mod h1:SBwIajubJHhxtWwsL9s8ss4safvEdbitLhGGK48rN6g= 125 | github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= 126 | github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= 127 | github.com/evanphx/json-patch v4.5.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= 128 | github.com/evanphx/json-patch v4.9.0+incompatible h1:kLcOMZeuLAJvL2BPWLMIj5oaZQobrkAqrL+WFZwQses= 129 | github.com/evanphx/json-patch v4.9.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= 130 | github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= 131 | github.com/franela/goblin v0.0.0-20200105215937-c9ffbefa60db/go.mod h1:7dvUGVsVBjqR7JHJk0brhHOZYGmfBYOrK0ZhYMEtBr4= 132 | github.com/franela/goreq v0.0.0-20171204163338-bcd34c9993f8/go.mod h1:ZhphrRTfi2rbfLwlschooIH4+wKKDR4Pdxhh+TRoA20= 133 | github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= 134 | github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4= 135 | github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= 136 | github.com/ghodss/yaml v0.0.0-20150909031657-73d445a93680/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= 137 | github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= 138 | github.com/globalsign/mgo v0.0.0-20180905125535-1ca0a4f7cbcb/go.mod h1:xkRDCp4j0OGD1HRkm4kmhM+pmpv3AKq5SU7GMg4oO/Q= 139 | github.com/globalsign/mgo v0.0.0-20181015135952-eeefdecb41b8/go.mod h1:xkRDCp4j0OGD1HRkm4kmhM+pmpv3AKq5SU7GMg4oO/Q= 140 | github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= 141 | github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= 142 | github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= 143 | github.com/go-kit/kit v0.10.0/go.mod h1:xUsJbQ/Fp4kEt7AFgCuvyX4a71u8h9jB8tj/ORgOZ7o= 144 | github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= 145 | github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= 146 | github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= 147 | github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas= 148 | github.com/go-logr/logr v0.2.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= 149 | github.com/go-logr/logr v0.3.0 h1:q4c+kbcR0d5rSurhBR8dIgieOaYpXtsdTYfx22Cu6rs= 150 | github.com/go-logr/logr v0.3.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= 151 | github.com/go-logr/zapr v0.2.0 h1:v6Ji8yBW77pva6NkJKQdHLAJKrIJKRHz0RXwPqCHSR4= 152 | github.com/go-logr/zapr v0.2.0/go.mod h1:qhKdvif7YF5GI9NWEpyxTSSBdGmzkNguibrdCNVPunU= 153 | github.com/go-openapi/analysis v0.0.0-20180825180245-b006789cd277/go.mod h1:k70tL6pCuVxPJOHXQ+wIac1FUrvNkHolPie/cLEU6hI= 154 | github.com/go-openapi/analysis v0.17.0/go.mod h1:IowGgpVeD0vNm45So8nr+IcQ3pxVtpRoBWb8PVZO0ik= 155 | github.com/go-openapi/analysis v0.18.0/go.mod h1:IowGgpVeD0vNm45So8nr+IcQ3pxVtpRoBWb8PVZO0ik= 156 | github.com/go-openapi/analysis v0.19.2/go.mod h1:3P1osvZa9jKjb8ed2TPng3f0i/UY9snX6gxi44djMjk= 157 | github.com/go-openapi/analysis v0.19.5/go.mod h1:hkEAkxagaIvIP7VTn8ygJNkd4kAYON2rCu0v0ObL0AU= 158 | github.com/go-openapi/errors v0.17.0/go.mod h1:LcZQpmvG4wyF5j4IhA73wkLFQg+QJXOQHVjmcZxhka0= 159 | github.com/go-openapi/errors v0.18.0/go.mod h1:LcZQpmvG4wyF5j4IhA73wkLFQg+QJXOQHVjmcZxhka0= 160 | github.com/go-openapi/errors v0.19.2/go.mod h1:qX0BLWsyaKfvhluLejVpVNwNRdXZhEbTA4kxxpKBC94= 161 | github.com/go-openapi/jsonpointer v0.0.0-20160704185906-46af16f9f7b1/go.mod h1:+35s3my2LFTysnkMfxsJBAMHj/DoqoB9knIWoYG/Vk0= 162 | github.com/go-openapi/jsonpointer v0.17.0/go.mod h1:cOnomiV+CVVwFLk0A/MExoFMjwdsUdVpsRhURCKh+3M= 163 | github.com/go-openapi/jsonpointer v0.18.0/go.mod h1:cOnomiV+CVVwFLk0A/MExoFMjwdsUdVpsRhURCKh+3M= 164 | github.com/go-openapi/jsonpointer v0.19.2/go.mod h1:3akKfEdA7DF1sugOqz1dVQHBcuDBPKZGEoHC/NkiQRg= 165 | github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= 166 | github.com/go-openapi/jsonpointer v0.19.5 h1:gZr+CIYByUqjcgeLXnQu2gHYQC9o73G2XUeOFYEICuY= 167 | github.com/go-openapi/jsonpointer v0.19.5/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= 168 | github.com/go-openapi/jsonreference v0.0.0-20160704190145-13c6e3589ad9/go.mod h1:W3Z9FmVs9qj+KR4zFKmDPGiLdk1D9Rlm7cyMvf57TTg= 169 | github.com/go-openapi/jsonreference v0.17.0/go.mod h1:g4xxGn04lDIRh0GJb5QlpE3HfopLOL6uZrK/VgnsK9I= 170 | github.com/go-openapi/jsonreference v0.18.0/go.mod h1:g4xxGn04lDIRh0GJb5QlpE3HfopLOL6uZrK/VgnsK9I= 171 | github.com/go-openapi/jsonreference v0.19.2/go.mod h1:jMjeRr2HHw6nAVajTXJ4eiUwohSTlpa0o73RUL1owJc= 172 | github.com/go-openapi/jsonreference v0.19.3/go.mod h1:rjx6GuL8TTa9VaixXglHmQmIL98+wF9xc8zWvFonSJ8= 173 | github.com/go-openapi/jsonreference v0.19.5 h1:1WJP/wi4OjB4iV8KVbH73rQaoialJrqv8gitZLxGLtM= 174 | github.com/go-openapi/jsonreference v0.19.5/go.mod h1:RdybgQwPxbL4UEjuAruzK1x3nE69AqPYEJeo/TWfEeg= 175 | github.com/go-openapi/loads v0.17.0/go.mod h1:72tmFy5wsWx89uEVddd0RjRWPZm92WRLhf7AC+0+OOU= 176 | github.com/go-openapi/loads v0.18.0/go.mod h1:72tmFy5wsWx89uEVddd0RjRWPZm92WRLhf7AC+0+OOU= 177 | github.com/go-openapi/loads v0.19.0/go.mod h1:72tmFy5wsWx89uEVddd0RjRWPZm92WRLhf7AC+0+OOU= 178 | github.com/go-openapi/loads v0.19.2/go.mod h1:QAskZPMX5V0C2gvfkGZzJlINuP7Hx/4+ix5jWFxsNPs= 179 | github.com/go-openapi/loads v0.19.4/go.mod h1:zZVHonKd8DXyxyw4yfnVjPzBjIQcLt0CCsn0N0ZrQsk= 180 | github.com/go-openapi/runtime v0.0.0-20180920151709-4f900dc2ade9/go.mod h1:6v9a6LTXWQCdL8k1AO3cvqx5OtZY/Y9wKTgaoP6YRfA= 181 | github.com/go-openapi/runtime v0.19.0/go.mod h1:OwNfisksmmaZse4+gpV3Ne9AyMOlP1lt4sK4FXt0O64= 182 | github.com/go-openapi/runtime v0.19.4/go.mod h1:X277bwSUBxVlCYR3r7xgZZGKVvBd/29gLDlFGtJ8NL4= 183 | github.com/go-openapi/spec v0.0.0-20160808142527-6aced65f8501/go.mod h1:J8+jY1nAiCcj+friV/PDoE1/3eeccG9LYBs0tYvLOWc= 184 | github.com/go-openapi/spec v0.17.0/go.mod h1:XkF/MOi14NmjsfZ8VtAKf8pIlbZzyoTvZsdfssdxcBI= 185 | github.com/go-openapi/spec v0.18.0/go.mod h1:XkF/MOi14NmjsfZ8VtAKf8pIlbZzyoTvZsdfssdxcBI= 186 | github.com/go-openapi/spec v0.19.2/go.mod h1:sCxk3jxKgioEJikev4fgkNmwS+3kuYdJtcsZsD5zxMY= 187 | github.com/go-openapi/spec v0.19.3/go.mod h1:FpwSN1ksY1eteniUU7X0N/BgJ7a4WvBFVA8Lj9mJglo= 188 | github.com/go-openapi/spec v0.20.3 h1:uH9RQ6vdyPSs2pSy9fL8QPspDF2AMIMPtmK5coSSjtQ= 189 | github.com/go-openapi/spec v0.20.3/go.mod h1:gG4F8wdEDN+YPBMVnzE85Rbhf+Th2DTvA9nFPQ5AYEg= 190 | github.com/go-openapi/strfmt v0.17.0/go.mod h1:P82hnJI0CXkErkXi8IKjPbNBM6lV6+5pLP5l494TcyU= 191 | github.com/go-openapi/strfmt v0.18.0/go.mod h1:P82hnJI0CXkErkXi8IKjPbNBM6lV6+5pLP5l494TcyU= 192 | github.com/go-openapi/strfmt v0.19.0/go.mod h1:+uW+93UVvGGq2qGaZxdDeJqSAqBqBdl+ZPMF/cC8nDY= 193 | github.com/go-openapi/strfmt v0.19.3/go.mod h1:0yX7dbo8mKIvc3XSKp7MNfxw4JytCfCD6+bY1AVL9LU= 194 | github.com/go-openapi/swag v0.0.0-20160704191624-1d0bd113de87/go.mod h1:DXUve3Dpr1UfpPtxFw+EFuQ41HhCWZfha5jSVRG7C7I= 195 | github.com/go-openapi/swag v0.17.0/go.mod h1:AByQ+nYG6gQg71GINrmuDXCPWdL640yX49/kXLo40Tg= 196 | github.com/go-openapi/swag v0.18.0/go.mod h1:AByQ+nYG6gQg71GINrmuDXCPWdL640yX49/kXLo40Tg= 197 | github.com/go-openapi/swag v0.19.2/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= 198 | github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= 199 | github.com/go-openapi/swag v0.19.14 h1:gm3vOOXfiuw5i9p5N9xJvfjvuofpyvLA9Wr6QfK5Fng= 200 | github.com/go-openapi/swag v0.19.14/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ= 201 | github.com/go-openapi/validate v0.18.0/go.mod h1:Uh4HdOzKt19xGIGm1qHf/ofbX1YQ4Y+MYsct2VUrAJ4= 202 | github.com/go-openapi/validate v0.19.2/go.mod h1:1tRCw7m3jtI8eNWEEliiAqUIcBztB2KDnRCRMUi7GTA= 203 | github.com/go-openapi/validate v0.19.5/go.mod h1:8DJv2CVJQ6kGNpFW6eV9N3JviE1C85nY1c2z52x1Gk4= 204 | github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= 205 | github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= 206 | github.com/gogo/googleapis v1.1.0/go.mod h1:gf4bu3Q80BeJ6H1S1vYPm8/ELATdvryBaNFGgqEef3s= 207 | github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= 208 | github.com/gogo/protobuf v1.2.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= 209 | github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= 210 | github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= 211 | github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= 212 | github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= 213 | github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= 214 | github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= 215 | github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= 216 | github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= 217 | github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7 h1:5ZkaAPbicIKTF2I64qf5Fh8Aa83Q/dnOafMYV0OMwjA= 218 | github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= 219 | github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= 220 | github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= 221 | github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= 222 | github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= 223 | github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= 224 | github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= 225 | github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= 226 | github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= 227 | github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= 228 | github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= 229 | github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= 230 | github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= 231 | github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= 232 | github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= 233 | github.com/golang/protobuf v1.4.3 h1:JjCZWpVbqXDqFVmTfYWEVTMIYrL/NPdPSCHPJ0T/raM= 234 | github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= 235 | github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= 236 | github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= 237 | github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= 238 | github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= 239 | github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= 240 | github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= 241 | github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= 242 | github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= 243 | github.com/google/go-cmp v0.5.4 h1:L8R9j+yAqZuZjsqh/z+F1NCffTKKLShY6zXTItVIZ8M= 244 | github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= 245 | github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= 246 | github.com/google/gofuzz v1.1.0 h1:Hsa8mG0dQ46ij8Sl2AYJDUv1oA9/d6Vk+3LG99Oe02g= 247 | github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= 248 | github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= 249 | github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= 250 | github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= 251 | github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= 252 | github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= 253 | github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= 254 | github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY= 255 | github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= 256 | github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= 257 | github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= 258 | github.com/googleapis/gnostic v0.4.1/go.mod h1:LRhVm6pbyptWbWbuZ38d1eyptfvIytN3ir6b65WBswg= 259 | github.com/googleapis/gnostic v0.5.1 h1:A8Yhf6EtqTv9RMsU6MQTyrtV1TjWlR6xU9BsZIwuTCM= 260 | github.com/googleapis/gnostic v0.5.1/go.mod h1:6U4PtQXGIEt/Z3h5MAT7FNofLnw9vXk2cUuW7uA/OeU= 261 | github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= 262 | github.com/gorilla/context v1.1.1/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51q0aT7Yg= 263 | github.com/gorilla/mux v1.6.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= 264 | github.com/gorilla/mux v1.7.3/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= 265 | github.com/gorilla/websocket v0.0.0-20170926233335-4201258b820c/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= 266 | github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= 267 | github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= 268 | github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= 269 | github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= 270 | github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= 271 | github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= 272 | github.com/grpc-ecosystem/grpc-gateway v1.9.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= 273 | github.com/hashicorp/consul/api v1.3.0/go.mod h1:MmDNSzIMUjNpY/mQ398R4bk2FnqQLoPndWW5VkKPlCE= 274 | github.com/hashicorp/consul/sdk v0.3.0/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8= 275 | github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= 276 | github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= 277 | github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= 278 | github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= 279 | github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= 280 | github.com/hashicorp/go-rootcerts v1.0.0/go.mod h1:K6zTfqpRlCUIjkwsN4Z+hiSfzSTQa6eBIzfwKfwNnHU= 281 | github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU= 282 | github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4= 283 | github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= 284 | github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= 285 | github.com/hashicorp/go-version v1.2.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= 286 | github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90= 287 | github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= 288 | github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= 289 | github.com/hashicorp/golang-lru v0.5.4 h1:YDjusn29QI/Das2iO9M0BHnIbxPeyuCHsjMW+lJfyTc= 290 | github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= 291 | github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= 292 | github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64= 293 | github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ= 294 | github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I= 295 | github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc= 296 | github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= 297 | github.com/hudl/fargo v1.3.0/go.mod h1:y3CKSmjA+wD2gak7sUSXTAoopbhU08POFhmITJgmKTg= 298 | github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= 299 | github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= 300 | github.com/imdario/mergo v0.3.10 h1:6q5mVkdH/vYmqngx7kZQTjJ5HRsx+ImorDIEQ+beJgc= 301 | github.com/imdario/mergo v0.3.10/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= 302 | github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= 303 | github.com/influxdata/influxdb1-client v0.0.0-20191209144304-8bf82d3c094d/go.mod h1:qj24IKcXYK6Iy9ceXlo3Tc+vtHo9lIhSX5JddghvEPo= 304 | github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= 305 | github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= 306 | github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= 307 | github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= 308 | github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= 309 | github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= 310 | github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= 311 | github.com/json-iterator/go v1.1.8/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= 312 | github.com/json-iterator/go v1.1.10 h1:Kz6Cvnvv2wGdaG/V8yMvfkmNiXq9Ya2KUv4rouJJr68= 313 | github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= 314 | github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= 315 | github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= 316 | github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= 317 | github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= 318 | github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= 319 | github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= 320 | github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= 321 | github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= 322 | github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= 323 | github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= 324 | github.com/konsorten/go-windows-terminal-sequences v1.0.3 h1:CE8S1cTafDpPvMhIxNJKvHsGVBgn1xWYf1NbHQhywc8= 325 | github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= 326 | github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= 327 | github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= 328 | github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= 329 | github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= 330 | github.com/kr/pty v1.1.5/go.mod h1:9r2w37qlBe7rQ6e1fg1S/9xpWHSnaqNdHD3WcMdbPDA= 331 | github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= 332 | github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= 333 | github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= 334 | github.com/kubeflow/common v0.4.1 h1:da/ruarFTTiR9+H6kQibsvAWbqkEt2nb+Jly4UkaydQ= 335 | github.com/kubeflow/common v0.4.1/go.mod h1:J1cQhvedpOJfXf+nqIJCUB14ctBxtYpm40xOYzI7DqE= 336 | github.com/kubeflow/training-operator v1.4.0 h1:1AuPCS0w6mAEDZPjxNOl5fQfM5yOzxTBiQQWQTJzFaw= 337 | github.com/kubeflow/training-operator v1.4.0/go.mod h1:EjLnGl/05gmInmChAWh1PjoQo6xBy7903MZM9ObbSJc= 338 | github.com/lightstep/lightstep-tracer-common/golang/gogo v0.0.0-20190605223551-bc2310a04743/go.mod h1:qklhhLq1aX+mtWk9cPHPzaBjWImj5ULL6C7HFJtXQMM= 339 | github.com/lightstep/lightstep-tracer-go v0.18.1/go.mod h1:jlF1pusYV4pidLvZ+XD0UBX0ZE6WURAspgAczcDHrL4= 340 | github.com/lyft/protoc-gen-validate v0.0.13/go.mod h1:XbGvPuh87YZc5TdIa2/I4pLk0QoUACkjt2znoq26NVQ= 341 | github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= 342 | github.com/mailru/easyjson v0.0.0-20160728113105-d5b7844b561a/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= 343 | github.com/mailru/easyjson v0.0.0-20180823135443-60711f1a8329/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= 344 | github.com/mailru/easyjson v0.0.0-20190312143242-1de009706dbe/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= 345 | github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= 346 | github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= 347 | github.com/mailru/easyjson v0.7.0/go.mod h1:KAzv3t3aY1NaHWoQz1+4F1ccyAH66Jk7yos7ldAVICs= 348 | github.com/mailru/easyjson v0.7.6 h1:8yTIVnZgCoiM1TgqoeTl+LfU5Jg6/xL3QhGQnimLYnA= 349 | github.com/mailru/easyjson v0.7.6/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= 350 | github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= 351 | github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= 352 | github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= 353 | github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= 354 | github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= 355 | github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 h1:I0XW9+e1XWDxdcEniV4rQAIOPUGDq67JSCiRCgGCZLI= 356 | github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= 357 | github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= 358 | github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= 359 | github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= 360 | github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= 361 | github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= 362 | github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS42BGNg= 363 | github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0QubkSMEySY= 364 | github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= 365 | github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= 366 | github.com/moby/term v0.0.0-20200312100748-672ec06f55cd/go.mod h1:DdlQx2hp0Ss5/fLikoLlEeIYiATotOjgB//nb973jeo= 367 | github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= 368 | github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= 369 | github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= 370 | github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= 371 | github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI= 372 | github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= 373 | github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= 374 | github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= 375 | github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= 376 | github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= 377 | github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw= 378 | github.com/nats-io/jwt v0.3.0/go.mod h1:fRYCDE99xlTsqUzISS1Bi75UBJ6ljOJQOAAu5VglpSg= 379 | github.com/nats-io/jwt v0.3.2/go.mod h1:/euKqTS1ZD+zzjYrY7pseZrTtWQSjujC7xjPc8wL6eU= 380 | github.com/nats-io/nats-server/v2 v2.1.2/go.mod h1:Afk+wRZqkMQs/p45uXdrVLuab3gwv3Z8C4HTBu8GD/k= 381 | github.com/nats-io/nats.go v1.9.1/go.mod h1:ZjDU1L/7fJ09jvUSRVBR2e7+RnLiiIQyqyzEE/Zbp4w= 382 | github.com/nats-io/nkeys v0.1.0/go.mod h1:xpnFELMwJABBLVhffcfd1MZx6VsNRFpEugbxziKVo7w= 383 | github.com/nats-io/nkeys v0.1.3/go.mod h1:xpnFELMwJABBLVhffcfd1MZx6VsNRFpEugbxziKVo7w= 384 | github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= 385 | github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs= 386 | github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= 387 | github.com/nxadm/tail v1.4.4 h1:DQuhQpB1tVlglWS2hLQ5OV6B5r8aGxSrPc5Qo6uTN78= 388 | github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= 389 | github.com/oklog/oklog v0.3.2/go.mod h1:FCV+B7mhrz4o+ueLpx+KqkyXRGMWOYEvfiXtdGtbWGs= 390 | github.com/oklog/run v1.0.0/go.mod h1:dlhp/R75TPv97u0XWUtDeV/lRKWPKSdTuV0TZvrmrQA= 391 | github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= 392 | github.com/olekukonko/tablewriter v0.0.0-20170122224234-a0225b3f23b5/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo= 393 | github.com/onsi/ginkgo v0.0.0-20170829012221-11459a886d9c/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= 394 | github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= 395 | github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= 396 | github.com/onsi/ginkgo v1.11.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= 397 | github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= 398 | github.com/onsi/ginkgo v1.14.1 h1:jMU0WaQrP0a/YAEq8eJmJKjBoMs+pClEr1vDMlM/Do4= 399 | github.com/onsi/ginkgo v1.14.1/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY= 400 | github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= 401 | github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= 402 | github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= 403 | github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= 404 | github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= 405 | github.com/onsi/gomega v1.10.2 h1:aY/nuoWlKJud2J6U0E3NWsjlg+0GtwXxgEqthRdzlcs= 406 | github.com/onsi/gomega v1.10.2/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= 407 | github.com/op/go-logging v0.0.0-20160315200505-970db520ece7/go.mod h1:HzydrMdWErDVzsI23lYNej1Htcns9BCg93Dk0bBINWk= 408 | github.com/opentracing-contrib/go-observer v0.0.0-20170622124052-a52f23424492/go.mod h1:Ngi6UdF0k5OKD5t5wlmGhe/EDKPoUM3BXZSSfIuJbis= 409 | github.com/opentracing/basictracer-go v1.0.0/go.mod h1:QfBfYuafItcjQuMwinw9GhYKwFXS9KnPs5lxoYwgW74= 410 | github.com/opentracing/opentracing-go v1.0.2/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= 411 | github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= 412 | github.com/openzipkin-contrib/zipkin-go-opentracing v0.4.5/go.mod h1:/wsWhb9smxSfWAKL3wpBW7V8scJMt8N8gnaMCS9E/cA= 413 | github.com/openzipkin/zipkin-go v0.1.6/go.mod h1:QgAqvLzwWbR/WpD4A3cGpPtJrZXNIiJc5AZX7/PBEpw= 414 | github.com/openzipkin/zipkin-go v0.2.1/go.mod h1:NaW6tEwdmWMaCDZzg8sh+IBNOxHMPnhQw8ySjnjRyN4= 415 | github.com/openzipkin/zipkin-go v0.2.2/go.mod h1:NaW6tEwdmWMaCDZzg8sh+IBNOxHMPnhQw8ySjnjRyN4= 416 | github.com/pact-foundation/pact-go v1.0.4/go.mod h1:uExwJY4kCzNPcHRj+hCR/HBbOOIwwtUjcrb0b5/5kLM= 417 | github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= 418 | github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k= 419 | github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= 420 | github.com/performancecopilot/speed v3.0.0+incompatible/go.mod h1:/CLtqpZ5gBg1M9iaPbIdPPGyKcA8hKdoy6hAWba7Yac= 421 | github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= 422 | github.com/pierrec/lz4 v1.0.2-0.20190131084431-473cd7ce01a1/go.mod h1:3/3N9NVKO0jef7pBehbT1qWhCMrIgbYNnFAZCqQ5LRc= 423 | github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= 424 | github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= 425 | github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= 426 | github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= 427 | github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= 428 | github.com/pkg/profile v1.2.1/go.mod h1:hJw3o1OdXxsrSjjVksARp5W95eeEaEfptyVZyv6JUPA= 429 | github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= 430 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 431 | github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= 432 | github.com/pquerna/cachecontrol v0.0.0-20171018203845-0dec1b30a021/go.mod h1:prYjPmNq4d1NPVmpShWobRqXY3q7Vp+80DqgxxUrUIA= 433 | github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= 434 | github.com/prometheus/client_golang v0.9.3-0.20190127221311-3c4408c8b829/go.mod h1:p2iRAGwDERtqlqzRXnrOVns+ignqQo//hLXqYxZYVNs= 435 | github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso= 436 | github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= 437 | github.com/prometheus/client_golang v1.3.0/go.mod h1:hJaj2vgQTGQmVCsAACORcieXFeDPbaTKGT+JTgUa3og= 438 | github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= 439 | github.com/prometheus/client_golang v1.10.0 h1:/o0BDeWzLWXNZ+4q5gXltUvaMpJqckTa+jTNoB+z4cg= 440 | github.com/prometheus/client_golang v1.10.0/go.mod h1:WJM3cc3yu7XKBKa/I8WeZm+V3eltZnBwfENSU7mdogU= 441 | github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= 442 | github.com/prometheus/client_model v0.0.0-20190115171406-56726106282f/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= 443 | github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= 444 | github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= 445 | github.com/prometheus/client_model v0.1.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= 446 | github.com/prometheus/client_model v0.2.0 h1:uq5h0d+GuxiXLJLNABMgp2qUWDPiLvgCzz2dUR+/W/M= 447 | github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= 448 | github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= 449 | github.com/prometheus/common v0.2.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= 450 | github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= 451 | github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= 452 | github.com/prometheus/common v0.7.0/go.mod h1:DjGbpBbp5NYNiECxcL/VnbXCCaQpKd3tt26CguLLsqA= 453 | github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= 454 | github.com/prometheus/common v0.18.0 h1:WCVKW7aL6LEe1uryfI9dnEc2ZqNB1Fn0ok930v0iL1Y= 455 | github.com/prometheus/common v0.18.0/go.mod h1:U+gB1OBLb1lF3O42bTCL+FK18tX9Oar16Clt/msog/s= 456 | github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= 457 | github.com/prometheus/procfs v0.0.0-20190117184657-bf6a532e95b1/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= 458 | github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= 459 | github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= 460 | github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A= 461 | github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= 462 | github.com/prometheus/procfs v0.6.0 h1:mxy4L2jP6qMonqmq+aTtOx1ifVWUgG/TAmntgbh3xv4= 463 | github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= 464 | github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= 465 | github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= 466 | github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= 467 | github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= 468 | github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= 469 | github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= 470 | github.com/samuel/go-zookeeper v0.0.0-20190923202752-2cc03de413da/go.mod h1:gi+0XIa01GRL2eRQVjQkKGqKF3SF9vZR/HnPullcV2E= 471 | github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= 472 | github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= 473 | github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= 474 | github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= 475 | github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= 476 | github.com/sirupsen/logrus v1.6.0 h1:UBcNElsrwanuuMsnGSlYmtmgbb23qDR5dG+6X6Oo89I= 477 | github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= 478 | github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= 479 | github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= 480 | github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= 481 | github.com/sony/gobreaker v0.4.1/go.mod h1:ZKptC7FHNvhBz7dN2LGjPVBz2sZJmc0/PkyDJOjmxWY= 482 | github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= 483 | github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= 484 | github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= 485 | github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= 486 | github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= 487 | github.com/spf13/cobra v1.0.0/go.mod h1:/6GTrnGXV9HjY+aR4k0oJ5tcvakLuG6EuKReYlHNrgE= 488 | github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= 489 | github.com/spf13/pflag v0.0.0-20170130214245-9ff6c6923cff/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= 490 | github.com/spf13/pflag v1.0.1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= 491 | github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= 492 | github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= 493 | github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= 494 | github.com/spf13/viper v1.4.0/go.mod h1:PTJ7Z/lr49W6bUbkmS1V3by4uWynFiR9p7+dSq/yZzE= 495 | github.com/stoewer/go-strcase v1.2.0/go.mod h1:IBiWB2sKIp3wVVQ3Y035++gc+knqhUQag1KpM8ahLw8= 496 | github.com/streadway/amqp v0.0.0-20190404075320-75d898a42a94/go.mod h1:AZpEONHx3DKn8O/DFsRAY58/XVQiIPMTMB1SddzLXVw= 497 | github.com/streadway/amqp v0.0.0-20190827072141-edfb9018d271/go.mod h1:AZpEONHx3DKn8O/DFsRAY58/XVQiIPMTMB1SddzLXVw= 498 | github.com/streadway/handy v0.0.0-20190108123426-d5acb3125c2a/go.mod h1:qNTQ5P5JnDBl6z3cMAg/SywNDC5ABu5ApDIw6lUbRmI= 499 | github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 500 | github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 501 | github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= 502 | github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= 503 | github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= 504 | github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= 505 | github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= 506 | github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0= 507 | github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= 508 | github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= 509 | github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= 510 | github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= 511 | github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc= 512 | github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= 513 | github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= 514 | github.com/vektah/gqlparser v1.1.2/go.mod h1:1ycwN7Ij5njmMkPPAOaRFY4rET2Enx7IkVv3vaXspKw= 515 | github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= 516 | github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= 517 | github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= 518 | github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= 519 | go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= 520 | go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= 521 | go.etcd.io/bbolt v1.3.5/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ= 522 | go.etcd.io/etcd v0.0.0-20191023171146-3cf2f69b5738/go.mod h1:dnLIgRNXwCJa5e+c6mIZCrds/GIG4ncV9HhK5PX7jPg= 523 | go.etcd.io/etcd v0.5.0-alpha.5.0.20200819165624-17cef6e3e9d5/go.mod h1:skWido08r9w6Lq/w70DO5XYIKMu4QFu1+4VsqLQuJy8= 524 | go.mongodb.org/mongo-driver v1.0.3/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qLUO4lqsUM= 525 | go.mongodb.org/mongo-driver v1.1.1/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qLUO4lqsUM= 526 | go.mongodb.org/mongo-driver v1.1.2/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qLUO4lqsUM= 527 | go.opencensus.io v0.20.1/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk= 528 | go.opencensus.io v0.20.2/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk= 529 | go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= 530 | go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= 531 | go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= 532 | go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= 533 | go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= 534 | go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= 535 | go.uber.org/atomic v1.6.0 h1:Ezj3JGmsOnG1MoRWQkPBsKLe9DwWD9QeXzTRzzldNVk= 536 | go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= 537 | go.uber.org/goleak v1.1.10 h1:z+mqJhf6ss6BSfSM671tgKyZBFPTTJM+HLxnhPC3wu0= 538 | go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A= 539 | go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= 540 | go.uber.org/multierr v1.3.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+4= 541 | go.uber.org/multierr v1.5.0 h1:KCa4XfM8CWFCpxXRGok+Q0SS/0XBhMDbHHGABQLvD2A= 542 | go.uber.org/multierr v1.5.0/go.mod h1:FeouvMocqHpRaaGuG9EjoKcStLC43Zu/fmqdUMPcKYU= 543 | go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee h1:0mgffUl7nfd+FpvXMVz4IDEaUSmT1ysygQC7qYo7sG4= 544 | go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9Ejo0C68/HhF8uaILCdgjnY+goOA= 545 | go.uber.org/zap v1.8.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= 546 | go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= 547 | go.uber.org/zap v1.13.0/go.mod h1:zwrFLgMcdUuIBviXEYEH1YKNaOBnKXsx2IPda5bBwHM= 548 | go.uber.org/zap v1.15.0 h1:ZZCA22JRF2gQE5FoNmhmrf7jeJJ2uhqDUNRYKm8dvmM= 549 | go.uber.org/zap v1.15.0/go.mod h1:Mb2vm2krFEG5DV0W9qcHBYFtp/Wku1cvYaqPsS/WYfc= 550 | golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= 551 | golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= 552 | golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= 553 | golang.org/x/crypto v0.0.0-20190320223903-b7391e95e576/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= 554 | golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= 555 | golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= 556 | golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= 557 | golang.org/x/crypto v0.0.0-20190617133340-57b3e21c3d56/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= 558 | golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= 559 | golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= 560 | golang.org/x/crypto v0.0.0-20191206172530-e9b2fee46413/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= 561 | golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 h1:psW17arqaxU48Z5kZ0CQnkZWQJsqcURM6tKiBApRjXI= 562 | golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= 563 | golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= 564 | golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= 565 | golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= 566 | golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek= 567 | golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= 568 | golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= 569 | golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= 570 | golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= 571 | golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= 572 | golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= 573 | golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= 574 | golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= 575 | golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= 576 | golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= 577 | golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f h1:J5lckAjkw6qYlOZNj90mLYNTEKDvWeuc1yieZ8qUzUE= 578 | golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= 579 | golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= 580 | golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= 581 | golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= 582 | golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= 583 | golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= 584 | golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= 585 | golang.org/x/mod v0.3.0 h1:RM4zey1++hCTbCVQfnWeKs9/IEsaBLA8vTkd0WVtmH4= 586 | golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= 587 | golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 588 | golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 589 | golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 590 | golang.org/x/net v0.0.0-20181005035420-146acd28ed58/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 591 | golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 592 | golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 593 | golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 594 | golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 595 | golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 596 | golang.org/x/net v0.0.0-20190125091013-d26f9f9a57f3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 597 | golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 598 | golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= 599 | golang.org/x/net v0.0.0-20190320064053-1272bf9dcd53/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= 600 | golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= 601 | golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= 602 | golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= 603 | golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= 604 | golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= 605 | golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 606 | golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 607 | golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 608 | golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 609 | golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 610 | golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 611 | golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= 612 | golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= 613 | golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= 614 | golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= 615 | golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= 616 | golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= 617 | golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= 618 | golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= 619 | golang.org/x/net v0.0.0-20220225172249-27dd8689420f h1:oA4XRj0qtSt8Yo1Zms0CUlsT3KG69V2UGQWPBxujDmc= 620 | golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= 621 | golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= 622 | golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= 623 | golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= 624 | golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6 h1:pE8b58s1HRDMi8RDc79m0HISf9D4TzseP40cEA6IGfs= 625 | golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= 626 | golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 627 | golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 628 | golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 629 | golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 630 | golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 631 | golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 632 | golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 633 | golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 634 | golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 635 | golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 636 | golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 637 | golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 638 | golang.org/x/sys v0.0.0-20181026203630-95b1ffbd15a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 639 | golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 640 | golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 641 | golang.org/x/sys v0.0.0-20181122145206-62eef0e2fa9b/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 642 | golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 643 | golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 644 | golang.org/x/sys v0.0.0-20190321052220-f7bb7a8bee54/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 645 | golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 646 | golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 647 | golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 648 | golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 649 | golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 650 | golang.org/x/sys v0.0.0-20190616124812-15dcb6c0061f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 651 | golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 652 | golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 653 | golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 654 | golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 655 | golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 656 | golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 657 | golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 658 | golang.org/x/sys v0.0.0-20191220142924-d4481acd189f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 659 | golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 660 | golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 661 | golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 662 | golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 663 | golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 664 | golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 665 | golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 666 | golang.org/x/sys v0.0.0-20200622214017-ed371f2e16b4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 667 | golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 668 | golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 669 | golang.org/x/sys v0.0.0-20201112073958-5cba982894dd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 670 | golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 671 | golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 672 | golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 673 | golang.org/x/sys v0.0.0-20210309074719-68d13333faf2/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 674 | golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 675 | golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 676 | golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 677 | golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e h1:fLOSk5Q00efkSvAm+4xcoXD+RRmLmmulPn5I3Y9F2EM= 678 | golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 679 | golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= 680 | golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= 681 | golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= 682 | golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= 683 | golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= 684 | golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= 685 | golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= 686 | golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk= 687 | golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= 688 | golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= 689 | golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= 690 | golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= 691 | golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= 692 | golang.org/x/time v0.0.0-20200630173020-3af7569d3a1e h1:EHBhcS0mlXEAVwNyO2dLfjToGsyY4j24pTs2ScHnX7s= 693 | golang.org/x/time v0.0.0-20200630173020-3af7569d3a1e/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= 694 | golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= 695 | golang.org/x/tools v0.0.0-20180828015842-6cd1fcedba52/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= 696 | golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= 697 | golang.org/x/tools v0.0.0-20181011042414-1f849cf54d09/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= 698 | golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= 699 | golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= 700 | golang.org/x/tools v0.0.0-20190125232054-d66bd3c5d5a6/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= 701 | golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= 702 | golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= 703 | golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= 704 | golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= 705 | golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= 706 | golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= 707 | golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= 708 | golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= 709 | golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= 710 | golang.org/x/tools v0.0.0-20190614205625-5aca471b1d59/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= 711 | golang.org/x/tools v0.0.0-20190617190820-da514acc4774/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= 712 | golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= 713 | golang.org/x/tools v0.0.0-20190624222133-a101b041ded4/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= 714 | golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= 715 | golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= 716 | golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= 717 | golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= 718 | golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= 719 | golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= 720 | golang.org/x/tools v0.0.0-20191108193012-7d206e10da11/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= 721 | golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= 722 | golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= 723 | golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= 724 | golang.org/x/tools v0.0.0-20200103221440-774c71fcf114/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= 725 | golang.org/x/tools v0.0.0-20200616133436-c1934b75d054/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= 726 | golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= 727 | golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= 728 | golang.org/x/tools v0.1.0 h1:po9/4sTYwZU9lPhi1tOrb4hCv3qrhiQ77LZfGa2OjwY= 729 | golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= 730 | golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 731 | golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 732 | golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 733 | golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= 734 | golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 735 | gomodules.xyz/jsonpatch/v2 v2.1.0 h1:Phva6wqu+xR//Njw6iorylFFgn/z547tw5Ne3HZPQ+k= 736 | gomodules.xyz/jsonpatch/v2 v2.1.0/go.mod h1:IhYNNY4jnS53ZnfE4PAmpKtDpTCj1JFXc+3mwe7XcUU= 737 | google.golang.org/api v0.3.1/go.mod h1:6wY9I6uQWHQ8EM57III9mq/AjF+i8G65rmVagqKMtkk= 738 | google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= 739 | google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= 740 | google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= 741 | google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= 742 | google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= 743 | google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= 744 | google.golang.org/appengine v1.2.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= 745 | google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= 746 | google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= 747 | google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= 748 | google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= 749 | google.golang.org/appengine v1.6.6 h1:lMO5rYAqUxkmaj76jAkRUvt5JZgFymx/+Q5Mzfivuhc= 750 | google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= 751 | google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= 752 | google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= 753 | google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= 754 | google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= 755 | google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= 756 | google.golang.org/genproto v0.0.0-20190530194941-fb225487d101/go.mod h1:z3L6/3dTEVtUr6QSP8miRzeRqwQOioJ9I66odjN4I7s= 757 | google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= 758 | google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= 759 | google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= 760 | google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= 761 | google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= 762 | google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs= 763 | google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= 764 | google.golang.org/grpc v1.20.0/go.mod h1:chYK+tFQF0nDUGJgXMSgLCQk3phJEuONr2DCgLDdAQM= 765 | google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= 766 | google.golang.org/grpc v1.21.0/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= 767 | google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= 768 | google.golang.org/grpc v1.22.1/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= 769 | google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= 770 | google.golang.org/grpc v1.23.1/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= 771 | google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= 772 | google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= 773 | google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= 774 | google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= 775 | google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= 776 | google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= 777 | google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= 778 | google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= 779 | google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= 780 | google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= 781 | google.golang.org/protobuf v1.24.0 h1:UhZDfRO8JRQru4/+LlLE0BRKGF8L+PICnvYZmx/fEGA= 782 | google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4= 783 | gopkg.in/alecthomas/kingpin.v2 v2.2.6 h1:jMFz6MfLP0/4fUyZle81rXUoxOBFi19VUFKVDOQfozc= 784 | gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= 785 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 786 | gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 787 | gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 788 | gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f h1:BLraFXnmrev5lT+xlilqcH8XK9/i0At2xKjWk4p6zsU= 789 | gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 790 | gopkg.in/cheggaaa/pb.v1 v1.0.25/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw= 791 | gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= 792 | gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= 793 | gopkg.in/gcfg.v1 v1.2.3/go.mod h1:yesOnuUOFQAhST5vPY4nbZsb/huCgGGXlipJsBn0b3o= 794 | gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= 795 | gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= 796 | gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k= 797 | gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= 798 | gopkg.in/square/go-jose.v2 v2.2.2/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI= 799 | gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= 800 | gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= 801 | gopkg.in/warnings.v0 v0.1.2/go.mod h1:jksf8JmL6Qr/oQM2OXTHunEvvTAsrWBLb6OOjuVWRNI= 802 | gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= 803 | gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 804 | gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 805 | gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 806 | gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 807 | gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 808 | gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 809 | gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= 810 | gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= 811 | gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 812 | gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776 h1:tQIYjPdBoyREyB9XMu+nnTclpTYkz2zFM+lzLJFO4gQ= 813 | gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 814 | gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= 815 | gotest.tools/v3 v3.0.2/go.mod h1:3SzNCllyD9/Y+b5r9JIKQ474KzkZyqLqEfYqMsX94Bk= 816 | honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= 817 | honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= 818 | honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= 819 | honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= 820 | honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= 821 | honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= 822 | honnef.co/go/tools v0.2.2 h1:MNh1AVMyVX23VUHE2O27jm6lNj3vjO5DexS4A1xvnzk= 823 | honnef.co/go/tools v0.2.2/go.mod h1:lPVVZ2BS5TfnjLyizF7o7hv7j9/L+8cZY2hLyjP9cGY= 824 | k8s.io/api v0.19.2/go.mod h1:IQpK0zFQ1xc5iNIQPqzgoOwuFugaYHK4iCknlAQP9nI= 825 | k8s.io/api v0.19.6/go.mod h1:Plxx44Nh4zVblkJrIgxVPgPre1mvng6tXf1Sj3bs0fU= 826 | k8s.io/api v0.19.9 h1:Bs6ihik0V+4WDO9eFWrCNwhSUg7dW/Y+HflDCpOfNGk= 827 | k8s.io/api v0.19.9/go.mod h1:RcFj+riKQ1fAITdmtA6InI3LVEeKi+9LuvU7GVMeXJI= 828 | k8s.io/apiextensions-apiserver v0.19.2 h1:oG84UwiDsVDu7dlsGQs5GySmQHCzMhknfhFExJMz9tA= 829 | k8s.io/apiextensions-apiserver v0.19.2/go.mod h1:EYNjpqIAvNZe+svXVx9j4uBaVhTB4C94HkY3w058qcg= 830 | k8s.io/apimachinery v0.19.2/go.mod h1:DnPGDnARWFvYa3pMHgSxtbZb7gpzzAZ1pTfaUNDVlmA= 831 | k8s.io/apimachinery v0.19.6/go.mod h1:6sRbGRAVY5DOCuZwB5XkqguBqpqLU6q/kOaOdk29z6Q= 832 | k8s.io/apimachinery v0.19.9 h1:ToBt9RaKt5BYD2uHNgom2O0MqvfXnEYimcxqCzZIkXA= 833 | k8s.io/apimachinery v0.19.9/go.mod h1:6sRbGRAVY5DOCuZwB5XkqguBqpqLU6q/kOaOdk29z6Q= 834 | k8s.io/apiserver v0.19.2/go.mod h1:FreAq0bJ2vtZFj9Ago/X0oNGC51GfubKK/ViOKfVAOA= 835 | k8s.io/apiserver v0.19.6/go.mod h1:05XquZxCDzQ27ebk7uV2LrFIK4lm5Yt47XkkUvLAoAM= 836 | k8s.io/client-go v0.19.2/go.mod h1:S5wPhCqyDNAlzM9CnEdgTGV4OqhsW3jGO1UM1epwfJA= 837 | k8s.io/client-go v0.19.6/go.mod h1:gEiS+efRlXYUEQ9Oz4lmNXlxAl5JZ8y2zbTDGhvXXnk= 838 | k8s.io/client-go v0.19.9 h1:Bs0ZnQOWnRYvOlAsT7tDro1j1B6ZaVX/O3C/k6EoaGE= 839 | k8s.io/client-go v0.19.9/go.mod h1:8GArfSmN7MwTidMGcLGM3QTYa7uekI/B6IOrM0s1XPs= 840 | k8s.io/code-generator v0.19.2/go.mod h1:moqLn7w0t9cMs4+5CQyxnfA/HV8MF6aAVENF+WZZhgk= 841 | k8s.io/code-generator v0.19.6/go.mod h1:lwEq3YnLYb/7uVXLorOJfxg+cUu2oihFhHZ0n9NIla0= 842 | k8s.io/code-generator v0.19.9 h1:nj1gVb/4P4C53hnBtdTaxZDlJ1jEkrQnAy+n4BYGVHs= 843 | k8s.io/code-generator v0.19.9/go.mod h1:lwEq3YnLYb/7uVXLorOJfxg+cUu2oihFhHZ0n9NIla0= 844 | k8s.io/component-base v0.19.2/go.mod h1:g5LrsiTiabMLZ40AR6Hl45f088DevyGY+cCE2agEIVo= 845 | k8s.io/component-base v0.19.6/go.mod h1:8Btsf8J00/fVDa/YFmXjei7gVkcFrlKZXjSeP4SZNJg= 846 | k8s.io/component-base v0.19.9 h1:GOjvFCDgTRfLz6v3xshO0QbqWJN5nAkJzypc2BIfxOw= 847 | k8s.io/component-base v0.19.9/go.mod h1:x9UmpImvXgVry1s9/hINgLz6iGBYUGvy3Xm7KZh1nnI= 848 | k8s.io/gengo v0.0.0-20200413195148-3a45101e95ac/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= 849 | k8s.io/gengo v0.0.0-20200428234225-8167cfdcfc14/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= 850 | k8s.io/klog v1.0.0 h1:Pt+yjF5aB1xDSVbau4VsWe+dQNzA0qv1LlXdC2dF6Q8= 851 | k8s.io/klog v1.0.0/go.mod h1:4Bi6QPql/J/LkTDqv7R/cd3hPo4k2DG6Ptcz060Ez5I= 852 | k8s.io/klog/v2 v2.0.0/go.mod h1:PBfzABfn139FHAV07az/IF9Wp1bkk3vpT2XSJ76fSDE= 853 | k8s.io/klog/v2 v2.2.0 h1:XRvcwJozkgZ1UQJmfMGpvRthQHOvihEhYtDfAaxMz/A= 854 | k8s.io/klog/v2 v2.2.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y= 855 | k8s.io/kube-openapi v0.0.0-20200805222855-6aeccd4b50c6 h1:+WnxoVtG8TMiudHBSEtrVL1egv36TkkJm+bA8AxicmQ= 856 | k8s.io/kube-openapi v0.0.0-20200805222855-6aeccd4b50c6/go.mod h1:UuqjUnNftUyPE5H64/qeyjQoUZhGpeFDVdxjTeEVN2o= 857 | k8s.io/utils v0.0.0-20200729134348-d5654de09c73/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= 858 | k8s.io/utils v0.0.0-20200912215256-4140de9c8800 h1:9ZNvfPvVIEsp/T1ez4GQuzCcCTEQWhovSofhqR73A6g= 859 | k8s.io/utils v0.0.0-20200912215256-4140de9c8800/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= 860 | rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= 861 | sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.9/go.mod h1:dzAXnQbTRyDlZPJX2SUPEqvnB+j7AJjtlox7PEwigU0= 862 | sigs.k8s.io/controller-runtime v0.7.2 h1:gD2JZp0bBLLuvSRYVNvox+bRCz1UUUxKDjPUCb56Ukk= 863 | sigs.k8s.io/controller-runtime v0.7.2/go.mod h1:pJ3YBrJiAqMAZKi6UVGuE98ZrroV1p+pIhoHsMm9wdU= 864 | sigs.k8s.io/structured-merge-diff/v4 v4.0.1 h1:YXTMot5Qz/X1iBRJhAt+vI+HVttY0WkSqqhKxQ0xVbA= 865 | sigs.k8s.io/structured-merge-diff/v4 v4.0.1/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw= 866 | sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o= 867 | sigs.k8s.io/yaml v1.2.0 h1:kr/MCeFWJWTwyaHoR9c8EjH9OumOmoF9YGiZd7lFm/Q= 868 | sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc= 869 | sourcegraph.com/sourcegraph/appdash v0.0.0-20190731080439-ebfcffb1b5c0/go.mod h1:hI742Nqp5OhwiqlzhgfbWU4mW4yO10fP+LoT9WOswdU= 870 | volcano.sh/apis v1.2.0-k8s1.19.6 h1:ddSHBrxHOVpGVSYQC/e696tDmCQ2dPm/Adg7aTQdOPI= 871 | volcano.sh/apis v1.2.0-k8s1.19.6/go.mod h1:UaeJ/s5Hyd+ZhFLc+Kw9YlgM8gRZ/5OzXqHa0yKOoXY= 872 | -------------------------------------------------------------------------------- /hack/boilerplate.go.txt: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2022 Huawei Technologies Co., Ltd. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ -------------------------------------------------------------------------------- /hack/update-codegen.sh: -------------------------------------------------------------------------------- 1 | SCRIPT_ROOT=$(dirname "${BASH_SOURCE[0]}")/.. 2 | # Grab code-generator version from go.sum 3 | CODEGEN_VERSION=$(grep 'k8s.io/code-generator' go.mod | awk '{print $2}') 4 | CODEGEN_PKG=$(echo $(go env GOPATH)"/pkg/mod/k8s.io/code-generator@${CODEGEN_VERSION}") 5 | 6 | if [[ ! -d ${CODEGEN_PKG} ]]; then 7 | echo "${CODEGEN_PKG} is missing. Running 'go mod download'." 8 | go mod download 9 | fi 10 | 11 | echo ">> Using ${CODEGEN_PKG}" 12 | 13 | # Grab openapi-gen version from go.mod 14 | OPENAPI_VERSION=$(grep 'k8s.io/kube-openapi' go.mod | awk '{print $2}') 15 | OPENAPI_PKG=$(echo $(go env GOPATH)"/pkg/mod/k8s.io/kube-openapi@${OPENAPI_VERSION}") 16 | 17 | if [[ ! -d ${OPENAPI_PKG} ]]; then 18 | echo "${OPENAPI_PKG} is missing. Running 'go mod download'." 19 | go mod download 20 | fi 21 | 22 | echo ">> Using ${OPENAPI_PKG}" 23 | 24 | # code-generator does work with go.mod but makes assumptions about 25 | # the project living in `$GOPATH/src`. To work around this and support 26 | # any location; create a temporary directory, use this as an output 27 | # base, and copy everything back once generated. 28 | TEMP_DIR=$(mktemp -d) 29 | cleanup() { 30 | echo ">> Removing ${TEMP_DIR}" 31 | rm -rf ${TEMP_DIR} 32 | } 33 | trap "cleanup" EXIT SIGINT 34 | 35 | echo ">> Temporary output directory ${TEMP_DIR}" 36 | 37 | # Ensure we can execute. 38 | chmod +x ${CODEGEN_PKG}/generate-groups.sh 39 | 40 | # generate the code with: 41 | # --output-base because this script should also be able to run inside the vendor dir of 42 | # k8s.io/kubernetes. The output-base is needed for the generators to output into the vendor dir 43 | # instead of the $GOPATH directly. For normal projects this can be dropped. 44 | cd ${SCRIPT_ROOT} 45 | ${CODEGEN_PKG}/generate-groups.sh "all" \ 46 | ms-operator/pkg/client ms-operator/pkg/apis \ 47 | v1 \ 48 | --output-base "${TEMP_DIR}" \ 49 | --go-header-file hack/boilerplate.go.txt 50 | 51 | 52 | # Notice: The code in code-generator does not generate defaulter by default. 53 | # We need to build binary from vendor cmd folder. 54 | # echo "Building defaulter-gen" 55 | # go build -o defaulter-gen ${CODEGEN_PKG}/cmd/defaulter-gen 56 | 57 | # ${GOPATH}/bin/defaulter-gen is automatically built from ${CODEGEN_PKG}/generate-groups.sh 58 | echo "Generating defaulters for mindspore v1" 59 | ${GOPATH}/bin/defaulter-gen --input-dirs ms-operator/pkg/apis/v1 \ 60 | -O zz_generated.defaults \ 61 | --output-package ms-operator/pkg/apis/v1 \ 62 | --go-header-file hack/boilerplate.go.txt "$@" 63 | -------------------------------------------------------------------------------- /pkg/apis/v1/common.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2022 Huawei Technologies Co., Ltd. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package v1 18 | 19 | // SuccessPolicy is the success policy. 20 | type SuccessPolicy string 21 | 22 | const ( 23 | SuccessPolicyDefault SuccessPolicy = "" 24 | SuccessPolicyAllWorkers SuccessPolicy = "AllWorkers" 25 | ) 26 | -------------------------------------------------------------------------------- /pkg/apis/v1/constants.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2022 Huawei Technologies Co., Ltd. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package v1 18 | 19 | import ( 20 | common "github.com/kubeflow/common/pkg/apis/common/v1" 21 | ) 22 | 23 | const ( 24 | // DefaultPortName is name of the port used to communicate between Scheduler, PS and 25 | // workers. 26 | DefaultPortName = "msjob-port" 27 | // DefaultContainerName is the name of the MSJob container. 28 | DefaultContainerName = "mindspore" 29 | // DefaultPort is default value of the port. 30 | DefaultPort = 2222 31 | // DefaultRestartPolicy is default RestartPolicy for MSReplicaSpec. 32 | DefaultRestartPolicy = common.RestartPolicyNever 33 | // Kind is the kind name. 34 | Kind = "MSJob" 35 | // Plural is the Plural for MSJob. 36 | Plural = "msjobs" 37 | // Singular is the singular for MSJob. 38 | Singular = "msjob" 39 | // FrameworkName is the name of the ML Framework 40 | FrameworkName = "mindspore" 41 | ) 42 | -------------------------------------------------------------------------------- /pkg/apis/v1/defaults.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2022 Huawei Technologies Co., Ltd. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package v1 18 | 19 | import ( 20 | "strings" 21 | 22 | "k8s.io/apimachinery/pkg/runtime" 23 | 24 | commonv1 "github.com/kubeflow/common/pkg/apis/common/v1" 25 | v1 "k8s.io/api/core/v1" 26 | ) 27 | 28 | // Int32 is a helper routine that allocates a new int32 value 29 | // to store v and returns a pointer to it. 30 | func Int32(v int32) *int32 { 31 | return &v 32 | } 33 | 34 | // addDefaultingFuncs is used to register default funcs 35 | func addDefaultingFuncs(scheme *runtime.Scheme) error { 36 | return RegisterDefaults(scheme) 37 | } 38 | 39 | // setDefaultPort sets the default ports for mindspore container. 40 | func setDefaultPort(spec *v1.PodSpec) { 41 | index := 0 42 | for i, container := range spec.Containers { 43 | if container.Name == DefaultContainerName { 44 | index = i 45 | break 46 | 47 | } 48 | } 49 | hasMSJobPort := false 50 | for _, port := range spec.Containers[index].Ports { 51 | if port.Name == DefaultPortName { 52 | hasMSJobPort = true 53 | break 54 | } 55 | } 56 | if !hasMSJobPort { 57 | spec.Containers[index].Ports = append(spec.Containers[index].Ports, v1.ContainerPort{ 58 | Name: DefaultPortName, 59 | ContainerPort: DefaultPort, 60 | }) 61 | } 62 | } 63 | 64 | func setDefaultReplicas(spec *commonv1.ReplicaSpec) { 65 | if spec.Replicas == nil { 66 | spec.Replicas = Int32(1) 67 | } 68 | if spec.RestartPolicy == "" { 69 | spec.RestartPolicy = DefaultRestartPolicy 70 | } 71 | } 72 | 73 | // setTypeNamesToCamelCase sets the name of all replica types from any case to correct case. 74 | func setTypeNamesToCamelCase(msJob *MSJob) { 75 | setTypeNameToCamelCase(msJob, MSReplicaTypePS) 76 | setTypeNameToCamelCase(msJob, MSReplicaTypeWorker) 77 | setTypeNameToCamelCase(msJob, MSReplicaTypeScheduler) 78 | } 79 | 80 | // setTypeNameToCamelCase sets the name of the replica type from any case to correct case. 81 | // E.g. from ps to PS; from WORKER to Worker. 82 | func setTypeNameToCamelCase(msJob *MSJob, typ commonv1.ReplicaType) { 83 | for t := range msJob.Spec.MSReplicaSpecs { 84 | if strings.EqualFold(string(t), string(typ)) && t != typ { 85 | spec := msJob.Spec.MSReplicaSpecs[t] 86 | delete(msJob.Spec.MSReplicaSpecs, t) 87 | msJob.Spec.MSReplicaSpecs[typ] = spec 88 | return 89 | } 90 | } 91 | } 92 | 93 | // SetDefaults_MSJob sets any unspecified values to defaults. 94 | func SetDefaults_MSJob(msjob *MSJob) { 95 | // Set default cleanpod policy to Running. 96 | if msjob.Spec.RunPolicy.CleanPodPolicy == nil { 97 | running := commonv1.CleanPodPolicyRunning 98 | msjob.Spec.RunPolicy.CleanPodPolicy = &running 99 | } 100 | // Set default success policy to "". 101 | if msjob.Spec.SuccessPolicy == nil { 102 | defaultPolicy := SuccessPolicyDefault 103 | msjob.Spec.SuccessPolicy = &defaultPolicy 104 | } 105 | 106 | // Update the key of MSReplicaSpecs to camel case. 107 | setTypeNamesToCamelCase(msjob) 108 | 109 | for _, spec := range msjob.Spec.MSReplicaSpecs { 110 | // Set default replicas to 1. 111 | setDefaultReplicas(spec) 112 | // Set default port to mindspore container. 113 | setDefaultPort(&spec.Template.Spec) 114 | } 115 | } 116 | -------------------------------------------------------------------------------- /pkg/apis/v1/doc.go: -------------------------------------------------------------------------------- 1 | // Copyright 2018 The Kubeflow Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | // +k8s:defaulter-gen=TypeMeta 16 | // +k8s:openapi-gen=true 17 | 18 | // Package v1 is the v1 version of the API. 19 | // +groupName=mindspore.gitee.com 20 | package v1 21 | -------------------------------------------------------------------------------- /pkg/apis/v1/groupversion_info.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2022 Huawei Technologies Co., Ltd. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | // Package v1 contains API Schema definitions for the mindspore v1 API group 18 | //+kubebuilder:object:generate=true 19 | //+groupName=mindspore.gitee.com 20 | package v1 21 | 22 | import ( 23 | "k8s.io/apimachinery/pkg/runtime/schema" 24 | "sigs.k8s.io/controller-runtime/pkg/scheme" 25 | ) 26 | 27 | var ( 28 | // GroupVersion is group version used to register these objects 29 | GroupVersion = schema.GroupVersion{Group: "mindspore.gitee.com", Version: "v1"} 30 | 31 | // SchemeBuilder is used to add go types to the GroupVersionKind scheme 32 | SchemeBuilder = &scheme.Builder{GroupVersion: GroupVersion} 33 | 34 | // AddToScheme adds the types in this group-version to the given scheme. 35 | AddToScheme = SchemeBuilder.AddToScheme 36 | ) 37 | -------------------------------------------------------------------------------- /pkg/apis/v1/msjob_types.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2022 Huawei Technologies Co., Ltd. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package v1 18 | 19 | import ( 20 | commonv1 "github.com/kubeflow/common/pkg/apis/common/v1" 21 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 22 | ) 23 | 24 | // +genclient 25 | // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object 26 | // +resource:path=msjob 27 | //+kubebuilder:object:root=true 28 | //+kubebuilder:subresource:status 29 | //+kubebuilder:printcolumn:name="State",type=string,JSONPath=`.status.conditions[-1:].type` 30 | //+kubebuilder:printcolumn:name="Age",type=date,JSONPath=`.metadata.creationTimestamp` 31 | // MSJob is the Schema for the msjobs API 32 | type MSJob struct { 33 | // Standard Kubernetes type metadata. 34 | metav1.TypeMeta `json:",inline"` 35 | 36 | // +optional 37 | metav1.ObjectMeta `json:"metadata,omitempty"` 38 | 39 | // Specification of the desired state of the MSJob. 40 | // +optional 41 | Spec MSJobSpec `json:"spec,omitempty"` 42 | 43 | // Most recently observed status of the MSJob. 44 | // Populated by the system. 45 | // Read-only. 46 | // +optional 47 | Status commonv1.JobStatus `json:"status,omitempty"` 48 | } 49 | 50 | // MSJobSpec defines the desired state of MSJob 51 | type MSJobSpec struct { 52 | // RunPolicy encapsulates various runtime policies of the distributed training 53 | // job, for example how to clean up resources and how long the job can stay 54 | // active. 55 | //+kubebuilder:validation:Optional 56 | RunPolicy commonv1.RunPolicy `json:"runPolicy"` 57 | 58 | // SuccessPolicy defines the policy to mark the MSJob as succeeded. 59 | // Default to "", using the default rules. 60 | // +optional 61 | SuccessPolicy *SuccessPolicy `json:"successPolicy,omitempty"` 62 | 63 | // A map of MSReplicaType (type) to ReplicaSpec (value). Specifies the MS cluster configuration. 64 | // For example, 65 | // { 66 | // "Scheduler": ReplacaSpec, 67 | // "PS": ReplicaSpec, 68 | // "Worker": ReplicaSpec, 69 | // } 70 | MSReplicaSpecs map[commonv1.ReplicaType]*commonv1.ReplicaSpec `json:"msReplicaSpecs"` 71 | 72 | // A switch to enable dynamic worker 73 | EnableDynamicWorker bool `json:"enableDynamicWorker,omitempty"` 74 | } 75 | 76 | // MSReplicaType is the type for MSReplica. Can be one of: "Scheduler", 77 | // "Worker" or "PS". 78 | 79 | const ( 80 | // MSReplicaTypePS is the type for parameter servers of distributed MS. 81 | MSReplicaTypePS commonv1.ReplicaType = "PS" 82 | 83 | // MSReplicaTypeWorker is the type for workers of distributed MS. 84 | // This is also used for non-distributed MS. 85 | MSReplicaTypeWorker commonv1.ReplicaType = "Worker" 86 | 87 | // MSReplicaTypeScheduler is the type for Scheduler of distributed MS. 88 | MSReplicaTypeScheduler commonv1.ReplicaType = "Scheduler" 89 | ) 90 | 91 | // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object 92 | // +resource:path=msjobs 93 | //+kubebuilder:object:root=true 94 | // MSJobList contains a list of MSJob 95 | type MSJobList struct { 96 | metav1.TypeMeta `json:",inline"` 97 | metav1.ListMeta `json:"metadata,omitempty"` 98 | Items []MSJob `json:"items"` 99 | } 100 | 101 | func init() { 102 | SchemeBuilder.Register(&MSJob{}, &MSJobList{}) 103 | SchemeBuilder.SchemeBuilder.Register(addDefaultingFuncs) 104 | } 105 | -------------------------------------------------------------------------------- /pkg/apis/v1/zz_generated.deepcopy.go: -------------------------------------------------------------------------------- 1 | //go:build !ignore_autogenerated 2 | // +build !ignore_autogenerated 3 | 4 | /* 5 | Copyright 2022 Huawei Technologies Co., Ltd. 6 | 7 | Licensed under the Apache License, Version 2.0 (the "License"); 8 | you may not use this file except in compliance with the License. 9 | You may obtain a copy of the License at 10 | 11 | http://www.apache.org/licenses/LICENSE-2.0 12 | 13 | Unless required by applicable law or agreed to in writing, software 14 | distributed under the License is distributed on an "AS IS" BASIS, 15 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | See the License for the specific language governing permissions and 17 | limitations under the License. 18 | */ 19 | 20 | // Code generated by controller-gen. DO NOT EDIT. 21 | 22 | package v1 23 | 24 | import ( 25 | commonv1 "github.com/kubeflow/common/pkg/apis/common/v1" 26 | "k8s.io/apimachinery/pkg/runtime" 27 | ) 28 | 29 | // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. 30 | func (in *MSJob) DeepCopyInto(out *MSJob) { 31 | *out = *in 32 | out.TypeMeta = in.TypeMeta 33 | in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) 34 | in.Spec.DeepCopyInto(&out.Spec) 35 | in.Status.DeepCopyInto(&out.Status) 36 | } 37 | 38 | // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new MSJob. 39 | func (in *MSJob) DeepCopy() *MSJob { 40 | if in == nil { 41 | return nil 42 | } 43 | out := new(MSJob) 44 | in.DeepCopyInto(out) 45 | return out 46 | } 47 | 48 | // DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. 49 | func (in *MSJob) DeepCopyObject() runtime.Object { 50 | if c := in.DeepCopy(); c != nil { 51 | return c 52 | } 53 | return nil 54 | } 55 | 56 | // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. 57 | func (in *MSJobList) DeepCopyInto(out *MSJobList) { 58 | *out = *in 59 | out.TypeMeta = in.TypeMeta 60 | in.ListMeta.DeepCopyInto(&out.ListMeta) 61 | if in.Items != nil { 62 | in, out := &in.Items, &out.Items 63 | *out = make([]MSJob, len(*in)) 64 | for i := range *in { 65 | (*in)[i].DeepCopyInto(&(*out)[i]) 66 | } 67 | } 68 | } 69 | 70 | // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new MSJobList. 71 | func (in *MSJobList) DeepCopy() *MSJobList { 72 | if in == nil { 73 | return nil 74 | } 75 | out := new(MSJobList) 76 | in.DeepCopyInto(out) 77 | return out 78 | } 79 | 80 | // DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. 81 | func (in *MSJobList) DeepCopyObject() runtime.Object { 82 | if c := in.DeepCopy(); c != nil { 83 | return c 84 | } 85 | return nil 86 | } 87 | 88 | // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. 89 | func (in *MSJobSpec) DeepCopyInto(out *MSJobSpec) { 90 | *out = *in 91 | in.RunPolicy.DeepCopyInto(&out.RunPolicy) 92 | if in.SuccessPolicy != nil { 93 | in, out := &in.SuccessPolicy, &out.SuccessPolicy 94 | *out = new(SuccessPolicy) 95 | **out = **in 96 | } 97 | if in.MSReplicaSpecs != nil { 98 | in, out := &in.MSReplicaSpecs, &out.MSReplicaSpecs 99 | *out = make(map[commonv1.ReplicaType]*commonv1.ReplicaSpec, len(*in)) 100 | for key, val := range *in { 101 | var outVal *commonv1.ReplicaSpec 102 | if val == nil { 103 | (*out)[key] = nil 104 | } else { 105 | in, out := &val, &outVal 106 | *out = new(commonv1.ReplicaSpec) 107 | (*in).DeepCopyInto(*out) 108 | } 109 | (*out)[key] = outVal 110 | } 111 | } 112 | } 113 | 114 | // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new MSJobSpec. 115 | func (in *MSJobSpec) DeepCopy() *MSJobSpec { 116 | if in == nil { 117 | return nil 118 | } 119 | out := new(MSJobSpec) 120 | in.DeepCopyInto(out) 121 | return out 122 | } 123 | -------------------------------------------------------------------------------- /pkg/apis/v1/zz_generated.defaults.go: -------------------------------------------------------------------------------- 1 | //go:build !ignore_autogenerated 2 | // +build !ignore_autogenerated 3 | 4 | /* 5 | Copyright 2022 Huawei Technologies Co., Ltd. 6 | 7 | Licensed under the Apache License, Version 2.0 (the "License"); 8 | you may not use this file except in compliance with the License. 9 | You may obtain a copy of the License at 10 | 11 | http://www.apache.org/licenses/LICENSE-2.0 12 | 13 | Unless required by applicable law or agreed to in writing, software 14 | distributed under the License is distributed on an "AS IS" BASIS, 15 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | See the License for the specific language governing permissions and 17 | limitations under the License. 18 | */ 19 | // Code generated by defaulter-gen. DO NOT EDIT. 20 | 21 | package v1 22 | 23 | import ( 24 | runtime "k8s.io/apimachinery/pkg/runtime" 25 | ) 26 | 27 | // RegisterDefaults adds defaulters functions to the given scheme. 28 | // Public to allow building arbitrary schemes. 29 | // All generated defaulters are covering - they call all nested defaulters. 30 | func RegisterDefaults(scheme *runtime.Scheme) error { 31 | scheme.AddTypeDefaultingFunc(&MSJob{}, func(obj interface{}) { SetObjectDefaults_MSJob(obj.(*MSJob)) }) 32 | scheme.AddTypeDefaultingFunc(&MSJobList{}, func(obj interface{}) { SetObjectDefaults_MSJobList(obj.(*MSJobList)) }) 33 | return nil 34 | } 35 | 36 | func SetObjectDefaults_MSJob(in *MSJob) { 37 | SetDefaults_MSJob(in) 38 | } 39 | 40 | func SetObjectDefaults_MSJobList(in *MSJobList) { 41 | for i := range in.Items { 42 | a := &in.Items[i] 43 | SetObjectDefaults_MSJob(a) 44 | } 45 | } 46 | -------------------------------------------------------------------------------- /pkg/controllers/v1/msjob_controller.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2022 Huawei Technologies Co., Ltd. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package controllers 18 | 19 | import ( 20 | "context" 21 | "fmt" 22 | "reflect" 23 | "strconv" 24 | "strings" 25 | "time" 26 | 27 | mindsporev1 "ms-operator/pkg/apis/v1" 28 | 29 | "github.com/go-logr/logr" 30 | commonv1 "github.com/kubeflow/common/pkg/apis/common/v1" 31 | "github.com/kubeflow/common/pkg/controller.v1/common" 32 | "github.com/kubeflow/common/pkg/controller.v1/control" 33 | "github.com/kubeflow/common/pkg/controller.v1/expectation" 34 | "github.com/kubeflow/common/pkg/core" 35 | commonutil "github.com/kubeflow/common/pkg/util" 36 | "github.com/kubeflow/common/pkg/util/k8sutil" 37 | utillabels "github.com/kubeflow/common/pkg/util/labels" 38 | train_util "github.com/kubeflow/common/pkg/util/train" 39 | trainingoperatorcommon "github.com/kubeflow/training-operator/pkg/common" 40 | "github.com/kubeflow/training-operator/pkg/common/util" 41 | "github.com/prometheus/client_golang/prometheus" 42 | "github.com/prometheus/client_golang/prometheus/promauto" 43 | "github.com/sirupsen/logrus" 44 | corev1 "k8s.io/api/core/v1" 45 | v1 "k8s.io/api/core/v1" 46 | "k8s.io/apimachinery/pkg/api/errors" 47 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 48 | "k8s.io/apimachinery/pkg/runtime" 49 | "k8s.io/apimachinery/pkg/runtime/schema" 50 | "k8s.io/apimachinery/pkg/types" 51 | utilruntime "k8s.io/apimachinery/pkg/util/runtime" 52 | "k8s.io/client-go/informers" 53 | kubeclientset "k8s.io/client-go/kubernetes" 54 | "k8s.io/client-go/tools/record" 55 | ctrl "sigs.k8s.io/controller-runtime" 56 | "sigs.k8s.io/controller-runtime/pkg/client" 57 | "sigs.k8s.io/controller-runtime/pkg/controller" 58 | "sigs.k8s.io/controller-runtime/pkg/event" 59 | "sigs.k8s.io/controller-runtime/pkg/handler" 60 | "sigs.k8s.io/controller-runtime/pkg/log" 61 | "sigs.k8s.io/controller-runtime/pkg/manager" 62 | "sigs.k8s.io/controller-runtime/pkg/predicate" 63 | "sigs.k8s.io/controller-runtime/pkg/source" 64 | "volcano.sh/apis/pkg/apis/scheduling/v1beta1" 65 | volcanoclient "volcano.sh/apis/pkg/client/clientset/versioned" 66 | ) 67 | 68 | const ( 69 | // msJobSucceededReason is added in a msjob when it is succeeded. 70 | msJobSucceededReason = "MSJobSucceeded" 71 | // msJobRunningReason is added in a msjob when it is running. 72 | msJobRunningReason = "MSJobRunning" 73 | // msJobFailedReason is added in a msjob when it is failed. 74 | msJobFailedReason = "MSJobFailed" 75 | // msJobRestarting is added in a msjob when it is restarting. 76 | msJobRestartingReason = "MSJobRestarting" 77 | 78 | FailedDeleteJobReason = "FailedDeleteJob" 79 | SuccessfulDeleteJobReason = "SuccessfulDeleteJob" 80 | 81 | controllerName = "msjob-controller" 82 | 83 | // labels for pods and servers. 84 | msReplicaTypeLabel = "replica-type" 85 | msReplicaIndexLabel = "replica-index" 86 | // volcanoTaskSpecKey task spec key used in pod annotation when EnableGangScheduling is true 87 | volcanoTaskSpecKey = "volcano.sh/task-spec" 88 | 89 | // gang scheduler name. 90 | gangSchedulerName = "volcano" 91 | //the environment variable name of Mindspore cluster spec. 92 | msServerNum = "MS_SERVER_NUM" 93 | msWorkerNum = "MS_WORKER_NUM" 94 | msSchedHost = "MS_SCHED_HOST" 95 | msSchedPort = "MS_SCHED_PORT" 96 | msRole = "MS_ROLE" 97 | msNodeId = "MS_NODE_ID" 98 | 99 | // exitedWithCodeReason is the normal reason when the pod is exited because of the exit code. 100 | exitedWithCodeReason = "ExitedWithCode" 101 | // podTemplateRestartPolicyReason is the warning reason when the restart 102 | // policy is set in pod template. 103 | podTemplateRestartPolicyReason = "SettedPodTemplateRestartPolicy" 104 | // podTemplateSchedulerNameReason is the warning reason when other scheduler name is set 105 | // in pod templates with gang-scheduling enabled 106 | podTemplateSchedulerNameReason = "SettedPodTemplateSchedulerName" 107 | // gangSchedulingPodGroupAnnotation is the annotation key used by batch schedulers 108 | gangSchedulingPodGroupAnnotation = "scheduling.k8s.io/group-name" 109 | ) 110 | 111 | var ( 112 | msRoleMap = map[string]string{ 113 | "scheduler": "MS_SCHED", 114 | "ps": "MS_PSERVER", 115 | "worker": "MS_WORKER", 116 | } 117 | succeededServiceCreationCount = promauto.With(prometheus.NewRegistry()).NewCounter(prometheus.CounterOpts{ 118 | Name: "succeeded_service_creation_total", 119 | Help: "The total number of succeeded service creation", 120 | }) 121 | failedServiceCreationCount = promauto.With(prometheus.NewRegistry()).NewCounter(prometheus.CounterOpts{ 122 | Name: "failed_service_creation_total", 123 | Help: "The total number of failed service creation", 124 | }) 125 | ) 126 | 127 | func NewReconciler(mgr manager.Manager, enableGangScheduling bool) *MSJobReconciler { 128 | r := &MSJobReconciler{ 129 | Client: mgr.GetClient(), 130 | Scheme: mgr.GetScheme(), 131 | recorder: mgr.GetEventRecorderFor(controllerName), 132 | Log: log.Log, 133 | } 134 | 135 | cfg := mgr.GetConfig() 136 | kubeClientSet := kubeclientset.NewForConfigOrDie(cfg) 137 | volcanoClientSet := volcanoclient.NewForConfigOrDie(cfg) 138 | sharedInformers := informers.NewSharedInformerFactory(kubeClientSet, 0) 139 | priorityClassInformer := sharedInformers.Scheduling().V1beta1().PriorityClasses() 140 | 141 | r.JobController = common.JobController{ 142 | Controller: r, 143 | Expectations: expectation.NewControllerExpectations(), 144 | Config: common.JobControllerConfiguration{EnableGangScheduling: enableGangScheduling}, 145 | WorkQueue: &util.FakeWorkQueue{}, 146 | Recorder: r.recorder, 147 | KubeClientSet: kubeClientSet, 148 | VolcanoClientSet: volcanoClientSet, 149 | PriorityClassLister: priorityClassInformer.Lister(), 150 | PriorityClassInformerSynced: priorityClassInformer.Informer().HasSynced, 151 | PodControl: control.RealPodControl{KubeClient: kubeClientSet, Recorder: r.recorder}, 152 | ServiceControl: control.RealServiceControl{KubeClient: kubeClientSet, Recorder: r.recorder}, 153 | } 154 | 155 | return r 156 | } 157 | 158 | // MSJobReconciler reconciles a MSJob object 159 | type MSJobReconciler struct { 160 | common.JobController 161 | client.Client 162 | Scheme *runtime.Scheme 163 | recorder record.EventRecorder 164 | Log logr.Logger 165 | } 166 | 167 | //+kubebuilder:rbac:groups=mindspore.gitee.com,resources=msjobs,verbs=get;list;watch;create;update;patch;delete 168 | //+kubebuilder:rbac:groups=mindspore.gitee.com,resources=msjobs/status,verbs=get;update;patch 169 | //+kubebuilder:rbac:groups=mindspore.gitee.com,resources=msjobs/finalizers,verbs=update 170 | //+kubebuilder:rbac:groups="",resources=pods,verbs=get;list;watch;create;update;patch;delete 171 | //+kubebuilder:rbac:groups="",resources=services,verbs=get;list;watch;create;delete 172 | 173 | // Reconcile is part of the main kubernetes reconciliation loop which aims to 174 | // move the current state of the cluster closer to the desired state. 175 | // TODO(user): Modify the Reconcile function to compare the state specified by 176 | // the MSJob object against the actual cluster state, and then 177 | // perform operations to make the cluster state reflect the state specified by 178 | // the user. 179 | // 180 | // For more details, check Reconcile and its Result here: 181 | // - https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.11.0/pkg/reconcile 182 | func (r *MSJobReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { 183 | _ = log.FromContext(ctx) 184 | logger := r.Log.WithValues(mindsporev1.Singular, req.NamespacedName) 185 | 186 | msjob := &mindsporev1.MSJob{} 187 | err := r.Get(ctx, req.NamespacedName, msjob) 188 | if err != nil { 189 | logger.Info(err.Error(), "unable to fetch MSJob", req.NamespacedName.String()) 190 | return ctrl.Result{}, client.IgnoreNotFound(err) 191 | } 192 | 193 | if err = validateV1ReplicaSpecs(msjob.Spec.MSReplicaSpecs); err != nil { 194 | logger.Info(err.Error(), "MSJob failed validation", req.NamespacedName.String()) 195 | } 196 | 197 | // Check if reconciliation is needed 198 | jobKey, err := common.KeyFunc(msjob) 199 | if err != nil { 200 | utilruntime.HandleError(fmt.Errorf("couldn't get jobKey for job object %#v: %v", msjob, err)) 201 | } 202 | 203 | replicaTypes := util.GetReplicaTypes(msjob.Spec.MSReplicaSpecs) 204 | needReconcile := util.SatisfiedExpectations(r.Expectations, jobKey, replicaTypes) 205 | 206 | if !needReconcile || msjob.GetDeletionTimestamp() != nil { 207 | logger.Info("reconcile cancelled, job does not need to do reconcile or has been deleted", 208 | "sync", needReconcile, "deleted", msjob.GetDeletionTimestamp() != nil) 209 | return ctrl.Result{}, nil 210 | } 211 | 212 | // Set default priorities to msjob 213 | r.Scheme.Default(msjob) 214 | 215 | // Use common to reconcile the job related pod and service 216 | err = r.ReconcileJobs(msjob, msjob.Spec.MSReplicaSpecs, msjob.Status, &msjob.Spec.RunPolicy) 217 | if err != nil { 218 | logrus.Warnf("Reconcile MindSpore Job error %v", err) 219 | return ctrl.Result{}, err 220 | } 221 | 222 | return ctrl.Result{}, nil 223 | } 224 | 225 | // SetupWithManager sets up the controller with the Manager. 226 | func (r *MSJobReconciler) SetupWithManager(mgr ctrl.Manager) error { 227 | 228 | c, err := controller.New(r.ControllerName(), mgr, controller.Options{ 229 | Reconciler: r, 230 | }) 231 | 232 | if err != nil { 233 | return err 234 | } 235 | 236 | // using onOwnerCreateFunc is easier to set defaults 237 | if err = c.Watch(&source.Kind{Type: &mindsporev1.MSJob{}}, &handler.EnqueueRequestForObject{}, 238 | predicate.Funcs{CreateFunc: r.onOwnerCreateFunc()}, 239 | ); err != nil { 240 | return err 241 | } 242 | 243 | // inject watching for job related pod 244 | if err = c.Watch(&source.Kind{Type: &corev1.Pod{}}, &handler.EnqueueRequestForOwner{ 245 | IsController: true, 246 | OwnerType: &mindsporev1.MSJob{}, 247 | }, predicate.Funcs{ 248 | CreateFunc: util.OnDependentCreateFunc(r.Expectations), 249 | UpdateFunc: util.OnDependentUpdateFunc(&r.JobController), 250 | DeleteFunc: util.OnDependentDeleteFunc(r.Expectations), 251 | }); err != nil { 252 | return err 253 | } 254 | 255 | // inject watching for job related service 256 | if err = c.Watch(&source.Kind{Type: &corev1.Service{}}, &handler.EnqueueRequestForOwner{ 257 | IsController: true, 258 | OwnerType: &mindsporev1.MSJob{}, 259 | }, predicate.Funcs{ 260 | CreateFunc: util.OnDependentCreateFunc(r.Expectations), 261 | UpdateFunc: util.OnDependentUpdateFunc(&r.JobController), 262 | DeleteFunc: util.OnDependentDeleteFunc(r.Expectations), 263 | }); err != nil { 264 | return err 265 | } 266 | 267 | return nil 268 | } 269 | 270 | func (jc *MSJobReconciler) ReconcileJobs( 271 | job interface{}, 272 | replicas map[commonv1.ReplicaType]*commonv1.ReplicaSpec, 273 | jobStatus commonv1.JobStatus, 274 | runPolicy *commonv1.RunPolicy) error { 275 | 276 | metaObject, ok := job.(metav1.Object) 277 | jobName := metaObject.GetName() 278 | if !ok { 279 | return fmt.Errorf("job is not of type metav1.Object") 280 | } 281 | runtimeObject, ok := job.(runtime.Object) 282 | if !ok { 283 | return fmt.Errorf("job is not of type runtime.Object") 284 | } 285 | jobKey, err := common.KeyFunc(job) 286 | if err != nil { 287 | utilruntime.HandleError(fmt.Errorf("Couldn't get key for job object %#v: %v", job, err)) 288 | return err 289 | } 290 | // Reset expectations 291 | // 1. Since `ReconcileJobs` is called, we expect that previous expectations are all satisfied, 292 | // and it's safe to reset the expectations 293 | // 2. Reset expectations can avoid dirty data such as `expectedDeletion = -1` 294 | // (pod or service was deleted unexpectedly) 295 | jc.ResetExpectations(jobKey, replicas) 296 | 297 | logrus.Infof("Reconciling for job %s", metaObject.GetName()) 298 | pods, err := jc.Controller.GetPodsForJob(job) 299 | if err != nil { 300 | logrus.Warnf("GetPodsForJob error %v", err) 301 | return err 302 | } 303 | 304 | services, err := jc.Controller.GetServicesForJob(job) 305 | if err != nil { 306 | logrus.Warnf("GetServicesForJob error %v", err) 307 | return err 308 | } 309 | 310 | oldStatus := jobStatus.DeepCopy() 311 | if commonutil.IsSucceeded(jobStatus) || commonutil.IsFailed(jobStatus) { 312 | // If the Job is succeed or failed, delete all pods and services. 313 | if err := jc.DeletePodsAndServices(runPolicy, job, pods); err != nil { 314 | return err 315 | } 316 | 317 | if jc.Config.EnableGangScheduling { 318 | jc.Recorder.Event(runtimeObject, v1.EventTypeNormal, "JobTerminated", "Job has been terminated. Deleting PodGroup") 319 | if err := jc.DeletePodGroup(metaObject); err != nil { 320 | jc.Recorder.Eventf(runtimeObject, v1.EventTypeWarning, "FailedDeletePodGroup", "Error deleting: %v", err) 321 | return err 322 | } else { 323 | jc.Recorder.Eventf(runtimeObject, v1.EventTypeNormal, "SuccessfulDeletePodGroup", "Deleted PodGroup: %v", jobName) 324 | } 325 | } 326 | 327 | if err := jc.CleanupJob(runPolicy, jobStatus, job); err != nil { 328 | return err 329 | } 330 | 331 | // At this point the pods may have been deleted. 332 | // 1) If the job succeeded, we manually set the replica status. 333 | // 2) If any replicas are still active, set their status to succeeded. 334 | if commonutil.IsSucceeded(jobStatus) { 335 | for rtype := range jobStatus.ReplicaStatuses { 336 | jobStatus.ReplicaStatuses[rtype].Succeeded += jobStatus.ReplicaStatuses[rtype].Active 337 | jobStatus.ReplicaStatuses[rtype].Active = 0 338 | } 339 | } 340 | 341 | // No need to update the job status if the status hasn't changed since last time. 342 | if !reflect.DeepEqual(*oldStatus, jobStatus) { 343 | return jc.Controller.UpdateJobStatusInApiServer(job, &jobStatus) 344 | } 345 | 346 | return nil 347 | } 348 | 349 | // retrieve the previous number of retry 350 | previousRetry := jc.WorkQueue.NumRequeues(jobKey) 351 | 352 | activePods := k8sutil.FilterActivePods(pods) 353 | 354 | core.RecordAbnormalPods(activePods, runtimeObject, jc.Recorder) 355 | 356 | active := int32(len(activePods)) 357 | failed := k8sutil.FilterPodCount(pods, v1.PodFailed) 358 | totalReplicas := k8sutil.GetTotalReplicas(replicas) 359 | prevReplicasFailedNum := k8sutil.GetTotalFailedReplicas(jobStatus.ReplicaStatuses) 360 | 361 | var failureMessage string 362 | jobExceedsLimit := false 363 | exceedsBackoffLimit := false 364 | pastBackoffLimit := false 365 | 366 | if runPolicy.BackoffLimit != nil { 367 | jobHasNewFailure := failed > prevReplicasFailedNum 368 | // new failures happen when status does not reflect the failures and active 369 | // is different than parallelism, otherwise the previous controller loop 370 | // failed updating status so even if we pick up failure it is not a new one 371 | exceedsBackoffLimit = jobHasNewFailure && (active != totalReplicas) && 372 | (int32(previousRetry)+1 > *runPolicy.BackoffLimit) 373 | 374 | pastBackoffLimit, err = jc.PastBackoffLimit(jobName, runPolicy, replicas, pods) 375 | if err != nil { 376 | return err 377 | } 378 | } 379 | 380 | if exceedsBackoffLimit || pastBackoffLimit { 381 | // check if the number of pod restart exceeds backoff (for restart OnFailure only) 382 | // OR if the number of failed jobs increased since the last syncJob 383 | jobExceedsLimit = true 384 | failureMessage = fmt.Sprintf("Job %s has failed because it has reached the specified backoff limit", jobName) 385 | } else if jc.PastActiveDeadline(runPolicy, jobStatus) { 386 | failureMessage = fmt.Sprintf("Job %s has failed because it was active longer than specified deadline", jobName) 387 | jobExceedsLimit = true 388 | } 389 | 390 | if jobExceedsLimit { 391 | // Set job completion time before resource cleanup 392 | if jobStatus.CompletionTime == nil { 393 | now := metav1.Now() 394 | jobStatus.CompletionTime = &now 395 | } 396 | 397 | // If the Job exceeds backoff limit or is past active deadline 398 | // delete all pods and services, then set the status to failed 399 | if err := jc.DeletePodsAndServices(runPolicy, job, pods); err != nil { 400 | return err 401 | } 402 | 403 | if err := jc.CleanupJob(runPolicy, jobStatus, job); err != nil { 404 | return err 405 | } 406 | 407 | if jc.Config.EnableGangScheduling { 408 | jc.Recorder.Event(runtimeObject, v1.EventTypeNormal, "JobTerminated", "Job has been terminated. Deleting PodGroup") 409 | if err := jc.DeletePodGroup(metaObject); err != nil { 410 | jc.Recorder.Eventf(runtimeObject, v1.EventTypeWarning, "FailedDeletePodGroup", "Error deleting: %v", err) 411 | return err 412 | } else { 413 | jc.Recorder.Eventf(runtimeObject, v1.EventTypeNormal, "SuccessfulDeletePodGroup", "Deleted PodGroup: %v", jobName) 414 | } 415 | } 416 | 417 | jc.Recorder.Event(runtimeObject, v1.EventTypeNormal, commonutil.JobFailedReason, failureMessage) 418 | 419 | if err := commonutil.UpdateJobConditions(&jobStatus, commonv1.JobFailed, commonutil.JobFailedReason, failureMessage); err != nil { 420 | logrus.Infof("Append job condition error: %v", err) 421 | return err 422 | } 423 | 424 | return jc.Controller.UpdateJobStatusInApiServer(job, &jobStatus) 425 | } else { 426 | // General cases which need to reconcile 427 | if jc.Config.EnableGangScheduling { 428 | minMember := totalReplicas 429 | queue := "" 430 | priorityClass := "" 431 | var minResources *v1.ResourceList 432 | 433 | if runPolicy.SchedulingPolicy != nil { 434 | if runPolicy.SchedulingPolicy.MinAvailable != nil { 435 | minMember = *runPolicy.SchedulingPolicy.MinAvailable 436 | } 437 | 438 | if runPolicy.SchedulingPolicy.Queue != "" { 439 | queue = runPolicy.SchedulingPolicy.Queue 440 | } 441 | 442 | if runPolicy.SchedulingPolicy.PriorityClass != "" { 443 | priorityClass = runPolicy.SchedulingPolicy.PriorityClass 444 | } 445 | 446 | if runPolicy.SchedulingPolicy.MinResources != nil { 447 | minResources = runPolicy.SchedulingPolicy.MinResources 448 | } 449 | } 450 | 451 | if minResources == nil { 452 | minResources = common.CalcPGMinResources(minMember, replicas, jc.PriorityClassLister.Get) 453 | } 454 | 455 | pgSpec := v1beta1.PodGroupSpec{ 456 | MinMember: minMember, 457 | Queue: queue, 458 | PriorityClassName: priorityClass, 459 | MinResources: minResources, 460 | } 461 | 462 | syncReplicas := true 463 | pg, err := jc.SyncPodGroup(metaObject, pgSpec) 464 | if err != nil { 465 | logrus.Warnf("Sync PodGroup %v: %v", jobKey, err) 466 | syncReplicas = false 467 | } 468 | 469 | // Delay pods creation until podgroup status is inqueue 470 | if pg == nil || pg.Status.Phase == "" || pg.Status.Phase == v1beta1.PodGroupPending { 471 | logrus.Warnf("PodGroup %v unschedulable", jobKey) 472 | syncReplicas = false 473 | } 474 | 475 | if !syncReplicas { 476 | now := metav1.Now() 477 | jobStatus.LastReconcileTime = &now 478 | 479 | // Update job status here to trigger a new reconciliation 480 | return jc.Controller.UpdateJobStatusInApiServer(job, &jobStatus) 481 | } 482 | } 483 | 484 | // Diff current active pods/services with replicas. 485 | // Create Scheduler pod and service first, and then Create the others pods. 486 | schedulerRoleSpec, hasSchedulerRole := replicas[mindsporev1.MSReplicaTypeScheduler] 487 | if hasSchedulerRole { 488 | err := jc.Controller.ReconcileServices(metaObject, services, mindsporev1.MSReplicaTypeScheduler, schedulerRoleSpec) 489 | if err != nil { 490 | logrus.Warnf("ReconcileServices error %v", err) 491 | return err 492 | } 493 | 494 | err = jc.Controller.ReconcilePods(metaObject, &jobStatus, pods, mindsporev1.MSReplicaTypeScheduler, schedulerRoleSpec, replicas) 495 | if err != nil { 496 | logrus.Warnf("ReconcilePods error %v", err) 497 | return err 498 | } 499 | } 500 | 501 | for rtype, spec := range replicas { 502 | if rtype != mindsporev1.MSReplicaTypeScheduler { 503 | err := jc.Controller.ReconcilePods(metaObject, &jobStatus, pods, rtype, spec, replicas) 504 | if err != nil { 505 | logrus.Warnf("ReconcilePods error %v", err) 506 | return err 507 | } 508 | } 509 | } 510 | } 511 | 512 | err = jc.Controller.UpdateJobStatus(job, replicas, &jobStatus) 513 | if err != nil { 514 | logrus.Warnf("UpdateJobStatus error %v", err) 515 | return err 516 | } 517 | // No need to update the job status if the status hasn't changed since last time. 518 | if !reflect.DeepEqual(*oldStatus, jobStatus) { 519 | return jc.Controller.UpdateJobStatusInApiServer(job, &jobStatus) 520 | } 521 | return nil 522 | } 523 | 524 | func (r *MSJobReconciler) ControllerName() string { 525 | return controllerName 526 | } 527 | 528 | func (r *MSJobReconciler) GetAPIGroupVersionKind() schema.GroupVersionKind { 529 | return mindsporev1.GroupVersion.WithKind(mindsporev1.Kind) 530 | } 531 | 532 | func (r *MSJobReconciler) GetAPIGroupVersion() schema.GroupVersion { 533 | return mindsporev1.GroupVersion 534 | } 535 | 536 | func (r *MSJobReconciler) GetGroupNameLabelValue() string { 537 | return mindsporev1.GroupVersion.Group 538 | } 539 | 540 | func (r *MSJobReconciler) GetJobFromInformerCache(namespace, name string) (metav1.Object, error) { 541 | msjob := &mindsporev1.MSJob{} 542 | err := r.Get(context.Background(), types.NamespacedName{ 543 | Namespace: namespace, Name: name, 544 | }, msjob) 545 | return msjob, err 546 | } 547 | 548 | func (r *MSJobReconciler) GetJobFromAPIClient(namespace, name string) (metav1.Object, error) { 549 | job := &mindsporev1.MSJob{} 550 | 551 | clientReader, err := util.GetDelegatingClientFromClient(r.Client) 552 | if err != nil { 553 | return nil, err 554 | } 555 | err = clientReader.Get(context.Background(), types.NamespacedName{Namespace: namespace, Name: name}, job) 556 | if err != nil { 557 | if errors.IsNotFound(err) { 558 | logrus.Error(err, "mindspore job not found", "namespace", namespace, "name", name) 559 | } else { 560 | logrus.Error(err, "failed to get job from api-server", "namespace", namespace, "name", name) 561 | } 562 | return nil, err 563 | } 564 | return job, nil 565 | } 566 | 567 | // GetPodsForJob returns the set of pods that this job should manage. 568 | // It also reconciles ControllerRef by adopting/orphaning. 569 | // Note that the returned Pods are pointers into the cache. 570 | func (r *MSJobReconciler) GetPodsForJob(jobObject interface{}) ([]*corev1.Pod, error) { 571 | job, ok := jobObject.(metav1.Object) 572 | if !ok { 573 | return nil, fmt.Errorf("job is not of type metav1.Object") 574 | } 575 | 576 | // Create selector. 577 | selector, err := metav1.LabelSelectorAsSelector(&metav1.LabelSelector{ 578 | MatchLabels: r.GenLabels(job.GetName()), 579 | }) 580 | 581 | if err != nil { 582 | return nil, fmt.Errorf("couldn't convert Job selector: %v", err) 583 | } 584 | // List all pods to include those that don't match the selector anymore 585 | // but have a ControllerRef pointing to this controller. 586 | podlist := &corev1.PodList{} 587 | err = r.List(context.Background(), podlist, 588 | client.MatchingLabelsSelector{Selector: selector}, client.InNamespace(job.GetNamespace())) 589 | if err != nil { 590 | return nil, err 591 | } 592 | 593 | pods := util.ConvertPodList(podlist.Items) 594 | 595 | // If any adoptions are attempted, we should first recheck for deletion 596 | // with an uncached quorum read sometime after listing Pods (see #42639). 597 | canAdoptFunc := common.RecheckDeletionTimestamp(func() (metav1.Object, error) { 598 | fresh, err := r.Controller.GetJobFromAPIClient(job.GetNamespace(), job.GetName()) 599 | if err != nil { 600 | return nil, err 601 | } 602 | if fresh.GetUID() != job.GetUID() { 603 | return nil, fmt.Errorf("original Job %v/%v is gone: got uid %v, wanted %v", job.GetNamespace(), job.GetName(), fresh.GetUID(), job.GetUID()) 604 | } 605 | return fresh, nil 606 | }) 607 | cm := control.NewPodControllerRefManager(r.PodControl, job, selector, r.Controller.GetAPIGroupVersionKind(), canAdoptFunc) 608 | return cm.ClaimPods(pods) 609 | } 610 | 611 | // GetServicesForJob returns the set of services that this job should manage. 612 | // It also reconciles ControllerRef by adopting/orphaning. 613 | // Note that the returned services are pointers into the cache. 614 | func (r *MSJobReconciler) GetServicesForJob(jobObject interface{}) ([]*corev1.Service, error) { 615 | job, ok := jobObject.(metav1.Object) 616 | if !ok { 617 | return nil, fmt.Errorf("job is not of type metav1.Object") 618 | } 619 | 620 | // Create selector 621 | selector, err := metav1.LabelSelectorAsSelector(&metav1.LabelSelector{ 622 | MatchLabels: r.GenLabels(job.GetName()), 623 | }) 624 | 625 | if err != nil { 626 | return nil, fmt.Errorf("couldn't convert Job selector: %v", err) 627 | } 628 | // List all services to include those that don't match the selector anymore 629 | // but have a ControllerRef pointing to this controller. 630 | svclist := &corev1.ServiceList{} 631 | err = r.List(context.Background(), svclist, 632 | client.MatchingLabelsSelector{Selector: selector}, client.InNamespace(job.GetNamespace())) 633 | if err != nil { 634 | return nil, fmt.Errorf("couldn't get Service: %v", err) 635 | } 636 | 637 | // If any adoptions are attempted, we should first recheck for deletion 638 | // with an uncached quorum read sometime after listing services (see #42639). 639 | canAdoptFunc := common.RecheckDeletionTimestamp(func() (metav1.Object, error) { 640 | fresh, err := r.GetJobFromInformerCache(job.GetNamespace(), job.GetName()) 641 | if err != nil { 642 | return nil, err 643 | } 644 | if fresh.GetUID() != job.GetUID() { 645 | return nil, fmt.Errorf("original Job %v/%v is gone: got uid %v, wanted %v", job.GetNamespace(), job.GetName(), fresh.GetUID(), job.GetUID()) 646 | } 647 | return fresh, nil 648 | }) 649 | cm := control.NewServiceControllerRefManager(r.ServiceControl, job, selector, r.Controller.GetAPIGroupVersionKind(), canAdoptFunc) 650 | 651 | services := util.ConvertServiceList(svclist.Items) 652 | 653 | return cm.ClaimServices(services) 654 | } 655 | 656 | func (r *MSJobReconciler) DeleteJob(job interface{}) error { 657 | msJob, ok := job.(*mindsporev1.MSJob) 658 | if !ok { 659 | return fmt.Errorf("%v is not a type of MSJob", msJob) 660 | } 661 | 662 | log := commonutil.LoggerForJob(msJob) 663 | if err := r.Delete(context.Background(), msJob); err != nil { 664 | r.recorder.Eventf(msJob, v1.EventTypeWarning, FailedDeleteJobReason, "Error deleting: %v", err) 665 | log.Errorf("failed to delete job %s/%s, %v", msJob.Namespace, msJob.Name, err) 666 | return err 667 | } 668 | 669 | r.recorder.Eventf(msJob, v1.EventTypeNormal, SuccessfulDeleteJobReason, "Deleted job: %v", msJob.Name) 670 | log.Infof("job %s/%s has been deleted", msJob.Namespace, msJob.Name) 671 | trainingoperatorcommon.DeletedJobsCounterInc(msJob.Namespace, mindsporev1.FrameworkName) 672 | return nil 673 | } 674 | 675 | func (r *MSJobReconciler) UpdateJobStatus(job interface{}, replicas map[commonv1.ReplicaType]*commonv1.ReplicaSpec, jobStatus *commonv1.JobStatus) error { 676 | msJob, ok := job.(*mindsporev1.MSJob) 677 | if !ok { 678 | return fmt.Errorf("%v is not a type of MSJob", msJob) 679 | } 680 | 681 | msJobKey, err := common.KeyFunc(msJob) 682 | if err != nil { 683 | utilruntime.HandleError(fmt.Errorf("couldn't get key for msjob object %#v: %v", msJob, err)) 684 | return err 685 | } 686 | 687 | logger := commonutil.LoggerForJob(msJob) 688 | 689 | worker0Completed, err := r.IsWorker0Completed(msJob, replicas) 690 | if err != nil { 691 | logger.Warnf("check if worker 0 completed error %v", err) 692 | return err 693 | } 694 | 695 | // Set StartTime. 696 | if jobStatus.StartTime == nil { 697 | now := metav1.Now() 698 | jobStatus.StartTime = &now 699 | // enqueue a sync to check if job past ActiveDeadlineSeconds 700 | if msJob.Spec.RunPolicy.ActiveDeadlineSeconds != nil { 701 | logger.Infof("Job with ActiveDeadlineSeconds will sync after %d seconds", *msJob.Spec.RunPolicy.ActiveDeadlineSeconds) 702 | r.WorkQueue.AddAfter(msJobKey, time.Duration(*msJob.Spec.RunPolicy.ActiveDeadlineSeconds)*time.Second) 703 | } 704 | } 705 | // iterate the replica spec based on this order 706 | allTypes := []commonv1.ReplicaType{ 707 | mindsporev1.MSReplicaTypeScheduler, 708 | mindsporev1.MSReplicaTypePS, 709 | mindsporev1.MSReplicaTypeWorker, 710 | } 711 | for _, rtype := range allTypes { 712 | if replicas[rtype] == nil { 713 | continue 714 | } 715 | spec := replicas[rtype] 716 | status := jobStatus.ReplicaStatuses[rtype] 717 | 718 | // Expect to have `replicas - succeeded` pods alive. 719 | succeeded := status.Succeeded 720 | expected := *(spec.Replicas) - succeeded 721 | running := status.Active 722 | failed := status.Failed 723 | 724 | logger.Infof("MSJob=%s/%s, ReplicaType=%s expected=%d, running=%d, failed=%d", 725 | msJob.Namespace, msJob.Name, rtype, expected, running, failed) 726 | 727 | if isWorker(rtype) { 728 | // Leave a succeeded condition for the following two cases: 729 | // 1. If default success policy is used and worker 0 has completed. 730 | // 2. If `SuccessPolicyAllWorkers` success policy is used and all workers are succeeded. 731 | if expected == 0 || (worker0Completed && *msJob.Spec.SuccessPolicy != mindsporev1.SuccessPolicyAllWorkers) { 732 | msg := fmt.Sprintf("MSJob %s/%s successfully completed.", 733 | msJob.Namespace, msJob.Name) 734 | r.recorder.Event(msJob, corev1.EventTypeNormal, msJobSucceededReason, msg) 735 | if jobStatus.CompletionTime == nil { 736 | now := metav1.Now() 737 | jobStatus.CompletionTime = &now 738 | } 739 | err := commonutil.UpdateJobConditions(jobStatus, 740 | commonv1.JobSucceeded, msJobSucceededReason, msg) 741 | if err != nil { 742 | commonutil.LoggerForJob(msJob).Infof("Append msjob condition error: %v", err) 743 | return err 744 | } 745 | trainingoperatorcommon.SuccessfulJobsCounterInc(msJob.Namespace, mindsporev1.FrameworkName) 746 | } else if running > 0 { 747 | // Some workers are still running, leave a running condition. 748 | msg := fmt.Sprintf("MSJob %s/%s is running.", 749 | msJob.Namespace, msJob.Name) 750 | err := commonutil.UpdateJobConditions(jobStatus, commonv1.JobRunning, msJobRunningReason, msg) 751 | if err != nil { 752 | commonutil.LoggerForJob(msJob).Infof("Append msjob condition error: %v", err) 753 | return err 754 | } 755 | } 756 | } 757 | 758 | if failed > 0 { 759 | restart := false 760 | for _, condition := range jobStatus.Conditions { 761 | if condition.Type == commonv1.JobRestarting { 762 | restart = true 763 | } 764 | } 765 | 766 | if restart { 767 | // job is restarting, no need to set it failed 768 | // we know it because we update the status condition when reconciling the replicas 769 | trainingoperatorcommon.RestartedJobsCounterInc(msJob.Namespace, mindsporev1.FrameworkName) 770 | } else { 771 | if msJob.Spec.EnableDynamicWorker && isWorker(rtype) { 772 | commonutil.LoggerForJob(msJob).Infof("MSJob %s/%s continues regardless %d Worker replica(s) failed as enableDynamicWorker is set true.", 773 | msJob.Namespace, msJob.Name, failed) 774 | continue 775 | } 776 | msg := fmt.Sprintf("MSJob %s/%s has failed because %d %s replica(s) failed.", 777 | msJob.Namespace, msJob.Name, failed, rtype) 778 | r.recorder.Event(msJob, corev1.EventTypeNormal, msJobFailedReason, msg) 779 | if jobStatus.CompletionTime == nil { 780 | now := metav1.Now() 781 | jobStatus.CompletionTime = &now 782 | } 783 | err := commonutil.UpdateJobConditions(jobStatus, 784 | commonv1.JobFailed, msJobFailedReason, msg) 785 | if err != nil { 786 | commonutil.LoggerForJob(msJob).Infof("Append msjob condition error: %v", err) 787 | return err 788 | } 789 | trainingoperatorcommon.FailedJobsCounterInc(msJob.Namespace, mindsporev1.FrameworkName) 790 | } 791 | } 792 | } 793 | // we assign the jobStatus to the msJob.Status for testing purpose 794 | // it won't effect the main reconcile logic 795 | // because we already use oldStatus := jobStatus.DeepCopy() to record the oldStatus 796 | // and use !reflect.DeepEqual(*oldStatus, jobStatus) to decide whether to update the msJob or not 797 | msJob.Status = *jobStatus.DeepCopy() 798 | 799 | return nil 800 | } 801 | 802 | func (r *MSJobReconciler) UpdateJobStatusInApiServer(job interface{}, jobStatus *commonv1.JobStatus) error { 803 | msJob, ok := job.(*mindsporev1.MSJob) 804 | if !ok { 805 | return fmt.Errorf("%v is not a type of MSJob", msJob) 806 | } 807 | 808 | startTime := time.Now() 809 | logger := commonutil.LoggerForJob(msJob) 810 | defer func() { 811 | logger.Infof("Finished updating MSJobs Status %q (%v)", 812 | msJob.Name, time.Since(startTime)) 813 | }() 814 | 815 | msJob = msJob.DeepCopy() 816 | msJob.Status = *jobStatus.DeepCopy() 817 | 818 | result := r.Status().Update(context.Background(), msJob) 819 | 820 | if result != nil { 821 | r.Log.WithValues("msjob", types.NamespacedName{ 822 | Namespace: msJob.GetNamespace(), 823 | Name: msJob.GetName(), 824 | }) 825 | return result 826 | } 827 | 828 | return nil 829 | } 830 | 831 | // Set Envs for MSJob 832 | func (r *MSJobReconciler) SetClusterSpec(job interface{}, podTemplate *corev1.PodTemplateSpec, rtype, index string) error { 833 | msjob, ok := job.(*mindsporev1.MSJob) 834 | if !ok { 835 | return fmt.Errorf("%v is not a type of MSJob", msjob) 836 | } 837 | 838 | services, _ := r.GetServicesForJob(job) 839 | if len(services) > 0 { 840 | msSchedHostStr, msSchedPortStr := getServiceIpAndPort(services[0]) 841 | msServerNumStr, msWorkerNumStr := genPSAndWorkerNum(msjob) 842 | if msSchedHostStr == "" || msSchedPortStr == "" { 843 | return fmt.Errorf("%v scheduler service has no hostip or port, which is %s:%s", msjob, msSchedHostStr, msSchedPortStr) 844 | } 845 | // // Add environment variable to ms container in the pod. 846 | for i := range podTemplate.Spec.Containers { 847 | if podTemplate.Spec.Containers[i].Name == mindsporev1.DefaultContainerName { 848 | if len(podTemplate.Spec.Containers[i].Env) == 0 { 849 | podTemplate.Spec.Containers[i].Env = make([]corev1.EnvVar, 0) 850 | } 851 | // Scheduler env of msSchedHost must be the podIP, and other roles's are the service IP of scheduler. 852 | if rtype == strings.ToLower(string(mindsporev1.MSReplicaTypeScheduler)) { 853 | podTemplate.Spec.Containers[i].Env = append(podTemplate.Spec.Containers[i].Env, corev1.EnvVar{ 854 | Name: msSchedHost, 855 | ValueFrom: &corev1.EnvVarSource{ 856 | FieldRef: &corev1.ObjectFieldSelector{ 857 | FieldPath: "status.podIP", 858 | }, 859 | }, 860 | }) 861 | } else { 862 | podTemplate.Spec.Containers[i].Env = append(podTemplate.Spec.Containers[i].Env, corev1.EnvVar{ 863 | Name: msSchedHost, 864 | Value: msSchedHostStr, 865 | }) 866 | } 867 | podTemplate.Spec.Containers[i].Env = append(podTemplate.Spec.Containers[i].Env, corev1.EnvVar{ 868 | Name: msNodeId, 869 | ValueFrom: &corev1.EnvVarSource{ 870 | FieldRef: &corev1.ObjectFieldSelector{ 871 | FieldPath: "metadata.name", 872 | }, 873 | }, 874 | }) 875 | podTemplate.Spec.Containers[i].Env = append(podTemplate.Spec.Containers[i].Env, corev1.EnvVar{ 876 | Name: msSchedPort, 877 | Value: msSchedPortStr, 878 | }) 879 | podTemplate.Spec.Containers[i].Env = append(podTemplate.Spec.Containers[i].Env, corev1.EnvVar{ 880 | Name: msServerNum, 881 | Value: msServerNumStr, 882 | }) 883 | podTemplate.Spec.Containers[i].Env = append(podTemplate.Spec.Containers[i].Env, corev1.EnvVar{ 884 | Name: msWorkerNum, 885 | Value: msWorkerNumStr, 886 | }) 887 | podTemplate.Spec.Containers[i].Env = append(podTemplate.Spec.Containers[i].Env, corev1.EnvVar{ 888 | Name: msRole, 889 | Value: msRoleMap[rtype], 890 | }) 891 | } 892 | } 893 | } 894 | return nil 895 | } 896 | 897 | func (r *MSJobReconciler) GetDefaultContainerName() string { 898 | return mindsporev1.DefaultContainerName 899 | } 900 | 901 | func (r *MSJobReconciler) GetDefaultContainerPortName() string { 902 | return mindsporev1.DefaultPortName 903 | } 904 | 905 | func (r *MSJobReconciler) IsMasterRole(replicas map[commonv1.ReplicaType]*commonv1.ReplicaSpec, 906 | rtype commonv1.ReplicaType, index int) bool { 907 | return false 908 | } 909 | 910 | // IsWorker0Completed returns true if pod of worker0 succeeded and exited with 0 911 | func (r *MSJobReconciler) IsWorker0Completed(msjob *mindsporev1.MSJob, replicas map[commonv1.ReplicaType]*commonv1.ReplicaSpec) (bool, error) { 912 | worker0Completed := false 913 | _, ok := replicas[mindsporev1.MSReplicaTypeWorker] 914 | if !ok { 915 | return true, nil 916 | } 917 | podSlices, err := r.getPodSlices(msjob, replicas[mindsporev1.MSReplicaTypeWorker].Replicas) 918 | if err != nil { 919 | return false, err 920 | } 921 | for index, podSlice := range podSlices { 922 | if len(podSlice) == 1 { 923 | pod := podSlice[0] 924 | exitCode := getContainerExitCode(pod) 925 | if index == 0 && exitCode == 0 && pod.Status.Phase == v1.PodSucceeded { 926 | worker0Completed = true 927 | } 928 | } 929 | } 930 | return worker0Completed, nil 931 | } 932 | 933 | // getPodSlices returns a slice, which element is the slice of pod. 934 | // It gives enough information to caller to make decision to up/down scale resources. 935 | func (r *MSJobReconciler) getPodSlices(msjob *mindsporev1.MSJob, replicasNum *int32) ([][]*v1.Pod, error) { 936 | logger := commonutil.LoggerForReplica(msjob, strings.ToLower(string(mindsporev1.MSReplicaTypeWorker))) 937 | 938 | pods, err := r.GetPodsForJob(msjob) 939 | if err != nil { 940 | commonutil.LoggerForJob(msjob).Warnf("getPodsForMSJob error %v", err) 941 | return nil, err 942 | } 943 | 944 | // Get all pods for the type rt. 945 | pods, err = r.JobController.FilterPodsForReplicaType(pods, strings.ToLower(string(mindsporev1.MSReplicaTypeWorker))) 946 | if err != nil { 947 | return nil, err 948 | } 949 | 950 | podSlices := r.GetPodSlices(pods, int(*replicasNum), logger) 951 | return podSlices, nil 952 | } 953 | 954 | // reconcilePods checks and updates pods for each given MSReplicaSpec. 955 | // It will requeue the msjob in case of an error while creating/deleting pods. 956 | func (r *MSJobReconciler) ReconcilePods( 957 | job interface{}, 958 | jobStatus *commonv1.JobStatus, 959 | pods []*v1.Pod, 960 | rtype commonv1.ReplicaType, 961 | spec *commonv1.ReplicaSpec, 962 | replicas map[commonv1.ReplicaType]*commonv1.ReplicaSpec, 963 | ) error { 964 | 965 | msJob, ok := job.(*mindsporev1.MSJob) 966 | if !ok { 967 | return fmt.Errorf("%v is not a type of MSJob", msJob) 968 | } 969 | 970 | // Convert ReplicaType to lower string. 971 | rt := strings.ToLower(string(rtype)) 972 | logger := commonutil.LoggerForJob(msJob) 973 | // Get all pods for the type rt. 974 | pods, err := r.FilterPodsForReplicaType(pods, rt) 975 | if err != nil { 976 | return err 977 | } 978 | numReplicas := int(*spec.Replicas) 979 | initializeReplicaStatuses(jobStatus, rtype) 980 | 981 | // GetPodSlices will return enough information here to make decision to add/remove/update resources. 982 | // 983 | // For example, let's assume we have pods with replica-index 0, 1, 2 984 | // If replica is 4, return a slice with size 4. [[0],[1],[2],[]], a pod with replica-index 3 will be created. 985 | // 986 | // If replica is 1, return a slice with size 3. [[0],[1],[2]], pod with replica-index 1 and 2 are out of range and will be deleted. 987 | podSlices := r.GetPodSlices(pods, numReplicas, logger) 988 | for index, podSlice := range podSlices { 989 | if len(podSlice) > 1 { 990 | logger.Warningf("We have too many pods for %s %d", rt, index) 991 | } else if len(podSlice) == 0 { 992 | logger.Infof("Need to create new pod: %s-%d", rt, index) 993 | 994 | // check if this replica is the master role 995 | // TODO: [should change to CreateNewPod] 996 | err = r.createNewPod(msJob, rt, strconv.Itoa(index), spec, replicas) 997 | if err != nil { 998 | return err 999 | } 1000 | } else { 1001 | // Check the status of the current pod. 1002 | pod := podSlice[0] 1003 | 1004 | // check if the index is in the valid range, if not, we should kill the pod 1005 | if index < 0 || index >= numReplicas { 1006 | err = r.PodControl.DeletePod(pod.Namespace, pod.Name, msJob) 1007 | if err != nil { 1008 | return err 1009 | } 1010 | } 1011 | // Get the exit code of the container. 1012 | var exitCode int32 = 0xbeef // magic number 1013 | for _, status := range pod.Status.ContainerStatuses { 1014 | state := status.State 1015 | if status.Name == r.GetDefaultContainerName() && state.Terminated != nil { 1016 | exitCode = state.Terminated.ExitCode 1017 | logger.Infof("Pod: %v.%v exited with code %v", pod.Namespace, pod.Name, exitCode) 1018 | r.Recorder.Eventf(msJob, v1.EventTypeNormal, exitedWithCodeReason, "Pod: %v.%v exited with code %v", pod.Namespace, pod.Name, exitCode) 1019 | } 1020 | } 1021 | // Check if the pod is retryable. 1022 | if spec.RestartPolicy == commonv1.RestartPolicyExitCode { 1023 | if pod.Status.Phase == v1.PodFailed && train_util.IsRetryableExitCode(exitCode) { 1024 | logger.Infof("Need to restart the pod: %v.%v", pod.Namespace, pod.Name) 1025 | if err := r.PodControl.DeletePod(pod.Namespace, pod.Name, msJob); err != nil { 1026 | return err 1027 | } 1028 | 1029 | // with common library framework, we have to handle restart status here 1030 | // or we won't know which replica has been restarted in updateJobStatus after reconciling all replicas 1031 | msg := fmt.Sprintf("MSJob %s is restarting because %s replica(s) failed.", 1032 | msJob.Name, rtype) 1033 | r.Recorder.Event(msJob, corev1.EventTypeWarning, msJobRestartingReason, msg) 1034 | err := commonutil.UpdateJobConditions(jobStatus, commonv1.JobRestarting, msJobRestartingReason, msg) 1035 | if err != nil { 1036 | commonutil.LoggerForJob(msJob).Infof("Append msjob condition error: %v", err) 1037 | return err 1038 | } 1039 | trainingoperatorcommon.RestartedJobsCounterInc(msJob.Namespace, mindsporev1.FrameworkName) 1040 | } 1041 | } 1042 | 1043 | updateJobReplicaStatuses(jobStatus, rtype, pod) 1044 | } 1045 | } 1046 | return nil 1047 | } 1048 | 1049 | // createNewPod creates a new pod for the given index and type. 1050 | func (r *MSJobReconciler) createNewPod(msjob *mindsporev1.MSJob, rt, index string, spec *commonv1.ReplicaSpec, 1051 | replicas map[commonv1.ReplicaType]*commonv1.ReplicaSpec) error { 1052 | 1053 | msjobKey, err := common.KeyFunc(msjob) 1054 | if err != nil { 1055 | utilruntime.HandleError(fmt.Errorf("couldn't get key for msjob object %#v: %v", msjob, err)) 1056 | return err 1057 | } 1058 | expectationPodsKey := expectation.GenExpectationPodsKey(msjobKey, rt) 1059 | err = r.Expectations.ExpectCreations(expectationPodsKey, 1) 1060 | if err != nil { 1061 | return err 1062 | } 1063 | logger := commonutil.LoggerForReplica(msjob, rt) 1064 | // Create OwnerReference. 1065 | controllerRef := r.GenOwnerReference(msjob) 1066 | 1067 | // Set type and index for the worker. 1068 | labels := r.GenLabels(msjob.Name) 1069 | labels[msReplicaTypeLabel] = rt 1070 | labels[msReplicaIndexLabel] = index 1071 | labels[commonv1.ReplicaTypeLabel] = rt 1072 | labels[commonv1.ReplicaIndexLabel] = index 1073 | 1074 | podTemplate := spec.Template.DeepCopy() 1075 | 1076 | // Set name for the template. 1077 | podTemplate.Name = common.GenGeneralName(msjob.Name, rt, index) 1078 | 1079 | if podTemplate.Labels == nil { 1080 | podTemplate.Labels = make(map[string]string) 1081 | } 1082 | 1083 | for key, value := range labels { 1084 | podTemplate.Labels[key] = value 1085 | } 1086 | 1087 | if err := r.SetClusterSpec(msjob, podTemplate, rt, index); err != nil { 1088 | return err 1089 | } 1090 | 1091 | // Submit a warning event if the user specifies restart policy for 1092 | // the pod template. We recommend to set it from the replica level. 1093 | if podTemplate.Spec.RestartPolicy != v1.RestartPolicy("") { 1094 | errMsg := "Restart policy in pod template will be overwritten by restart policy in replica spec" 1095 | logger.Warning(errMsg) 1096 | r.Recorder.Event(msjob, v1.EventTypeWarning, podTemplateRestartPolicyReason, errMsg) 1097 | } 1098 | setRestartPolicy(podTemplate, spec) 1099 | 1100 | // if gang-scheduling is enabled: 1101 | // 1. if user has specified other scheduler, we report a warning without overriding any fields. 1102 | // 2. if no SchedulerName is set for pods, then we set the SchedulerName to "volcano". 1103 | if r.Config.EnableGangScheduling { 1104 | podSchedulerName := util.GetSchedulerName(replicas) 1105 | if len(podSchedulerName) == 0 { 1106 | podTemplate.Spec.SchedulerName = gangSchedulerName 1107 | } else if strings.Compare(podSchedulerName, gangSchedulerName) != 0 { 1108 | errMsg := "Another scheduler is specified when gang-scheduling is enabled and it will not be overwritten" 1109 | logger.Warning(errMsg) 1110 | r.Recorder.Event(msjob, v1.EventTypeWarning, podTemplateSchedulerNameReason, errMsg) 1111 | } 1112 | 1113 | if podTemplate.Annotations == nil { 1114 | podTemplate.Annotations = map[string]string{} 1115 | } 1116 | podTemplate.Annotations[gangSchedulingPodGroupAnnotation] = msjob.GetName() 1117 | podTemplate.Annotations[volcanoTaskSpecKey] = rt 1118 | } 1119 | 1120 | err = r.PodControl.CreatePodsWithControllerRef(msjob.Namespace, podTemplate, msjob, controllerRef) 1121 | if err != nil && errors.IsTimeout(err) { 1122 | // Pod is created but its initialization has timed out. 1123 | // If the initialization is successful eventually, the 1124 | // controller will observe the creation via the informer. 1125 | // If the initialization fails, or if the pod keeps 1126 | // uninitialized for a long time, the informer will not 1127 | // receive any update, and the controller will create a new 1128 | // pod when the expectation expires. 1129 | return nil 1130 | } else if err != nil { 1131 | // Decrement the expected number of creates because the informer won't observe this pod 1132 | logger.Infof( 1133 | "Failed creation, decrementing expectations for msjob %s/%s, key %s", 1134 | msjob.Namespace, msjob.Name, expectationPodsKey) 1135 | r.Expectations.CreationObserved(expectationPodsKey) 1136 | return err 1137 | } 1138 | return nil 1139 | } 1140 | 1141 | // reconcileServices checks and updates services for each given ReplicaSpec. 1142 | // It will requeue the job in case of an error while creating/deleting services. 1143 | func (jc *MSJobReconciler) ReconcileServices( 1144 | job metav1.Object, 1145 | services []*v1.Service, 1146 | rtype commonv1.ReplicaType, 1147 | spec *commonv1.ReplicaSpec) error { 1148 | 1149 | if rtype != mindsporev1.MSReplicaTypeScheduler { 1150 | return nil 1151 | } 1152 | // Convert ReplicaType to lower string. 1153 | rt := strings.ToLower(string(rtype)) 1154 | replicas := int(*spec.Replicas) 1155 | // Get all services for the type rt. 1156 | services, err := jc.FilterServicesForReplicaType(services, rt) 1157 | if err != nil { 1158 | return err 1159 | } 1160 | 1161 | // GetServiceSlices will return enough information here to make decision to add/remove/update resources. 1162 | // 1163 | // For example, let's assume we have services with replica-index 0, 1, 2 1164 | // If replica is 4, return a slice with size 4. [[0],[1],[2],[]], a svc with replica-index 3 will be created. 1165 | // 1166 | // If replica is 1, return a slice with size 3. [[0],[1],[2]], svc with replica-index 1 and 2 are out of range and will be deleted. 1167 | serviceSlices := jc.GetServiceSlices(services, replicas, commonutil.LoggerForReplica(job, rt)) 1168 | 1169 | for index, serviceSlice := range serviceSlices { 1170 | if len(serviceSlice) > 1 { 1171 | commonutil.LoggerForReplica(job, rt).Warningf("We have too many services for %s %d", rtype, index) 1172 | } else if len(serviceSlice) == 0 { 1173 | commonutil.LoggerForReplica(job, rt).Infof("need to create new service: %s-%d", rtype, index) 1174 | err = jc.CreateNewService(job, rtype, spec, strconv.Itoa(index)) 1175 | if err != nil { 1176 | return err 1177 | } 1178 | } else { 1179 | // Check the status of the current svc. 1180 | svc := serviceSlice[0] 1181 | 1182 | // check if the index is in the valid range, if not, we should kill the svc 1183 | if index < 0 || index >= replicas { 1184 | err = jc.ServiceControl.DeleteService(svc.Namespace, svc.Name, job.(runtime.Object)) 1185 | if err != nil { 1186 | return err 1187 | } 1188 | } 1189 | } 1190 | } 1191 | return nil 1192 | } 1193 | 1194 | // createNewService creates a new service for the given index and type. 1195 | func (jc *MSJobReconciler) CreateNewService(job metav1.Object, rtype commonv1.ReplicaType, 1196 | spec *commonv1.ReplicaSpec, index string) error { 1197 | jobKey, err := common.KeyFunc(job) 1198 | if err != nil { 1199 | utilruntime.HandleError(fmt.Errorf("couldn't get key for job object %#v: %v", job, err)) 1200 | return err 1201 | } 1202 | 1203 | rt := strings.ToLower(string(rtype)) 1204 | // Append ReplicaTypeLabelDeprecated and ReplicaIndexLabelDeprecated labels. 1205 | labels := jc.GenLabels(job.GetName()) 1206 | utillabels.SetReplicaType(labels, rt) 1207 | utillabels.SetReplicaIndexStr(labels, index) 1208 | 1209 | ports, err := jc.GetPortsFromJob(spec) 1210 | if err != nil { 1211 | return err 1212 | } 1213 | 1214 | service := &v1.Service{ 1215 | Spec: v1.ServiceSpec{ 1216 | Selector: labels, 1217 | Ports: []v1.ServicePort{}, 1218 | }, 1219 | } 1220 | 1221 | // Add service ports 1222 | for name, port := range ports { 1223 | svcPort := v1.ServicePort{Name: name, Port: port} 1224 | service.Spec.Ports = append(service.Spec.Ports, svcPort) 1225 | } 1226 | 1227 | service.Name = common.GenGeneralName(job.GetName(), rt, index) 1228 | service.Labels = labels 1229 | // Create OwnerReference. 1230 | controllerRef := jc.GenOwnerReference(job) 1231 | 1232 | // Creation is expected when there is no error returned 1233 | expectationServicesKey := expectation.GenExpectationServicesKey(jobKey, rt) 1234 | jc.Expectations.RaiseExpectations(expectationServicesKey, 1, 0) 1235 | 1236 | err = jc.ServiceControl.CreateServicesWithControllerRef(job.GetNamespace(), service, job.(runtime.Object), controllerRef) 1237 | if err != nil && errors.IsTimeout(err) { 1238 | // Service is created but its initialization has timed out. 1239 | // If the initialization is successful eventually, the 1240 | // controller will observe the creation via the informer. 1241 | // If the initialization fails, or if the service keeps 1242 | // uninitialized for a long time, the informer will not 1243 | // receive any update, and the controller will create a new 1244 | // service when the expectation expires. 1245 | succeededServiceCreationCount.Inc() 1246 | return nil 1247 | } else if err != nil { 1248 | // Since error occurred(the informer won't observe this service), 1249 | // we decrement the expected number of creates 1250 | // and wait until next reconciliation 1251 | jc.Expectations.CreationObserved(expectationServicesKey) 1252 | failedServiceCreationCount.Inc() 1253 | return err 1254 | } 1255 | succeededServiceCreationCount.Inc() 1256 | return nil 1257 | } 1258 | 1259 | // onOwnerCreateFunc modify creation condition. 1260 | func (r *MSJobReconciler) onOwnerCreateFunc() func(event.CreateEvent) bool { 1261 | return func(e event.CreateEvent) bool { 1262 | msJob, ok := e.Object.(*mindsporev1.MSJob) 1263 | if !ok { 1264 | return true 1265 | } 1266 | 1267 | r.Scheme.Default(msJob) 1268 | msg := fmt.Sprintf("MSJob %s is created.", e.Object.GetName()) 1269 | logrus.Info(msg) 1270 | trainingoperatorcommon.CreatedJobsCounterInc(msJob.Namespace, mindsporev1.FrameworkName) 1271 | if err := commonutil.UpdateJobConditions(&msJob.Status, commonv1.JobCreated, "MSJobCreated", msg); err != nil { 1272 | log.Log.Error(err, "append job condition error") 1273 | return false 1274 | } 1275 | return true 1276 | } 1277 | } 1278 | -------------------------------------------------------------------------------- /pkg/controllers/v1/suite_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2022 Huawei Technologies Co., Ltd. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package controllers 18 | 19 | import ( 20 | "path/filepath" 21 | "testing" 22 | 23 | . "github.com/onsi/ginkgo" 24 | . "github.com/onsi/gomega" 25 | "k8s.io/client-go/kubernetes/scheme" 26 | "k8s.io/client-go/rest" 27 | "sigs.k8s.io/controller-runtime/pkg/client" 28 | "sigs.k8s.io/controller-runtime/pkg/envtest" 29 | "sigs.k8s.io/controller-runtime/pkg/envtest/printer" 30 | logf "sigs.k8s.io/controller-runtime/pkg/log" 31 | "sigs.k8s.io/controller-runtime/pkg/log/zap" 32 | 33 | mindsporev1 "ms-operator/pkg/apis/v1" 34 | //+kubebuilder:scaffold:imports 35 | ) 36 | 37 | // These tests use Ginkgo (BDD-style Go testing framework). Refer to 38 | // http://onsi.github.io/ginkgo/ to learn more about Ginkgo. 39 | 40 | var cfg *rest.Config 41 | var k8sClient client.Client 42 | var testEnv *envtest.Environment 43 | 44 | func TestAPIs(t *testing.T) { 45 | RegisterFailHandler(Fail) 46 | 47 | RunSpecsWithDefaultAndCustomReporters(t, 48 | "Controller Suite", 49 | []Reporter{printer.NewlineReporter{}}) 50 | } 51 | 52 | var _ = BeforeSuite(func() { 53 | logf.SetLogger(zap.New(zap.WriteTo(GinkgoWriter), zap.UseDevMode(true))) 54 | 55 | By("bootstrapping test environment") 56 | testEnv = &envtest.Environment{ 57 | CRDDirectoryPaths: []string{filepath.Join("..", "config", "crd", "bases")}, 58 | ErrorIfCRDPathMissing: true, 59 | } 60 | 61 | cfg, err := testEnv.Start() 62 | Expect(err).NotTo(HaveOccurred()) 63 | Expect(cfg).NotTo(BeNil()) 64 | 65 | err = mindsporev1.AddToScheme(scheme.Scheme) 66 | Expect(err).NotTo(HaveOccurred()) 67 | 68 | //+kubebuilder:scaffold:scheme 69 | 70 | k8sClient, err = client.New(cfg, client.Options{Scheme: scheme.Scheme}) 71 | Expect(err).NotTo(HaveOccurred()) 72 | Expect(k8sClient).NotTo(BeNil()) 73 | 74 | }, 60) 75 | 76 | var _ = AfterSuite(func() { 77 | By("tearing down the test environment") 78 | err := testEnv.Stop() 79 | Expect(err).NotTo(HaveOccurred()) 80 | }) 81 | -------------------------------------------------------------------------------- /pkg/controllers/v1/utils.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2022 Huawei Technologies Co., Ltd. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package controllers 18 | 19 | import ( 20 | "fmt" 21 | mindsporev1 "ms-operator/pkg/apis/v1" 22 | "strconv" 23 | 24 | commonv1 "github.com/kubeflow/common/pkg/apis/common/v1" 25 | "github.com/prometheus/common/log" 26 | corev1 "k8s.io/api/core/v1" 27 | v1 "k8s.io/api/core/v1" 28 | ) 29 | 30 | func getContainerExitCode(pod *corev1.Pod) int32 { 31 | var exitCode int32 = 0xbeef // magic number 32 | for _, status := range pod.Status.ContainerStatuses { 33 | state := status.State 34 | if status.Name == mindsporev1.DefaultContainerName && state.Terminated != nil { 35 | exitCode = state.Terminated.ExitCode 36 | } 37 | } 38 | return exitCode 39 | } 40 | 41 | func setRestartPolicy(podTemplateSpec *corev1.PodTemplateSpec, spec *commonv1.ReplicaSpec) { 42 | // This is necessary since restartPolicyExitCode is not supported in v1.PodTemplateSpec 43 | if spec.RestartPolicy == commonv1.RestartPolicyExitCode { 44 | podTemplateSpec.Spec.RestartPolicy = corev1.RestartPolicyNever 45 | } else { 46 | podTemplateSpec.Spec.RestartPolicy = corev1.RestartPolicy(spec.RestartPolicy) 47 | } 48 | } 49 | 50 | // isDistributed returns if the MSJob is a distributed training job. 51 | func isDistributed(msjob *mindsporev1.MSJob) bool { 52 | replicas := msjob.Spec.MSReplicaSpecs 53 | distributionCount := 0 54 | allTypes := []commonv1.ReplicaType{ 55 | mindsporev1.MSReplicaTypeScheduler, 56 | mindsporev1.MSReplicaTypePS, 57 | mindsporev1.MSReplicaTypeWorker, 58 | } 59 | // Check if there is only one replica. 60 | for _, typ := range allTypes { 61 | if replicas[typ] != nil { 62 | if replicas[typ].Replicas == nil { 63 | distributionCount++ 64 | } else { 65 | distributionCount += int(*replicas[typ].Replicas) 66 | } 67 | } 68 | } 69 | return distributionCount != 1 70 | } 71 | 72 | // initializeReplicaStatuses initializes the ReplicaStatuses for replica. 73 | func initializeReplicaStatuses(jobStatus *commonv1.JobStatus, rtype commonv1.ReplicaType) { 74 | if jobStatus.ReplicaStatuses == nil { 75 | jobStatus.ReplicaStatuses = make(map[commonv1.ReplicaType]*commonv1.ReplicaStatus) 76 | } 77 | 78 | jobStatus.ReplicaStatuses[rtype] = &commonv1.ReplicaStatus{} 79 | } 80 | 81 | // updateJobReplicaStatuses updates the JobReplicaStatuses according to the pod. 82 | func updateJobReplicaStatuses(jobStatus *commonv1.JobStatus, rtype commonv1.ReplicaType, pod *corev1.Pod) { 83 | switch pod.Status.Phase { 84 | case corev1.PodRunning: 85 | jobStatus.ReplicaStatuses[rtype].Active++ 86 | case corev1.PodSucceeded: 87 | jobStatus.ReplicaStatuses[rtype].Succeeded++ 88 | case corev1.PodFailed: 89 | jobStatus.ReplicaStatuses[rtype].Failed++ 90 | } 91 | } 92 | 93 | func getServiceIpAndPort(service *v1.Service) (string, string) { 94 | schedulerPort := "" 95 | for _, port := range service.Spec.Ports { 96 | if port.Name == mindsporev1.DefaultPortName { 97 | schedulerPort = strconv.Itoa(int(port.Port)) 98 | break 99 | } 100 | } 101 | return service.Spec.ClusterIP, schedulerPort 102 | } 103 | 104 | func genPSAndWorkerNum(msjob *mindsporev1.MSJob) (string, string) { 105 | msServerNumStr, msWorkerNumStr := "0", "0" 106 | for rtype, spec := range msjob.Spec.MSReplicaSpecs { 107 | if isServer(rtype) { 108 | msServerNumStr = strconv.Itoa(int(*spec.Replicas)) 109 | } 110 | if isWorker(rtype) { 111 | msWorkerNumStr = strconv.Itoa(int(*spec.Replicas)) 112 | } 113 | } 114 | return msServerNumStr, msWorkerNumStr 115 | 116 | } 117 | 118 | // isScheduler returns true if the type is Scheduler. 119 | func isScheduler(typ commonv1.ReplicaType) bool { 120 | return typ == mindsporev1.MSReplicaTypeScheduler 121 | } 122 | 123 | // isWorker returns true if the type is Worker. 124 | func isWorker(typ commonv1.ReplicaType) bool { 125 | return typ == mindsporev1.MSReplicaTypeWorker 126 | } 127 | 128 | // isServer returns true if the type is PS. 129 | func isServer(typ commonv1.ReplicaType) bool { 130 | return typ == mindsporev1.MSReplicaTypePS 131 | } 132 | 133 | func validateV1ReplicaSpecs(specs map[commonv1.ReplicaType]*commonv1.ReplicaSpec) error { 134 | if specs == nil { 135 | return fmt.Errorf("MSJobSpec is not valid") 136 | } 137 | foundScheduler := 0 138 | for rType, value := range specs { 139 | if value == nil || len(value.Template.Spec.Containers) == 0 { 140 | return fmt.Errorf("MSJobSpec is not valid: containers definition expected in %v", rType) 141 | } 142 | if isScheduler(rType) { 143 | foundScheduler++ 144 | } 145 | // Make sure the image is defined in the container. 146 | numNamedMindSpore := 0 147 | for _, container := range value.Template.Spec.Containers { 148 | if container.Image == "" { 149 | msg := fmt.Sprintf("MSJobSpec is not valid: Image is undefined in the container of %v", rType) 150 | log.Error(msg) 151 | return fmt.Errorf(msg) 152 | } 153 | if container.Name == mindsporev1.DefaultContainerName { 154 | numNamedMindSpore++ 155 | } 156 | } 157 | if numNamedMindSpore == 0 { 158 | msg := fmt.Sprintf("MSJobSpec is not valid: There is no container named %s in %v", mindsporev1.DefaultContainerName, rType) 159 | log.Error(msg) 160 | return fmt.Errorf(msg) 161 | } 162 | } 163 | if foundScheduler > 1 { 164 | return fmt.Errorf("MSJobSpec is not valid: %d Scheduler found", foundScheduler) 165 | } 166 | 167 | return nil 168 | } 169 | --------------------------------------------------------------------------------