├── .github ├── ISSUE_TEMPLATE │ └── bug_report.md └── workflows │ ├── go.yml │ └── greetings.yml ├── .gitignore ├── ADOPTERS.md ├── Dockerfile.descheduler ├── Dockerfile.provider ├── Dockerfile.webhook ├── LICENSE ├── Makefile ├── README.md ├── cmd ├── descheduler │ ├── app │ │ ├── options │ │ │ └── options.go │ │ ├── server.go │ │ └── version.go │ └── descheduler.go ├── provider │ └── main.go └── webhook │ ├── app │ ├── options.go │ └── server.go │ └── webhook.go ├── docs ├── multi.png └── tensile-kube.png ├── go.mod ├── go.sum ├── manifeasts ├── descheduler.yaml ├── virtual-node.yaml └── webhook.yaml └── pkg ├── common ├── node.go ├── resource.go └── resource_test.go ├── controllers ├── base.go ├── common_controller.go ├── common_controller_test.go ├── pv_controller.go ├── service_controller.go ├── service_controller_test.go ├── utils.go └── utils_test.go ├── descheduler ├── descheduler.go ├── evictions │ ├── evictions.go │ └── evictions_test.go ├── pod │ ├── pods.go │ └── pods_test.go └── strategies │ └── pod_lifetime.go ├── provider ├── helper.go ├── helper_test.go ├── metrics.go ├── node.go ├── node_test.go ├── pod.go ├── pod_test.go └── provider.go ├── testbase └── test.go ├── util ├── cache.go ├── cache_test.go ├── conversions.go ├── conversions_test.go ├── k8s.go ├── k8s_test.go ├── util.go └── util_test.go └── webhook ├── hook.go └── hook_test.go /.github/ISSUE_TEMPLATE/bug_report.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Bug report 3 | about: Create a report to help us improve 4 | title: '' 5 | labels: bug 6 | assignees: cwdsuzhou 7 | 8 | --- 9 | 10 | **Describe the bug** 11 | A clear and concise description of what the bug is. 12 | 13 | **To Reproduce** 14 | Steps to reproduce the behavior: 15 | 16 | 17 | **Expected behavior** 18 | A clear and concise description of what you expected to happen. 19 | 20 | **Kubernetes version (please complete the following information):** 21 | - Version of Upper K8s cluster 22 | - Version of Lower K8s cluster 23 | 24 | **Additional context** 25 | Add any other context about the problem here. 26 | -------------------------------------------------------------------------------- /.github/workflows/go.yml: -------------------------------------------------------------------------------- 1 | name: Go 2 | 3 | on: 4 | push: 5 | branches: [ master ] 6 | pull_request: 7 | branches: [ master ] 8 | 9 | jobs: 10 | lint: 11 | name: Lint 12 | runs-on: ubuntu-latest 13 | steps: 14 | - name: Set up Go 15 | uses: actions/setup-go@v1 16 | with: 17 | go-version: ^1.13 18 | 19 | - name: Check out code 20 | uses: actions/checkout@v2 21 | 22 | - name: Lint Go Code 23 | run: | 24 | export PATH=$PATH:$(go env GOPATH)/bin # temporary fix. See https://github.com/actions/setup-go/issues/14 25 | go install golang.org/x/lint/golint@latest 26 | golint ./... 27 | build: 28 | name: Build 29 | runs-on: ubuntu-latest 30 | steps: 31 | 32 | - name: Set up Go 1.x 33 | uses: actions/setup-go@v2 34 | with: 35 | go-version: ^1.13 36 | id: go 37 | 38 | - name: Check out code into the Go module directory 39 | uses: actions/checkout@v2 40 | 41 | - name: Get dependencies 42 | run: | 43 | go get -v -t -d ./... 44 | if [ -f Gopkg.toml ]; then 45 | curl https://raw.githubusercontent.com/golang/dep/master/install.sh | sh 46 | dep ensure 47 | fi 48 | 49 | - name: Test 50 | run: make test 51 | 52 | - name: Build 53 | run: make build 54 | -------------------------------------------------------------------------------- /.github/workflows/greetings.yml: -------------------------------------------------------------------------------- 1 | name: Greetings 2 | 3 | on: [pull_request, issues] 4 | 5 | jobs: 6 | greeting: 7 | runs-on: ubuntu-latest 8 | steps: 9 | - uses: actions/first-interaction@v1 10 | with: 11 | repo-token: ${{ secrets.GITHUB_TOKEN }} 12 | issue-message: 'Thank your for opening your first issue :)' 13 | pr-message: 'Thank your for opening your first pr and contributing to tensile-kube!!' 14 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Binaries for programs and plugins 2 | *.exe 3 | *.exe~ 4 | *.dll 5 | *.so 6 | *.dylib 7 | bin/ 8 | 9 | # Test binary, built with `go test -c` 10 | *.test 11 | 12 | # Output of the go coverage tool, specifically when used with LiteIDE 13 | *.out 14 | 15 | # Dependency directories (remove the comment below to include it) 16 | vendor/ 17 | .idea/ 18 | 19 | # VS Code files 20 | .vscode/ 21 | -------------------------------------------------------------------------------- /ADOPTERS.md: -------------------------------------------------------------------------------- 1 | # Production users 2 | 3 | ## YOUZAN 4 | 5 | - Application: Use elastic clusters as a virtual node. 6 | - Location: Hangzhou, China 7 | - Operator: liuyan@youzan.com 8 | 9 | ## Tencent Games 10 | 11 | - Application: Make use of the fragmented resources located in multi clusters. 12 | - Location: Shenzhen, China 13 | - Operator: cwdsuzhou@gamil.com 14 | -------------------------------------------------------------------------------- /Dockerfile.descheduler: -------------------------------------------------------------------------------- 1 | FROM centos:centos7 2 | LABEL description="descheduler" 3 | 4 | COPY ./bin/descheduler descheduler 5 | 6 | CMD ["/descheduler", "--help"] 7 | -------------------------------------------------------------------------------- /Dockerfile.provider: -------------------------------------------------------------------------------- 1 | FROM centos:centos7 2 | LABEL description="/virtual-node" 3 | 4 | COPY ./bin/virtual-node virtual-node 5 | ENTRYPOINT ["/virtual-node"] 6 | -------------------------------------------------------------------------------- /Dockerfile.webhook: -------------------------------------------------------------------------------- 1 | FROM centos:centos7 2 | LABEL description="webhook" 3 | 4 | COPY ./bin/webhook webhook 5 | ENTRYPOINT ["/webhook"] 6 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright [yyyy] [name of copyright owner] 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright ©2020. The virtual-kubelet authors 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | 17 | 18 | REGISTRY_NAME=xxxx 19 | GIT_COMMIT=$(shell git rev-parse "HEAD^{commit}") 20 | VERSION=$(shell git describe --tags --abbrev=14 "${GIT_COMMIT}^{commit}" --always) 21 | BUILD_TIME=$(shell TZ=Asia/Shanghai date +%FT%T%z) 22 | 23 | CMDS=build-vk 24 | all: test build 25 | 26 | build: fmt vet provider webhook descheduler 27 | 28 | fmt: 29 | go fmt ./pkg/... 30 | 31 | vet: 32 | go vet ./pkg/... 33 | 34 | provider: 35 | mkdir -p bin 36 | CGO_ENABLED=0 GOOS=linux go build -ldflags "-X 'main.buildVersion=$(VERSION)' -X 'main.buildTime=${BUILD_TIME}'" -o ./bin/virtual-node ./cmd/provider 37 | 38 | webhook: 39 | mkdir -p bin 40 | CGO_ENABLED=0 GOOS=linux go build -ldflags "-X 'github.com/virtual-kubelet/tensile-kube/cmd/webhook/app.Version=$(VERSION)'" -o ./bin/webhook ./cmd/webhook 41 | 42 | descheduler: 43 | mkdir -p bin 44 | CGO_ENABLED=0 GOOS=linux go build -ldflags "-X 'github.com/virtual-kubelet/tensile-kube/cmd/descheduler/app.version=$(VERSION)'" -o ./bin/descheduler ./cmd/descheduler 45 | 46 | container: container-provider container-webhook container-descheduler 47 | 48 | container-provider: provider 49 | docker build -t $(REGISTRY_NAME)/virtual-node:$(VERSION) -f $(shell if [ -e ./cmd/$*/Dockerfile ]; then echo ./cmd/$*/Dockerfile; else echo Dockerfile.provider; fi) --label revision=$(REV) . 50 | 51 | container-webhook: webhook 52 | docker build -t $(REGISTRY_NAME)/virtual-webhook:$(VERSION) -f $(shell if [ -e ./cmd/$*/Dockerfile ]; then echo ./cmd/$*/Dockerfile; else echo Dockerfile.webhook; fi) --label revision=$(REV) . 53 | 54 | container-descheduler: descheduler 55 | docker build -t $(REGISTRY_NAME)/descheduler:$(VERSION) -f $(shell if [ -e ./cmd/$*/Dockerfile ]; then echo ./cmd/$*/Dockerfile; else echo Dockerfile.descheduler; fi) --label revision=$(REV) . 56 | 57 | push: container 58 | docker push $(REGISTRY_NAME)/virtual-k8s:$(VERSION) 59 | 60 | test: 61 | go test -count=1 ./pkg/... -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # tensile-kube 2 | 3 | ## Overview 4 | 5 | `tensile-kube` enables kubernetes clusters work together. Based on [virtual-kubelet](https://github.com/virtual-kubelet/virtual-kubelet), `tensile-kube` 6 | provides the following abilities: 7 | 8 | - Cluster resource automatically discovery 9 | - Notify pods modification async, decrease the cost of frequently list 10 | - Support all actions of `kubectl logs` and `kubectl exec` 11 | - Globally schedule pods to avoid unschedulable pod due to resource fragmentation when using multi-scheduler 12 | - Re-schedule pods if pod can not be scheduled in lower clusters by using descheduler 13 | - PV/PVC 14 | - Service 15 | 16 | ## Components 17 | 18 | ### Arch 19 | 20 | ![tensile kube](./docs/tensile-kube.png) 21 | 22 | - virtual-node 23 | 24 | This is a kubernetes provider implemented based on virtual-kubelet. Pods created in the upper cluster 25 | will be synced to the lower cluster. If pods are depend on configmaps or secrets, dependencies would 26 | also be created in the cluster. 27 | 28 | - multi-cluster scheduler 29 | 30 | The scheduler is implemented based on [K8s scheduling framework](https://kubernetes.io/docs/concepts/scheduling-eviction/scheduling-framework/). It would watch all of the lower 31 | clusters's capacity and call `filter` while scheduling pods. If the number of available nodes is greater 32 | than or equal to 1, the pods can be scheduler. As you see, this may cost more resources, so we add another 33 | implementation(descheduler). 34 | 35 | The multi-cluster scheduler repo has been removed to [super-scheduling](https://github.com/cwdsuzhou/super-scheduling) 36 | 37 | - descheduler 38 | 39 | descheduler is inspired by [K8s descheduler](https://github.com/kubernetes-sigs/descheduler), but it cannot 40 | satisfy all our requirements, so we change some logic. Some unschedulable pods would be re-created by it with some 41 | nodeAffinity injected. 42 | 43 | We can choose one of the multi-scheduler and descheduler in the upper cluster or both. 44 | 45 | > - Large cluster is not recommended to use multi-scheduler, e.g. sum of nodes in sub cluster is more than 46 | 10000, descheduler would cost less. 47 | > - Multi-scheduler would be better when there are fewer nodes in a cluster, e.g. we have 10 clusters but each cluster 48 | only has 100 nodes. 49 | 50 | - webhook 51 | 52 | Webhook are designed based on K8s mutation webhook. It helps convert some fields that can affect scheduling pods(not in kube-system) in the upper cluster, e.g. `nodeSelector`, `nodeAffinity` and `tolerations`. But only the pods have a label `virtual-pod:true` would be converted. These fields would be converted into the annotation as follows: 53 | 54 | ```text 55 | clusterSelector: '{"tolerations":[{"key":"node.kubernetes.io/not-ready","operator":"Exists","effect":"NoExecute"},{"key":"node.kubernetes.io/unreachable","operator":"Exists","effect":"NoExecute"},{"key":"test","operator":"Exists","effect":"NoExecute"},{"key":"test1","operator":"Exists","effect":"NoExecute"},{"key":"test2","operator":"Exists","effect":"NoExecute"},{"key":"test3","operator":"Exists","effect":"NoExecute"}]}' 56 | ``` 57 | 58 | This fields we would be added back when the pods created in the lower cluster. 59 | 60 | Pods are strongly recommended to run in the lower clusters and add a label `virtual-pod:true`, except for those pods must be deployed in `kube-system` in the upper cluster. 61 | 62 | > - For K8s< 1.16, pods without the label would not be converted. But queries would still send to the webhook. 63 | > - For K8s>=1.16, we can use label selector to enable the webhook for some specified pods. 64 | > - Overall, the initial idea is that we only run pods in lower clusters.** 65 | 66 | ## Restrictions 67 | 68 | - If you want to use service, must keep inter-pods communication normal. Pod A in cluster A can be accessed by pod B in cluster B through ip. The service `kubernetes` 69 | in `default` namespaces and other services in `kube-system` would be synced to lower clusters. 70 | 71 | - multi-scheduler developed in the repo may cost more resource because it would sync all objects that a scheduler 72 | need from all of lower clusters. 73 | 74 | - descheduler cannot absolutely avoid resource fragmentation. 75 | 76 | - PV/PVC only support `WaitForFirstConsumer`for local PV, the scheduler in the upper cluster should ignore 77 | `VolumeBindCheck` 78 | 79 | ## Use Case 80 | 81 | ![multi](./docs/multi.png) 82 | 83 | In [Tencent Games](https://game.qq.com/), they build the kubernetes cluster based on the [flannel](https://github.com/coreos/flannel) and all node CIDR allocation 84 | is based on the same etcd. So the pods actually can access each other directly by IP. 85 | 86 | ## Build 87 | 88 | ```build 89 | git clone https://github.com/virtual-kubelet/tensile-kube.git && make 90 | ``` 91 | ### virtual node parameters 92 | 93 | ```build 94 | --client-burst int qpi burst for client cluster. (default 1000) 95 | --client-kubeconfig string kube config for client cluster. 96 | --client-qps int qpi qps for client cluster. (default 500) 97 | --enable-controllers string support PVControllers,ServiceControllers, default, all of these (default "PVControllers,ServiceControllers") 98 | --enable-serviceaccount enable service account for pods, like spark driver, mpi launcher (default true) 99 | --ignore-labels string ignore-labels are the labels we would like to ignore when build pod for client clusters, usually these labels will infulence schedule, default group.batch.scheduler.tencent.com, multi labels should be seperated by comma(,) (default "group.batch.scheduler.tencent.com") 100 | --log-level string set the log level, e.g. "debug", "info", "warn", "error" (default "info") 101 | ... 102 | ``` 103 | 104 | ### deploy the virtual node 105 | 106 | ```build 107 | export KUBELET_PORT=10350 108 | export APISERVER_CERT_LOCATION=/etc/k8s.cer 109 | export APISERVER_KEY_LOCATION=/etc/k8s.key 110 | 111 | nohup ./virtual-node --nodename $IP --provider k8s --kube-api-qps 500 --kube-api-burst 1000 --client-qps 500 --client 112 | -burst 1000 --kubeconfig /root/server-kube.config --client-kubeconfig /client-kube.config --klog.v 4 --log-level 113 | debug 2>&1 > node.log & 114 | ``` 115 | or deploy in K8s 116 | 117 | ```shell 118 | # change the config in secret `virtual-kubelet` in `manifeasts/virtual-node.yaml` first, 119 | # change the image 120 | # then deploy it 121 | kubectl apply -f manifeasts/virtual-node.yaml 122 | ``` 123 | 124 | ### deploy the webhook 125 | 126 | it is recommended to be deployed in K8s cluster 127 | 128 | 1. replace the ${caBoudle}, ${cert}, ${key} with yours 129 | 2. replace the image of webhook 130 | 3. deploy it in K8s cluster 131 | 132 | ```shell 133 | kubectl apply -f manifeasts/webhook.yaml 134 | ``` 135 | 136 | ### deploy the descheduler 137 | 138 | 1. replace the image with yours 139 | 2. deploy it in K8s cluster 140 | 141 | ```shell 142 | kubectl apply -f manifeasts/descheduler.yaml 143 | ``` 144 | 145 | ## Main Contributors 146 | 147 | - [Weidong Cai](https://github.com/cwdsuzhou) from Tencent 148 | - [Ye Yin](https://github.com/hustcat) from Tencent 149 | 150 | ## Other Contributors 151 | - [Runzhong Liu](https://github.com/runzhliu) 152 | - [LeoLiuYan](https://github.com/LeoLiuYan) 153 | - [Thomas](https://github.com/tghartland) 154 | -------------------------------------------------------------------------------- /cmd/descheduler/app/options/options.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright ©2020. The virtual-kubelet authors 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | // Package options provides the descheduler flags 18 | package options 19 | 20 | import ( 21 | clientset "k8s.io/client-go/kubernetes" 22 | 23 | // install the componentconfig api so we get its defaulting and conversion functions 24 | "sigs.k8s.io/descheduler/pkg/apis/componentconfig" 25 | "sigs.k8s.io/descheduler/pkg/apis/componentconfig/v1alpha1" 26 | deschedulerscheme "sigs.k8s.io/descheduler/pkg/descheduler/scheme" 27 | 28 | "github.com/spf13/pflag" 29 | ) 30 | 31 | // DeschedulerServer configuration 32 | type DeschedulerServer struct { 33 | componentconfig.DeschedulerConfiguration 34 | Client clientset.Interface 35 | } 36 | 37 | // NewDeschedulerServer creates a new DeschedulerServer with default parameters 38 | func NewDeschedulerServer() *DeschedulerServer { 39 | versioned := v1alpha1.DeschedulerConfiguration{} 40 | deschedulerscheme.Scheme.Default(&versioned) 41 | cfg := componentconfig.DeschedulerConfiguration{} 42 | deschedulerscheme.Scheme.Convert(versioned, &cfg, nil) 43 | s := DeschedulerServer{ 44 | DeschedulerConfiguration: cfg, 45 | } 46 | return &s 47 | } 48 | 49 | // AddFlags adds flags for a specific SchedulerServer to the specified FlagSet 50 | func (rs *DeschedulerServer) AddFlags(fs *pflag.FlagSet) { 51 | fs.DurationVar(&rs.DeschedulingInterval, "descheduling-interval", rs.DeschedulingInterval, "Time interval between two consecutive descheduler executions. Setting this value instructs the descheduler to run in a continuous loop at the interval specified.") 52 | fs.StringVar(&rs.KubeconfigFile, "kubeconfig", rs.KubeconfigFile, "File with kube configuration.") 53 | fs.StringVar(&rs.PolicyConfigFile, "policy-config-file", rs.PolicyConfigFile, "File with descheduler policy configuration.") 54 | fs.BoolVar(&rs.DryRun, "dry-run", rs.DryRun, "execute descheduler in dry run mode.") 55 | // node-selector query causes descheduler to run only on nodes that matches the node labels in the query 56 | fs.StringVar(&rs.NodeSelector, "node-selector", rs.NodeSelector, "Selector (label query) to filter on, supports '=', '==', and '!='.(e.g. -l key1=value1,key2=value2)") 57 | // max-no-pods-to-evict limits the maximum number of pods to be evicted per node by descheduler. 58 | fs.IntVar(&rs.MaxNoOfPodsToEvictPerNode, "max-pods-to-evict-per-node", rs.MaxNoOfPodsToEvictPerNode, "Limits the maximum number of pods to be evicted per node by descheduler") 59 | // evict-local-storage-pods allows eviction of pods that are using local storage. This is false by default. 60 | fs.BoolVar(&rs.EvictLocalStoragePods, "evict-local-storage-pods", rs.EvictLocalStoragePods, "Enables evicting pods using local storage by descheduler") 61 | } 62 | -------------------------------------------------------------------------------- /cmd/descheduler/app/server.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright ©2020. The virtual-kubelet authors 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | // Package app implements a Server object for running the descheduler. 18 | package app 19 | 20 | import ( 21 | "flag" 22 | "io" 23 | 24 | "github.com/virtual-kubelet/tensile-kube/cmd/descheduler/app/options" 25 | "github.com/virtual-kubelet/tensile-kube/pkg/descheduler" 26 | 27 | "github.com/spf13/cobra" 28 | 29 | aflag "k8s.io/component-base/cli/flag" 30 | "k8s.io/component-base/logs" 31 | "k8s.io/klog" 32 | ) 33 | 34 | // NewDeschedulerCommand creates a *cobra.Command object with default parameters 35 | func NewDeschedulerCommand(out io.Writer) *cobra.Command { 36 | s := options.NewDeschedulerServer() 37 | cmd := &cobra.Command{ 38 | Use: "descheduler", 39 | Short: "descheduler", 40 | Long: `The descheduler evicts pods which may be bound to less desired nodes`, 41 | Run: func(cmd *cobra.Command, args []string) { 42 | logs.InitLogs() 43 | defer logs.FlushLogs() 44 | err := Run(s) 45 | if err != nil { 46 | klog.Errorf("%v", err) 47 | } 48 | }, 49 | } 50 | cmd.SetOutput(out) 51 | 52 | flags := cmd.Flags() 53 | flags.SetNormalizeFunc(aflag.WordSepNormalizeFunc) 54 | flags.AddGoFlagSet(flag.CommandLine) 55 | s.AddFlags(flags) 56 | return cmd 57 | } 58 | 59 | // Run starts run the descheduler 60 | func Run(rs *options.DeschedulerServer) error { 61 | return descheduler.Run(rs) 62 | } 63 | -------------------------------------------------------------------------------- /cmd/descheduler/app/version.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright ©2020. The virtual-kubelet authors 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package app 18 | 19 | import ( 20 | "fmt" 21 | "runtime" 22 | "strings" 23 | 24 | "github.com/spf13/cobra" 25 | ) 26 | 27 | var ( 28 | // gitCommit is a constant representing the source version that 29 | // generated this build. It should be set during build via -ldflags. 30 | gitCommit string 31 | // version is a constant representing the version tag that 32 | // generated this build. It should be set during build via -ldflags. 33 | version string 34 | // buildDate in ISO8601 format, output of $(date -u +'%Y-%m-%dT%H:%M:%SZ') 35 | //It should be set during build via -ldflags. 36 | buildDate string 37 | ) 38 | 39 | // Info holds the information related to descheduler app version. 40 | type Info struct { 41 | Major string `json:"major"` 42 | Minor string `json:"minor"` 43 | GitCommit string `json:"gitCommit"` 44 | GitVersion string `json:"gitVersion"` 45 | BuildDate string `json:"buildDate"` 46 | GoVersion string `json:"goVersion"` 47 | Compiler string `json:"compiler"` 48 | Platform string `json:"platform"` 49 | } 50 | 51 | // Get returns the overall codebase version. It's for detecting 52 | // what code a binary was built from. 53 | func Get() Info { 54 | majorVersion, minorVersion := splitVersion(version) 55 | return Info{ 56 | Major: majorVersion, 57 | Minor: minorVersion, 58 | GitCommit: gitCommit, 59 | GitVersion: version, 60 | BuildDate: buildDate, 61 | GoVersion: runtime.Version(), 62 | Compiler: runtime.Compiler, 63 | Platform: fmt.Sprintf("%s/%s", runtime.GOOS, runtime.GOARCH), 64 | } 65 | } 66 | 67 | // NewVersionCommand is for supporting the flag version 68 | func NewVersionCommand() *cobra.Command { 69 | var versionCmd = &cobra.Command{ 70 | Use: "version", 71 | Short: "Version of descheduler", 72 | Long: `Prints the version of descheduler.`, 73 | Run: func(cmd *cobra.Command, args []string) { 74 | fmt.Printf("Descheduler version %+v\n", Get()) 75 | }, 76 | } 77 | return versionCmd 78 | } 79 | 80 | // splitVersion splits the git version to generate major and minor versions needed. 81 | func splitVersion(version string) (string, string) { 82 | if version == "" { 83 | return "", "" 84 | } 85 | // A sample version would be of form v0.1.0-7-ge884046, so split at first '.' and 86 | // then return 0 and 1+(+ appended to follow semver convention) for major and minor versions. 87 | return strings.Trim(strings.Split(version, ".")[0], "v"), strings.Split(version, ".")[1] + "+" 88 | } 89 | -------------------------------------------------------------------------------- /cmd/descheduler/descheduler.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright ©2020. The virtual-kubelet authors 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package main 18 | 19 | import ( 20 | "flag" 21 | "fmt" 22 | "os" 23 | 24 | "github.com/virtual-kubelet/tensile-kube/cmd/descheduler/app" 25 | ) 26 | 27 | func main() { 28 | out := os.Stdout 29 | cmd := app.NewDeschedulerCommand(out) 30 | cmd.AddCommand(app.NewVersionCommand()) 31 | flag.CommandLine.Parse([]string{}) 32 | if err := cmd.Execute(); err != nil { 33 | fmt.Println(err) 34 | os.Exit(1) 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /cmd/provider/main.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright ©2020. The virtual-kubelet authors 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package main 18 | 19 | import ( 20 | "context" 21 | "fmt" 22 | "strings" 23 | "time" 24 | 25 | "github.com/sirupsen/logrus" 26 | "github.com/spf13/pflag" 27 | cli "github.com/virtual-kubelet/node-cli" 28 | logruscli "github.com/virtual-kubelet/node-cli/logrus" 29 | "github.com/virtual-kubelet/node-cli/opts" 30 | "github.com/virtual-kubelet/node-cli/provider" 31 | "github.com/virtual-kubelet/tensile-kube/pkg/util" 32 | "github.com/virtual-kubelet/virtual-kubelet/log" 33 | logruslogger "github.com/virtual-kubelet/virtual-kubelet/log/logrus" 34 | "golang.org/x/time/rate" 35 | kubeinformers "k8s.io/client-go/informers" 36 | "k8s.io/client-go/kubernetes" 37 | "k8s.io/client-go/util/workqueue" 38 | "k8s.io/klog" 39 | 40 | "github.com/virtual-kubelet/tensile-kube/pkg/controllers" 41 | k8sprovider "github.com/virtual-kubelet/tensile-kube/pkg/provider" 42 | ) 43 | 44 | var ( 45 | buildVersion = "N/A" 46 | buildTime = "N/A" 47 | k8sVersion = "v1.14.3" 48 | numberOfWorkers = 50 49 | ignoreLabels = "" 50 | enableControllers = "" 51 | enableServiceAccount = true 52 | providerName = "k8s" 53 | ) 54 | 55 | func main() { 56 | var cc k8sprovider.ClientConfig 57 | ctx := cli.ContextWithCancelOnSignal(context.Background()) 58 | flags := pflag.NewFlagSet("client", pflag.ContinueOnError) 59 | flags.IntVar(&cc.KubeClientBurst, "client-burst", 1000, "qpi burst for client cluster.") 60 | flags.IntVar(&cc.KubeClientQPS, "client-qps", 500, "qpi qps for client cluster.") 61 | flags.StringVar(&cc.ClientKubeConfigPath, "client-kubeconfig", "", "kube config for client cluster.") 62 | flags.StringVar(&ignoreLabels, "ignore-labels", util.BatchPodLabel, 63 | fmt.Sprintf("ignore-labels are the labels we would like to ignore when build pod for client clusters, "+ 64 | "usually these labels will infulence schedule, default %v, multi labels should be seperated by comma(,"+ 65 | ")", util.BatchPodLabel)) 66 | flags.StringVar(&enableControllers, "enable-controllers", "PVControllers,ServiceControllers", 67 | "support PVControllers,ServiceControllers, default, all of these") 68 | 69 | flags.BoolVar(&enableServiceAccount, "enable-serviceaccount", true, 70 | "enable service account for pods, like spark driver, mpi launcher") 71 | 72 | logger := logrus.StandardLogger() 73 | 74 | log.L = logruslogger.FromLogrus(logrus.NewEntry(logger)) 75 | logConfig := &logruscli.Config{LogLevel: "info"} 76 | 77 | o, err := opts.FromEnv() 78 | if err != nil { 79 | panic(err) 80 | } 81 | o.Provider = providerName 82 | o.PodSyncWorkers = numberOfWorkers 83 | o.Version = strings.Join([]string{k8sVersion, providerName, buildVersion}, "-") 84 | o.SyncPodsFromKubernetesRateLimiter = rateLimiter() 85 | o.DeletePodsFromKubernetesRateLimiter = rateLimiter() 86 | o.SyncPodStatusFromProviderRateLimiter = rateLimiter() 87 | node, err := cli.New(ctx, 88 | cli.WithBaseOpts(o), 89 | cli.WithProvider(providerName, func(cfg provider.InitConfig) (provider.Provider, error) { 90 | cfg.ConfigPath = o.KubeConfigPath 91 | provider, err := k8sprovider.NewVirtualK8S(cfg, &cc, ignoreLabels, enableServiceAccount, o) 92 | if err == nil { 93 | go RunController(ctx, provider, cfg.NodeName, numberOfWorkers) 94 | } 95 | return provider, err 96 | }), 97 | cli.WithCLIVersion(buildVersion, buildTime), 98 | cli.WithKubernetesNodeVersion(k8sVersion), 99 | // Adds flags and parsing for using logrus as the configured logger 100 | cli.WithPersistentFlags(logConfig.FlagSet()), 101 | cli.WithPersistentFlags(flags), 102 | cli.WithPersistentPreRunCallback(func() error { 103 | return logruscli.Configure(logConfig, logger) 104 | }), 105 | ) 106 | 107 | if err != nil { 108 | log.G(ctx).Fatal(err) 109 | } 110 | 111 | if err := node.Run(ctx); err != nil { 112 | log.G(ctx).Fatal(err) 113 | } 114 | } 115 | 116 | // RunController starts controllers for objects needed to be synced 117 | func RunController(ctx context.Context, p *k8sprovider.VirtualK8S, hostIP string, 118 | workers int) *controllers.ServiceController { 119 | master := p.GetMaster() 120 | client := p.GetClient() 121 | masterInformer := kubeinformers.NewSharedInformerFactory(master, 0) 122 | if masterInformer == nil { 123 | return nil 124 | } 125 | clientInformer := kubeinformers.NewSharedInformerFactory(client, 1*time.Minute) 126 | if clientInformer == nil { 127 | return nil 128 | } 129 | 130 | runningControllers := []controllers.Controller{buildCommonControllers(client, masterInformer, clientInformer)} 131 | 132 | controllerSlice := strings.Split(enableControllers, ",") 133 | for _, c := range controllerSlice { 134 | if len(c) == 0 { 135 | continue 136 | } 137 | switch c { 138 | case "PVControllers": 139 | pvCtrl := controllers.NewPVController(master, client, masterInformer, clientInformer, hostIP) 140 | runningControllers = append(runningControllers, pvCtrl) 141 | case "ServiceControllers": 142 | serviceCtrl := controllers.NewServiceController(master, client, masterInformer, clientInformer, p.GetNameSpaceLister()) 143 | runningControllers = append(runningControllers, serviceCtrl) 144 | default: 145 | klog.Warningf("Skip: %v", c) 146 | } 147 | } 148 | masterInformer.Start(ctx.Done()) 149 | clientInformer.Start(ctx.Done()) 150 | for _, ctrl := range runningControllers { 151 | go ctrl.Run(workers, ctx.Done()) 152 | } 153 | <-ctx.Done() 154 | return nil 155 | } 156 | 157 | func buildCommonControllers(client kubernetes.Interface, masterInformer, 158 | clientInformer kubeinformers.SharedInformerFactory) controllers.Controller { 159 | 160 | configMapRateLimiter := workqueue.NewItemExponentialFailureRateLimiter(time.Second, 30*time.Second) 161 | secretRateLimiter := workqueue.NewItemExponentialFailureRateLimiter(time.Second, 30*time.Second) 162 | 163 | return controllers.NewCommonController(client, masterInformer, clientInformer, configMapRateLimiter, secretRateLimiter) 164 | } 165 | 166 | func rateLimiter() workqueue.RateLimiter { 167 | return workqueue.NewMaxOfRateLimiter( 168 | workqueue.NewItemExponentialFailureRateLimiter(5*time.Millisecond, 10*time.Second), 169 | // 100 qps, 1000 bucket size. This is only for retry speed and its only the overall factor (not per item) 170 | &workqueue.BucketRateLimiter{Limiter: rate.NewLimiter(rate.Limit(100), 1000)}, 171 | ) 172 | } 173 | -------------------------------------------------------------------------------- /cmd/webhook/app/options.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright ©2020. The virtual-kubelet authors 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package app 18 | 19 | import ( 20 | "fmt" 21 | "net" 22 | 23 | "github.com/spf13/pflag" 24 | "github.com/virtual-kubelet/tensile-kube/pkg/util" 25 | ) 26 | 27 | var ( 28 | // Version is used for support printing version 29 | Version = "unknown" 30 | ) 31 | 32 | // ServerRunOptions defines the options of webhook server 33 | type ServerRunOptions struct { 34 | // webhook listen address 35 | Address string 36 | // listen port 37 | Port int 38 | // ca of tls 39 | TLSCA string 40 | // cert of tls 41 | TLSCert string 42 | // key of tls 43 | TLSKey string 44 | // kubeconfig file path if running out of cluster 45 | Kubeconfig string 46 | // url of master 47 | MasterURL string 48 | // run in the k8s 49 | InCluster bool 50 | // ignoreSelectorKeys represents those nodeSelector keys should not be converted 51 | // and it would affect the scheduling in then upper cluster 52 | IgnoreSelectorKeys string 53 | // ShowVersion is used for version 54 | ShowVersion bool 55 | } 56 | 57 | // NewServerRunOptions returns the run options 58 | func NewServerRunOptions() *ServerRunOptions { 59 | options := &ServerRunOptions{} 60 | options.addFlags() 61 | return options 62 | } 63 | 64 | func (s *ServerRunOptions) addFlags() { 65 | pflag.StringVar(&s.Address, "address", "0.0.0.0", "The address of scheduler manager.") 66 | pflag.IntVar(&s.Port, "port", 8080, "The port of scheduler manager.") 67 | pflag.StringVar(&s.TLSCert, "tlscert", "", "Path to TLS certificate file") 68 | pflag.StringVar(&s.TLSKey, "tlskey", "", "Path to TLS key file") 69 | pflag.StringVar(&s.TLSCA, "CA", "", "Path to certificate file") 70 | pflag.StringVar(&s.Kubeconfig, "kubeconfig", "", "Absolute path to the kubeconfig file.") 71 | pflag.StringVar(&s.MasterURL, "master", "", "Master url.") 72 | pflag.BoolVar(&s.InCluster, "incluster", false, "If this extender running in the cluster.") 73 | pflag.StringVar(&s.IgnoreSelectorKeys, "ignore-selector-keys", util.ClusterID, 74 | "IgnoreSelectorKeys represents those nodeSelector keys should not be converted, "+ 75 | "it would affect the scheduling in then upper cluster, multi values should split by comma(,)") 76 | pflag.BoolVar(&s.ShowVersion, "version", false, "Show version.") 77 | } 78 | 79 | // Validate is used for validate address 80 | func (s *ServerRunOptions) Validate() error { 81 | address := net.ParseIP(s.Address) 82 | if address.To4() == nil { 83 | return fmt.Errorf("%v is not a valid IP address", s.Address) 84 | } 85 | return nil 86 | } 87 | -------------------------------------------------------------------------------- /cmd/webhook/app/server.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright ©2020. The virtual-kubelet authors 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package app 18 | 19 | import ( 20 | "context" 21 | "crypto/tls" 22 | "crypto/x509" 23 | "fmt" 24 | "io/ioutil" 25 | "net" 26 | "net/http" 27 | "net/http/pprof" 28 | "strconv" 29 | "strings" 30 | "time" 31 | 32 | "k8s.io/client-go/tools/cache" 33 | "k8s.io/klog" 34 | 35 | "github.com/virtual-kubelet/tensile-kube/pkg/util" 36 | "github.com/virtual-kubelet/tensile-kube/pkg/webhook" 37 | kubeinformers "k8s.io/client-go/informers" 38 | ) 39 | 40 | // Run the server according to options 41 | func Run(s *ServerRunOptions) error { 42 | 43 | stopCh := util.SetupSignalHandler() 44 | 45 | client, err := util.NewClient(s.Kubeconfig) 46 | if err != nil { 47 | panic(err) 48 | } 49 | kubeInformer := kubeinformers.NewSharedInformerFactory(client, 0) 50 | if kubeInformer == nil { 51 | panic("informer nil") 52 | } 53 | pvcInformer := kubeInformer.Core().V1().PersistentVolumeClaims() 54 | pvcLister := pvcInformer.Lister() 55 | 56 | kubeInformer.Start(stopCh) 57 | 58 | if !cache.WaitForCacheSync(stopCh, pvcInformer.Informer().HasSynced) { 59 | panic("wait for cache sync failed") 60 | } 61 | seletorKeys := strings.Split(s.IgnoreSelectorKeys, ",") 62 | webHook := webhook.NewWebhookServer(pvcLister, seletorKeys) 63 | 64 | // Start debug monitor. 65 | mux := http.NewServeMux() 66 | mux.HandleFunc("/", webHook.Serve) 67 | mux.HandleFunc("/debug/pprof/", pprof.Index) 68 | mux.HandleFunc("/debug/pprof/profile", pprof.Profile) 69 | mux.HandleFunc("/debug/pprof/symbol", pprof.Symbol) 70 | mux.HandleFunc("/healthz", func(w http.ResponseWriter, r *http.Request) { 71 | fmt.Fprintf(w, "%s", "ok") 72 | }) 73 | 74 | server := &http.Server{ 75 | Addr: net.JoinHostPort(s.Address, strconv.Itoa(s.Port)), 76 | Handler: mux, 77 | ReadTimeout: 300 * time.Second, 78 | WriteTimeout: 300 * time.Second, 79 | } 80 | 81 | klog.V(1).Infof("listening on %v", server.Addr) 82 | if s.TLSCert != "" && s.TLSKey != "" { 83 | klog.V(1).Infof("using HTTPS service") 84 | tlsConfig, err := getTLSConfig(s) 85 | if err != nil { 86 | return err 87 | } 88 | server.TLSConfig = tlsConfig 89 | go func() { 90 | klog.Fatal(server.ListenAndServeTLS(s.TLSCert, s.TLSKey)) 91 | }() 92 | } else { 93 | go func() { 94 | klog.V(1).Infof("using HTTP service") 95 | klog.Fatal(server.ListenAndServe()) 96 | }() 97 | } 98 | 99 | select { 100 | case <-stopCh: 101 | klog.Info("http server received stop signal, waiting for all requests to finish") 102 | ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) 103 | defer cancel() 104 | if err := server.Shutdown(ctx); err != nil { 105 | klog.Error(err) 106 | } 107 | } 108 | return nil 109 | } 110 | 111 | func getTLSConfig(s *ServerRunOptions) (*tls.Config, error) { 112 | tlsConfig := &tls.Config{ 113 | NextProtos: []string{"http/1.1"}, 114 | // Certificates: []tls.Certificate{cert}, 115 | // Avoid fallback on insecure SSL protocols 116 | MinVersion: tls.VersionTLS10, 117 | } 118 | if s.TLSCA != "" { 119 | certPool := x509.NewCertPool() 120 | file, err := ioutil.ReadFile(s.TLSCA) 121 | if err != nil { 122 | return nil, fmt.Errorf("Could not read CA certificate: %v", err) 123 | } 124 | certPool.AppendCertsFromPEM(file) 125 | tlsConfig.ClientAuth = tls.RequireAndVerifyClientCert 126 | tlsConfig.ClientCAs = certPool 127 | } 128 | 129 | return tlsConfig, nil 130 | } 131 | -------------------------------------------------------------------------------- /cmd/webhook/webhook.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright ©2020. The virtual-kubelet authors 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package main 18 | 19 | import ( 20 | "flag" 21 | "fmt" 22 | "os" 23 | 24 | "github.com/spf13/pflag" 25 | "k8s.io/klog" 26 | 27 | "github.com/virtual-kubelet/tensile-kube/cmd/webhook/app" 28 | ) 29 | 30 | func main() { 31 | klog.InitFlags(nil) 32 | options := app.NewServerRunOptions() 33 | pflag.CommandLine.AddGoFlagSet(flag.CommandLine) 34 | pflag.Parse() 35 | if options.ShowVersion { 36 | fmt.Println(os.Args[0], app.Version) 37 | return 38 | } 39 | 40 | klog.Infof("starting webhook server.") 41 | if err := options.Validate(); err != nil { 42 | fmt.Fprintf(os.Stderr, "%v\n", err) 43 | os.Exit(1) 44 | } 45 | 46 | if err := app.Run(options); err != nil { 47 | fmt.Fprintf(os.Stderr, "%v\n", err) 48 | os.Exit(1) 49 | } 50 | } 51 | -------------------------------------------------------------------------------- /docs/multi.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/virtual-kubelet/tensile-kube/ad40680e1dcafba1ac50d36cd59c3f9d7ed2a7f3/docs/multi.png -------------------------------------------------------------------------------- /docs/tensile-kube.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/virtual-kubelet/tensile-kube/ad40680e1dcafba1ac50d36cd59c3f9d7ed2a7f3/docs/tensile-kube.png -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/virtual-kubelet/tensile-kube 2 | 3 | go 1.13 4 | 5 | require ( 6 | github.com/evanphx/json-patch v4.9.0+incompatible 7 | github.com/mattbaird/jsonpatch v0.0.0 8 | github.com/patrickmn/go-cache v2.1.0+incompatible 9 | github.com/sirupsen/logrus v1.4.2 10 | github.com/spf13/cobra v0.0.7 11 | github.com/spf13/pflag v1.0.5 12 | github.com/virtual-kubelet/node-cli v0.5.2-0.20210302175044-b3a8c550471d 13 | github.com/virtual-kubelet/virtual-kubelet v1.5.0 14 | golang.org/x/time v0.0.0-20190308202827-9d24e82272b4 15 | k8s.io/api v0.18.6 16 | k8s.io/apimachinery v0.18.6 17 | k8s.io/client-go v10.0.0+incompatible 18 | k8s.io/component-base v0.18.4 19 | k8s.io/klog v1.0.0 20 | k8s.io/kube-openapi v0.0.0-20200410163147-594e756bea31 // indirect 21 | k8s.io/kubernetes v1.18.19 22 | k8s.io/metrics v1.18.4 23 | sigs.k8s.io/descheduler v0.18.0 24 | ) 25 | 26 | replace ( 27 | github.com/Sirupsen/logrus => github.com/sirupsen/logrus v1.4.1 28 | github.com/mattbaird/jsonpatch => github.com/cwdsuzhou/jsonpatch v0.0.0-20210423033938-bbec2435b178 29 | k8s.io/api => k8s.io/api v0.18.4 30 | k8s.io/apiextensions-apiserver => k8s.io/apiextensions-apiserver v0.18.4 31 | k8s.io/apimachinery => k8s.io/apimachinery v0.18.4 32 | k8s.io/apiserver => k8s.io/apiserver v0.18.4 33 | k8s.io/cli-runtime => k8s.io/cli-runtime v0.18.4 34 | k8s.io/client-go => k8s.io/client-go v0.18.4 35 | k8s.io/cloud-provider => k8s.io/cloud-provider v0.18.4 36 | k8s.io/cluster-bootstrap => k8s.io/cluster-bootstrap v0.18.4 37 | k8s.io/code-generator => k8s.io/code-generator v0.18.4 38 | k8s.io/component-base => k8s.io/component-base v0.18.4 39 | k8s.io/cri-api => k8s.io/cri-api v0.18.4 40 | k8s.io/csi-translation-lib => k8s.io/csi-translation-lib v0.18.4 41 | k8s.io/kube-aggregator => k8s.io/kube-aggregator v0.18.4 42 | k8s.io/kube-controller-manager => k8s.io/kube-controller-manager v0.18.4 43 | k8s.io/kube-openapi => k8s.io/kube-openapi v0.0.0-20200410145947-61e04a5be9a6 44 | k8s.io/kube-proxy => k8s.io/kube-proxy v0.18.4 45 | k8s.io/kube-scheduler => k8s.io/kube-scheduler v0.18.4 46 | k8s.io/kubectl => k8s.io/kubectl v0.18.4 47 | k8s.io/kubelet => k8s.io/kubelet v0.18.4 48 | k8s.io/kubernetes => k8s.io/kubernetes v1.18.4 49 | k8s.io/legacy-cloud-providers => k8s.io/legacy-cloud-providers v0.18.4 50 | k8s.io/metrics => k8s.io/metrics v0.18.4 51 | k8s.io/sample-apiserver => k8s.io/sample-apiserver v0.18.4 52 | sigs.k8s.io/structured-merge-diff/v3 => sigs.k8s.io/structured-merge-diff/v3 v3.0.0 53 | ) 54 | -------------------------------------------------------------------------------- /manifeasts/descheduler.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: v1 3 | kind: ConfigMap 4 | metadata: 5 | name: descheduler-policy-configmap 6 | namespace: kube-system 7 | data: 8 | policy.yaml: | 9 | apiVersion: "descheduler/v1alpha1" 10 | kind: "DeschedulerPolicy" 11 | strategies: 12 | "LowNodeUtilization": 13 | enabled: false 14 | "RemoveDuplicates": 15 | enabled: false 16 | "RemovePodsViolatingInterPodAntiAffinity": 17 | enabled: false 18 | "RemovePodsViolatingNodeAffinity": 19 | enabled: false 20 | "RemovePodsViolatingNodeTaints": 21 | enabled: false 22 | "RemovePodsHavingTooManyRestarts": 23 | enabled: false 24 | "PodLifeTime": 25 | enabled: true 26 | params: 27 | maxPodLifeTimeSeconds: 180 # 7 days 28 | --- 29 | kind: ClusterRole 30 | apiVersion: rbac.authorization.k8s.io/v1 31 | metadata: 32 | name: descheduler-cluster-role 33 | namespace: kube-system 34 | rules: 35 | - apiGroups: [""] 36 | resources: ["events"] 37 | verbs: ["create", "update"] 38 | - apiGroups: [""] 39 | resources: ["nodes"] 40 | verbs: ["get", "watch", "list"] 41 | - apiGroups: [""] 42 | resources: ["pods"] 43 | verbs: ["create", "get", "watch", "list", "delete", "patch"] 44 | - apiGroups: [""] 45 | resources: ["pods/eviction"] 46 | verbs: ["create"] 47 | --- 48 | apiVersion: v1 49 | kind: ServiceAccount 50 | metadata: 51 | name: descheduler-sa 52 | namespace: kube-system 53 | --- 54 | apiVersion: rbac.authorization.k8s.io/v1 55 | kind: ClusterRoleBinding 56 | metadata: 57 | name: descheduler-cluster-role-binding 58 | namespace: kube-system 59 | roleRef: 60 | apiGroup: rbac.authorization.k8s.io 61 | kind: ClusterRole 62 | name: descheduler-cluster-role 63 | subjects: 64 | - name: descheduler-sa 65 | kind: ServiceAccount 66 | namespace: kube-system 67 | --- 68 | apiVersion: apps/v1 69 | kind: Deployment 70 | metadata: 71 | name: descheduler 72 | namespace: kube-system 73 | spec: 74 | selector: 75 | matchLabels: 76 | app: descheduler 77 | replicas: 1 78 | template: 79 | metadata: 80 | labels: 81 | app: descheduler 82 | spec: 83 | affinity: 84 | nodeAffinity: 85 | requiredDuringSchedulingIgnoredDuringExecution: 86 | nodeSelectorTerms: 87 | - matchExpressions: 88 | - key: type 89 | operator: NotIn 90 | values: 91 | - virtual-kubelet 92 | priorityClassName: system-cluster-critical 93 | containers: 94 | - name: descheduler 95 | image: descheduler:v1.0.0 96 | volumeMounts: 97 | - mountPath: /policy-dir 98 | name: policy-volume 99 | command: 100 | - "/descheduler" 101 | args: 102 | - "--policy-config-file=/policy-dir/policy.yaml" 103 | - "--v=3" 104 | - "--descheduling-interval=10s" 105 | restartPolicy: "Always" 106 | serviceAccountName: descheduler-sa 107 | volumes: 108 | - name: policy-volume 109 | configMap: 110 | name: descheduler-policy-configmap -------------------------------------------------------------------------------- /manifeasts/virtual-node.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: v1 3 | kind: ServiceAccount 4 | metadata: 5 | name: virtual-kubelet 6 | namespace: kube-system 7 | labels: 8 | k8s-app: virtual-kubelet 9 | --- 10 | apiVersion: rbac.authorization.k8s.io/v1beta1 11 | kind: ClusterRoleBinding 12 | metadata: 13 | name: virtual-kubelet 14 | namespace: kube-system 15 | subjects: 16 | - kind: ServiceAccount 17 | name: virtual-kubelet 18 | namespace: kube-system 19 | roleRef: 20 | apiGroup: rbac.authorization.k8s.io 21 | kind: ClusterRole 22 | name: cluster-admin 23 | --- 24 | apiVersion: v1 25 | kind: Secret 26 | metadata: 27 | name: virtual-kubelet 28 | namespace: kube-system 29 | type: Opaque 30 | data: 31 | cert.pem: ${cert.pem} 32 | key.pem: ${key.pem} 33 | ca.pem: ${ca.pem} 34 | --- 35 | apiVersion: apps/v1 36 | kind: Deployment 37 | metadata: 38 | name: virtual-kubelet 39 | namespace: kube-system 40 | labels: 41 | k8s-app: kubelet 42 | spec: 43 | replicas: 1 44 | selector: 45 | matchLabels: 46 | k8s-app: virtual-kubelet 47 | template: 48 | metadata: 49 | labels: 50 | pod-type: virtual-kubelet 51 | k8s-app: virtual-kubelet 52 | spec: 53 | affinity: 54 | nodeAffinity: 55 | requiredDuringSchedulingIgnoredDuringExecution: 56 | nodeSelectorTerms: 57 | - matchExpressions: 58 | - key: type 59 | operator: NotIn 60 | values: 61 | - virtual-kubelet 62 | podAntiAffinity: 63 | requiredDuringSchedulingIgnoredDuringExecution: 64 | - podAffinityTerm: 65 | labelSelector: 66 | matchExpressions: 67 | - key: pod-type 68 | operator: In 69 | values: 70 | - virtual-kubelet 71 | topologyKey: kubernetes.io/hostname 72 | tolerations: 73 | - effect: NoSchedule 74 | key: role 75 | value: not-vk 76 | operator: Equal 77 | hostNetwork: true 78 | containers: 79 | - name: virtual-kubelet 80 | image: virtual-node:v1.0.1 81 | imagePullPolicy: IfNotPresent 82 | env: 83 | - name: KUBELET_PORT 84 | value: "10450" 85 | - name: APISERVER_CERT_LOCATION 86 | value: /etc/virtual-kubelet/cert/cert.pem 87 | - name: APISERVER_KEY_LOCATION 88 | value: /etc/virtual-kubelet/cert/key.pem 89 | - name: APISERVER_CA_CERT_LOCATION 90 | value: /etc/virtual-kubelet/cert/ca.pem 91 | - name: DEFAULT_NODE_NAME 92 | value: virtual-kubelet 93 | - name: VKUBELET_POD_IP 94 | valueFrom: 95 | fieldRef: 96 | fieldPath: status.podIP 97 | volumeMounts: 98 | - name: credentials 99 | mountPath: "/etc/virtual-kubelet/cert" 100 | readOnly: true 101 | - name: kube 102 | mountPath: "/root" 103 | readOnly: true 104 | args: 105 | - --provider=k8s 106 | - --nodename=$(DEFAULT_NODE_NAME) 107 | - --disable-taint=true 108 | - --kube-api-qps=500 109 | - --kube-api-burst=1000 110 | - --client-qps=500 111 | - --client-burst=1000 112 | - --client-kubeconfig=/root/kube.config 113 | - --klog.v=5 114 | - --log-level=debug 115 | - --metrics-addr=:10455 116 | livenessProbe: 117 | tcpSocket: 118 | port: 10455 119 | initialDelaySeconds: 20 120 | periodSeconds: 20 121 | volumes: 122 | - name: credentials 123 | secret: 124 | secretName: virtual-kubelet 125 | - name: kube 126 | configMap: 127 | name: vk 128 | items: 129 | - key: kube.config 130 | path: kube.config 131 | defaultMode: 420 132 | serviceAccountName: virtual-kubelet 133 | --- 134 | apiVersion: v1 135 | kind: ConfigMap 136 | metadata: 137 | name: vk 138 | namespace: kube-system 139 | data: 140 | kube.config: ${kube.config} -------------------------------------------------------------------------------- /manifeasts/webhook.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: admissionregistration.k8s.io/v1beta1 2 | kind: MutatingWebhookConfiguration 3 | metadata: 4 | name: vk-mutator 5 | webhooks: 6 | - clientConfig: 7 | caBundle: ${caBundle} 8 | service: 9 | name: vk-mutator 10 | namespace: kube-system 11 | path: /mutate 12 | failurePolicy: Fail 13 | name: xxx 14 | rules: 15 | - apiGroups: 16 | - "" 17 | apiVersions: 18 | - v1 19 | operations: 20 | - CREATE 21 | resources: 22 | - pods 23 | sideEffects: None 24 | --- 25 | apiVersion: v1 26 | data: 27 | cert.pem: ${cert} 28 | key.pem: ${key} 29 | kind: Secret 30 | metadata: 31 | name: wbssecret 32 | namespace: kube-system 33 | type: Opaque 34 | --- 35 | apiVersion: v1 36 | kind: Service 37 | metadata: 38 | labels: 39 | app: vk-mutator 40 | name: vk-mutator 41 | namespace: kube-system 42 | spec: 43 | ports: 44 | - port: 443 45 | protocol: TCP 46 | targetPort: 443 47 | selector: 48 | app: vk-mutator 49 | sessionAffinity: None 50 | type: ClusterIP 51 | --- 52 | apiVersion: extensions/v1beta1 53 | kind: Deployment 54 | metadata: 55 | labels: 56 | app: vk-mutator 57 | name: webhook 58 | namespace: kube-system 59 | spec: 60 | progressDeadlineSeconds: 600 61 | replicas: 1 62 | selector: 63 | matchLabels: 64 | app: vk-mutator 65 | strategy: 66 | rollingUpdate: 67 | maxSurge: 25% 68 | maxUnavailable: 25% 69 | type: RollingUpdate 70 | template: 71 | metadata: 72 | labels: 73 | app: vk-mutator 74 | spec: 75 | containers: 76 | - args: 77 | - --tlscert=/root/cert.pem 78 | - --tlskey=/root/key.pem 79 | - --port=443 80 | - --v=6 81 | image: virtual-webhook:v1.0.0 82 | imagePullPolicy: Always 83 | name: webhook 84 | ports: 85 | - containerPort: 443 86 | protocol: TCP 87 | resources: {} 88 | terminationMessagePath: /dev/termination-log 89 | terminationMessagePolicy: File 90 | volumeMounts: 91 | - mountPath: /root 92 | name: wbssecret 93 | dnsPolicy: ClusterFirst 94 | volumes: 95 | - name: wbssecret 96 | secret: 97 | defaultMode: 420 98 | items: 99 | - key: key.pem 100 | path: key.pem 101 | - key: cert.pem 102 | path: cert.pem 103 | secretName: wbssecret 104 | -------------------------------------------------------------------------------- /pkg/common/node.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright ©2020. The virtual-kubelet authors 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package common 18 | 19 | import ( 20 | "fmt" 21 | "sync" 22 | 23 | corev1 "k8s.io/api/core/v1" 24 | ) 25 | 26 | // ProviderNode defines the virtual kubelet node of tensile-kube 27 | type ProviderNode struct { 28 | sync.Mutex 29 | *corev1.Node 30 | } 31 | 32 | // AddResource add resource to the node 33 | func (n *ProviderNode) AddResource(resource *Resource) error { 34 | if n.Node == nil { 35 | return fmt.Errorf("ProviderNode node has not init") 36 | } 37 | n.Lock() 38 | defer n.Unlock() 39 | vkResource := ConvertResource(n.Status.Capacity) 40 | 41 | vkResource.Add(resource) 42 | vkResource.SetCapacityToNode(n.Node) 43 | return nil 44 | } 45 | 46 | // SubResource sub resource from the node 47 | func (n *ProviderNode) SubResource(resource *Resource) error { 48 | if n.Node == nil { 49 | return fmt.Errorf("ProviderNode node has not init") 50 | } 51 | n.Lock() 52 | defer n.Unlock() 53 | vkResource := ConvertResource(n.Status.Capacity) 54 | 55 | vkResource.Sub(resource) 56 | vkResource.SetCapacityToNode(n.Node) 57 | return nil 58 | } 59 | 60 | // DeepCopy deepcopy node with lock, to avoid concurrent read-write 61 | func (n *ProviderNode) DeepCopy() *corev1.Node { 62 | n.Lock() 63 | node := n.Node.DeepCopy() 64 | n.Unlock() 65 | return node 66 | } 67 | -------------------------------------------------------------------------------- /pkg/common/resource.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright ©2020. The virtual-kubelet authors 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package common 18 | 19 | import ( 20 | corev1 "k8s.io/api/core/v1" 21 | "k8s.io/apimachinery/pkg/api/resource" 22 | "k8s.io/klog" 23 | ) 24 | 25 | // CustomResources is a key-value map for defining custom resources 26 | type CustomResources map[corev1.ResourceName]resource.Quantity 27 | 28 | // DeepCopy copy the custom resource 29 | func (cr CustomResources) DeepCopy() CustomResources { 30 | crCopy := CustomResources{} 31 | for name, quota := range cr { 32 | crCopy[name] = quota 33 | } 34 | return crCopy 35 | } 36 | 37 | // Equal return if resources is equal 38 | func (cr CustomResources) Equal(other CustomResources) bool { 39 | if len(cr) != len(other) { 40 | return false 41 | } 42 | for k, v := range cr { 43 | v1, ok := other[k] 44 | if !ok { 45 | return false 46 | } 47 | if !v1.Equal(v) { 48 | return false 49 | } 50 | } 51 | return true 52 | } 53 | 54 | // Resource defines the resources of a pod, it provides func `Add`, `Sub` 55 | // to make computation flexible 56 | type Resource struct { 57 | // CPU requirement 58 | CPU resource.Quantity 59 | // Memory requirement 60 | Memory resource.Quantity 61 | // Pods requirement 62 | Pods resource.Quantity 63 | // EphemeralStorage requirement 64 | EphemeralStorage resource.Quantity 65 | // Custom resource requirement 66 | Custom CustomResources 67 | } 68 | 69 | // NewResource returns A resource struct 70 | func NewResource() *Resource { 71 | return &Resource{ 72 | Custom: CustomResources{}, 73 | } 74 | } 75 | 76 | // Equal is for two resources comparision 77 | func (r *Resource) Equal(other *Resource) bool { 78 | return r.CPU.Equal(other.CPU) && r.Memory.Equal(other.Memory) && r.Pods.Equal(other.Pods) && r. 79 | EphemeralStorage.Equal(other.EphemeralStorage) && r.Custom.Equal(other.Custom) 80 | } 81 | 82 | // Add adds resource to the current one 83 | func (r *Resource) Add(nc *Resource) { 84 | r.CPU.Add(nc.CPU) 85 | r.Memory.Add(nc.Memory) 86 | r.Pods.Add(nc.Pods) 87 | r.EphemeralStorage.Add(nc.EphemeralStorage) 88 | if len(nc.Custom) == 0 { 89 | return 90 | } 91 | for name, quota := range nc.Custom { 92 | if r.Custom == nil { 93 | r.Custom = CustomResources{} 94 | } 95 | old := r.Custom[name] 96 | old.Add(quota) 97 | r.Custom[name] = old 98 | } 99 | } 100 | 101 | // Sub subs resource from the current one 102 | func (r *Resource) Sub(nc *Resource) { 103 | r.CPU.Sub(nc.CPU) 104 | r.Memory.Sub(nc.Memory) 105 | r.Pods.Sub(nc.Pods) 106 | r.EphemeralStorage.Sub(nc.EphemeralStorage) 107 | if len(nc.Custom) == 0 { 108 | return 109 | } 110 | for name, quota := range nc.Custom { 111 | if r.Custom == nil { 112 | r.Custom = CustomResources{} 113 | } 114 | old := r.Custom[name] 115 | old.Sub(quota) 116 | r.Custom[name] = old 117 | } 118 | } 119 | 120 | // SetCapacityToNode set the resource the virtual-kubelet node 121 | func (r *Resource) SetCapacityToNode(node *corev1.Node) { 122 | var CPU, mem, Pods, empStorage resource.Quantity 123 | if !r.CPU.IsZero() { 124 | CPU = r.CPU 125 | } 126 | if !r.Memory.IsZero() { 127 | mem = r.Memory 128 | } 129 | if !r.Pods.IsZero() { 130 | Pods = r.Pods 131 | } 132 | if !r.EphemeralStorage.IsZero() { 133 | empStorage = r.EphemeralStorage 134 | } 135 | node.Status.Capacity = corev1.ResourceList{ 136 | corev1.ResourceCPU: CPU, 137 | corev1.ResourceMemory: mem, 138 | corev1.ResourcePods: Pods, 139 | corev1.ResourceEphemeralStorage: empStorage, 140 | } 141 | for name, quota := range r.Custom { 142 | node.Status.Capacity[name] = quota 143 | } 144 | 145 | node.Status.Allocatable = node.Status.Capacity.DeepCopy() 146 | klog.Infof("%v", node.Status.Capacity) 147 | } 148 | 149 | // ConvertResource converts ResourceList to Resource 150 | func ConvertResource(resources corev1.ResourceList) *Resource { 151 | var cpu, mem, pods, empStorage resource.Quantity 152 | customResource := CustomResources{} 153 | for resourceName, quota := range resources { 154 | switch resourceName { 155 | case corev1.ResourceCPU: 156 | cpu = quota 157 | case corev1.ResourceMemory: 158 | mem = quota 159 | case corev1.ResourcePods: 160 | pods = quota 161 | case corev1.ResourceEphemeralStorage: 162 | empStorage = quota 163 | default: 164 | customResource[resourceName] = quota 165 | } 166 | } 167 | return &Resource{ 168 | CPU: cpu, 169 | Memory: mem, 170 | Pods: pods, 171 | EphemeralStorage: empStorage, 172 | Custom: customResource, 173 | } 174 | } 175 | -------------------------------------------------------------------------------- /pkg/common/resource_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright ©2020. The virtual-kubelet authors 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package common 18 | 19 | import ( 20 | "testing" 21 | 22 | "github.com/virtual-kubelet/tensile-kube/pkg/testbase" 23 | "k8s.io/apimachinery/pkg/api/resource" 24 | ) 25 | 26 | func TestResource(t *testing.T) { 27 | node := ProviderNode{ 28 | Node: testbase.NodeForTest(), 29 | } 30 | node1 := ProviderNode{ 31 | Node: node.Node.DeepCopy(), 32 | } 33 | cap := &Resource{ 34 | CPU: resource.MustParse("1"), 35 | Memory: resource.MustParse("1Gi"), 36 | Custom: CustomResources{"ip": resource.MustParse("30")}, 37 | } 38 | node.AddResource(cap) 39 | if !node.Status.Capacity["cpu"].Equal(resource.MustParse("51")) || 40 | !node.Status.Capacity["memory"].Equal(resource.MustParse("51Gi")) { 41 | t.Fatalf("nodeAddCapacity unexpected %v", node.Status.Capacity) 42 | } 43 | 44 | node1.SubResource(cap) 45 | if !node1.Status.Capacity["cpu"].Equal(resource.MustParse("49")) || 46 | !node1.Status.Capacity["memory"].Equal(resource.MustParse("49Gi")) { 47 | t.Fatalf("nodeRemoveCapacity unexpected %v", node1.Status.Capacity) 48 | } 49 | } 50 | -------------------------------------------------------------------------------- /pkg/controllers/base.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright ©2020. The virtual-kubelet authors 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package controllers 18 | 19 | // Controller is an interface which should only contains a method Run 20 | type Controller interface { 21 | // Run starts run a controller 22 | Run(int, <-chan struct{}) 23 | } 24 | -------------------------------------------------------------------------------- /pkg/controllers/common_controller_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright ©2020. The virtual-kubelet authors 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package controllers 18 | 19 | import ( 20 | "context" 21 | "reflect" 22 | "testing" 23 | "time" 24 | 25 | v1 "k8s.io/api/core/v1" 26 | "k8s.io/apimachinery/pkg/api/errors" 27 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 28 | "k8s.io/apimachinery/pkg/util/wait" 29 | "k8s.io/client-go/informers" 30 | "k8s.io/client-go/kubernetes/fake" 31 | "k8s.io/client-go/util/workqueue" 32 | "k8s.io/kubernetes/pkg/controller" 33 | ) 34 | 35 | type commonTestBase struct { 36 | c *CommonController 37 | masterInformer informers.SharedInformerFactory 38 | clientInformer informers.SharedInformerFactory 39 | master *fake.Clientset 40 | client *fake.Clientset 41 | } 42 | 43 | func TestCommonController_RunUpdateConfigMap(t *testing.T) { 44 | configMap := newConfigMap() 45 | configMap.Data = map[string]string{"test": "test1"} 46 | configMapCopy := configMap.DeepCopy() 47 | configMapCopy1 := configMap.DeepCopy() 48 | configMapCopy1.Annotations = map[string]string{} 49 | cases := []struct { 50 | name string 51 | configMap *v1.ConfigMap 52 | shouldChanged bool 53 | }{ 54 | { 55 | name: "should update", 56 | configMap: configMapCopy, 57 | shouldChanged: true, 58 | }, 59 | { 60 | name: "should not update", 61 | configMap: configMapCopy1, 62 | shouldChanged: false, 63 | }, 64 | } 65 | for _, c := range cases { 66 | t.Run(c.name, func(t *testing.T) { 67 | b := newCommonController() 68 | stopCh := make(chan struct{}) 69 | go test(b.c, 1, stopCh) 70 | b.clientInformer.Start(stopCh) 71 | b.masterInformer.Start(stopCh) 72 | if _, err := b.master.CoreV1().ConfigMaps(c.configMap.Namespace).Update( 73 | context.TODO(), c.configMap, metav1.UpdateOptions{}); err != nil { 74 | t.Fatal(err) 75 | } 76 | err := wait.Poll(10*time.Millisecond, 10*time.Second, func() (bool, error) { 77 | cm, err := b.c.clientConfigMapLister.ConfigMaps(c.configMap.Namespace).Get(configMap.Name) 78 | if err != nil { 79 | return false, nil 80 | } 81 | old := newConfigMap() 82 | t.Logf("New data: %v, old data: %v", cm.Data, old.Data) 83 | if reflect.DeepEqual(cm.Data, old.Data) != c.shouldChanged { 84 | t.Logf("New data: %v, old data: %v", cm.Data, old.Data) 85 | t.Log("configMap update satisfied") 86 | return true, nil 87 | } 88 | return false, nil 89 | }) 90 | if err != nil { 91 | t.Error("configMap update failed") 92 | } 93 | }) 94 | } 95 | } 96 | 97 | func TestCommonController_RunUpdateSecret(t *testing.T) { 98 | 99 | secret := newSecret() 100 | secret.StringData = map[string]string{"test": "test1"} 101 | secretCopy := secret.DeepCopy() 102 | secretCopy1 := secret.DeepCopy() 103 | secretCopy1.Annotations = map[string]string{} 104 | cases := []struct { 105 | name string 106 | secret *v1.Secret 107 | shouldChanged bool 108 | }{ 109 | { 110 | name: "should update", 111 | secret: secretCopy, 112 | shouldChanged: true, 113 | }, 114 | { 115 | name: "should not update", 116 | secret: secretCopy1, 117 | shouldChanged: false, 118 | }, 119 | } 120 | for _, c := range cases { 121 | t.Run(c.name, func(t *testing.T) { 122 | b := newCommonController() 123 | stopCh := make(chan struct{}) 124 | go test(b.c, 1, stopCh) 125 | b.clientInformer.Start(stopCh) 126 | b.masterInformer.Start(stopCh) 127 | if _, err := b.master.CoreV1().Secrets(c.secret.Namespace).Update(context.TODO(), 128 | c.secret, metav1.UpdateOptions{}); err != nil { 129 | t.Fatal(err) 130 | } 131 | err := wait.Poll(10*time.Millisecond, 10*time.Second, func() (bool, error) { 132 | newSvc, err := b.c.clientSecretLister.Secrets(c.secret.Namespace).Get(c.secret.Name) 133 | if err != nil { 134 | return false, nil 135 | } 136 | oldSvc := newSecret() 137 | t.Logf("New data: %v, old data: %v", newSvc.StringData, oldSvc.StringData) 138 | if reflect.DeepEqual(newSvc.StringData, oldSvc.StringData) != c.shouldChanged { 139 | t.Log("secret update satisfied") 140 | return true, nil 141 | } 142 | return false, nil 143 | }) 144 | if err != nil { 145 | t.Error("secret update failed") 146 | } 147 | }) 148 | } 149 | } 150 | 151 | func TestCommonController_RunDeleteConfigMap(t *testing.T) { 152 | ctx := context.TODO() 153 | cm := newConfigMap() 154 | cm.Data = map[string]string{"test": "test1"} 155 | cm1 := cm.DeepCopy() 156 | cm1.Annotations = map[string]string{} 157 | var err error 158 | cases := []struct { 159 | name string 160 | configMap *v1.ConfigMap 161 | deleted bool 162 | }{ 163 | { 164 | name: "should not delete", 165 | configMap: cm1, 166 | deleted: false, 167 | }, 168 | { 169 | name: "should delete", 170 | configMap: cm, 171 | deleted: true, 172 | }, 173 | } 174 | 175 | for _, c := range cases { 176 | t.Run(c.name, func(t *testing.T) { 177 | b := newCommonController() 178 | stopCh := make(chan struct{}) 179 | go test(b.c, 1, stopCh) 180 | b.clientInformer.Start(stopCh) 181 | b.masterInformer.Start(stopCh) 182 | delete := false 183 | err = b.master.CoreV1().ConfigMaps(c.configMap.Namespace).Delete(ctx, 184 | c.configMap.Name, metav1.DeleteOptions{}) 185 | if err != nil { 186 | t.Fatal(err) 187 | } 188 | _, err = b.c.clientConfigMapLister.ConfigMaps(c.configMap.Namespace).Get(c.configMap.Name) 189 | if err != nil { 190 | if !errors.IsNotFound(err) { 191 | t.Fatal(err) 192 | } 193 | delete = true 194 | } 195 | if delete == c.deleted { 196 | t.Log("configmap delete satisfied") 197 | } 198 | _, err = b.master.CoreV1().ConfigMaps(c.configMap.Namespace).Create(ctx, c.configMap, metav1.CreateOptions{}) 199 | }) 200 | } 201 | } 202 | 203 | func TestCommonController_RunDeleteSecret(t *testing.T) { 204 | ctx := context.TODO() 205 | secret := newSecret() 206 | secret.StringData = map[string]string{"test": "test1"} 207 | secret1 := secret.DeepCopy() 208 | secret1.Annotations = map[string]string{} 209 | cases := []struct { 210 | name string 211 | secret *v1.Secret 212 | deleted bool 213 | }{ 214 | { 215 | name: "should not delete", 216 | secret: secret1, 217 | deleted: false, 218 | }, 219 | { 220 | name: "should delete", 221 | secret: secret, 222 | deleted: true, 223 | }, 224 | } 225 | for _, c := range cases { 226 | t.Run(c.name, func(t *testing.T) { 227 | b := newCommonController() 228 | stopCh := make(chan struct{}) 229 | go test(b.c, 1, stopCh) 230 | b.clientInformer.Start(stopCh) 231 | b.masterInformer.Start(stopCh) 232 | delete := false 233 | err := b.master.CoreV1().Secrets(c.secret.Namespace).Delete(ctx, c.secret.Name, 234 | metav1.DeleteOptions{}) 235 | if err != nil { 236 | t.Fatal(err) 237 | } 238 | _, err = b.c.clientSecretLister.Secrets(c.secret.Namespace).Get(c.secret.Name) 239 | if err != nil { 240 | if !errors.IsNotFound(err) { 241 | t.Fatal(err) 242 | } 243 | delete = true 244 | } 245 | if delete == c.deleted { 246 | t.Log("secret delete satisfied") 247 | } 248 | _, err = b.master.CoreV1().Secrets(c.secret.Namespace).Create(ctx, 249 | c.secret, metav1.CreateOptions{}) 250 | }) 251 | } 252 | } 253 | 254 | func newCommonController() *commonTestBase { 255 | client := fake.NewSimpleClientset(newConfigMap(), newSecret()) 256 | master := fake.NewSimpleClientset(newConfigMap(), newSecret()) 257 | 258 | clientInformer := informers.NewSharedInformerFactory(client, controller.NoResyncPeriodFunc()) 259 | masterInformer := informers.NewSharedInformerFactory(master, controller.NoResyncPeriodFunc()) 260 | 261 | configMapRateLimiter := workqueue.NewItemExponentialFailureRateLimiter(time.Second, 30*time.Second) 262 | secretRateLimiter := workqueue.NewItemExponentialFailureRateLimiter(time.Second, 30*time.Second) 263 | controller := NewCommonController(client, masterInformer, clientInformer, configMapRateLimiter, secretRateLimiter) 264 | c := controller.(*CommonController) 265 | return &commonTestBase{ 266 | c: c, 267 | masterInformer: masterInformer, 268 | clientInformer: clientInformer, 269 | master: master, 270 | client: client, 271 | } 272 | } 273 | 274 | func test(c Controller, t int, stopCh chan struct{}) { 275 | go func() { 276 | timer := time.NewTimer(10 * time.Second) 277 | defer timer.Stop() 278 | select { 279 | case <-timer.C: 280 | close(stopCh) 281 | case <-stopCh: 282 | return 283 | } 284 | }() 285 | c.Run(t, stopCh) 286 | } 287 | 288 | func newConfigMap() *v1.ConfigMap { 289 | return &v1.ConfigMap{ 290 | TypeMeta: metav1.TypeMeta{ 291 | Kind: "ConfigMap", 292 | APIVersion: "v1", 293 | }, 294 | ObjectMeta: metav1.ObjectMeta{ 295 | Name: "test", 296 | Namespace: "default", 297 | Annotations: map[string]string{"global": "true"}, 298 | }, 299 | Data: map[string]string{"test": "test"}, 300 | } 301 | } 302 | 303 | func newSecret() *v1.Secret { 304 | return &v1.Secret{ 305 | TypeMeta: metav1.TypeMeta{ 306 | Kind: "ConfigMap", 307 | APIVersion: "v1", 308 | }, 309 | ObjectMeta: metav1.ObjectMeta{ 310 | Name: "test", 311 | Namespace: "default", 312 | Annotations: map[string]string{"global": "true"}, 313 | }, 314 | } 315 | } 316 | -------------------------------------------------------------------------------- /pkg/controllers/utils.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright ©2020. The virtual-kubelet authors 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package controllers 18 | 19 | import ( 20 | "context" 21 | "encoding/json" 22 | 23 | v1 "k8s.io/api/core/v1" 24 | apierrs "k8s.io/apimachinery/pkg/api/errors" 25 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 26 | "k8s.io/client-go/kubernetes" 27 | corelisters "k8s.io/client-go/listers/core/v1" 28 | 29 | "github.com/virtual-kubelet/tensile-kube/pkg/util" 30 | ) 31 | 32 | func ensureNamespace(ns string, client kubernetes.Interface, nsLister corelisters.NamespaceLister) error { 33 | _, err := nsLister.Get(ns) 34 | if err == nil { 35 | return nil 36 | } 37 | if !apierrs.IsNotFound(err) { 38 | return err 39 | } 40 | if _, err = client.CoreV1().Namespaces().Create(context.TODO(), &v1.Namespace{ 41 | ObjectMeta: metav1.ObjectMeta{Name: ns}}, metav1.CreateOptions{}); err != nil { 42 | if !apierrs.IsAlreadyExists(err) { 43 | return err 44 | } 45 | return nil 46 | } 47 | 48 | return err 49 | } 50 | 51 | func filterPVC(pvcInSub *v1.PersistentVolumeClaim, hostIP string) error { 52 | labelSelector := pvcInSub.Spec.Selector.DeepCopy() 53 | pvcInSub.Spec.Selector = nil 54 | util.TrimObjectMeta(&pvcInSub.ObjectMeta) 55 | SetObjectGlobal(&pvcInSub.ObjectMeta) 56 | if labelSelector != nil { 57 | labelStr, err := json.Marshal(labelSelector) 58 | if err != nil { 59 | return err 60 | } 61 | pvcInSub.Annotations["labelSelector"] = string(labelStr) 62 | } 63 | if len(pvcInSub.Annotations[util.SelectedNodeKey]) != 0 { 64 | pvcInSub.Annotations[util.SelectedNodeKey] = hostIP 65 | } 66 | return nil 67 | } 68 | 69 | func filterPV(pvInSub *v1.PersistentVolume, hostIP string) { 70 | util.TrimObjectMeta(&pvInSub.ObjectMeta) 71 | if pvInSub.Annotations == nil { 72 | pvInSub.Annotations = make(map[string]string) 73 | } 74 | if pvInSub.Spec.NodeAffinity == nil { 75 | return 76 | } 77 | if pvInSub.Spec.NodeAffinity.Required == nil { 78 | return 79 | } 80 | terms := pvInSub.Spec.NodeAffinity.Required.NodeSelectorTerms 81 | for k, v := range pvInSub.Spec.NodeAffinity.Required.NodeSelectorTerms { 82 | mf := v.MatchFields 83 | me := v.MatchExpressions 84 | for k, val := range v.MatchFields { 85 | if val.Key == util.HostNameKey || val.Key == util.BetaHostNameKey { 86 | val.Values = []string{hostIP} 87 | } 88 | mf[k] = val 89 | } 90 | for k, val := range v.MatchExpressions { 91 | if val.Key == util.HostNameKey || val.Key == util.BetaHostNameKey { 92 | val.Values = []string{hostIP} 93 | } 94 | me[k] = val 95 | } 96 | terms[k].MatchFields = mf 97 | terms[k].MatchExpressions = me 98 | } 99 | pvInSub.Spec.NodeAffinity.Required.NodeSelectorTerms = terms 100 | return 101 | } 102 | 103 | func filterCommon(meta *metav1.ObjectMeta) error { 104 | util.TrimObjectMeta(meta) 105 | SetObjectGlobal(meta) 106 | return nil 107 | } 108 | 109 | func filterService(serviceInSub *v1.Service) error { 110 | labelSelector := serviceInSub.Spec.Selector 111 | serviceInSub.Spec.Selector = nil 112 | if serviceInSub.Spec.ClusterIP != "None" { 113 | serviceInSub.Spec.ClusterIP = "" 114 | } 115 | util.TrimObjectMeta(&serviceInSub.ObjectMeta) 116 | SetObjectGlobal(&serviceInSub.ObjectMeta) 117 | if labelSelector == nil { 118 | return nil 119 | } 120 | labelStr, err := json.Marshal(labelSelector) 121 | if err != nil { 122 | return err 123 | } 124 | serviceInSub.Annotations["labelSelector"] = string(labelStr) 125 | return nil 126 | } 127 | 128 | // CheckGlobalLabelEqual checks if two objects both has the global label 129 | func CheckGlobalLabelEqual(obj, clone *metav1.ObjectMeta) bool { 130 | oldGlobal := IsObjectGlobal(obj) 131 | if !oldGlobal { 132 | return false 133 | } 134 | newGlobal := IsObjectGlobal(clone) 135 | if !newGlobal { 136 | return false 137 | } 138 | return true 139 | } 140 | 141 | // IsObjectGlobal return if an object is global 142 | func IsObjectGlobal(obj *metav1.ObjectMeta) bool { 143 | if obj.Annotations == nil { 144 | return false 145 | } 146 | 147 | if obj.Annotations[util.GlobalLabel] == "true" { 148 | return true 149 | } 150 | 151 | return false 152 | } 153 | 154 | // SetObjectGlobal add global annotation to an object 155 | func SetObjectGlobal(obj *metav1.ObjectMeta) { 156 | if obj.Annotations == nil { 157 | obj.Annotations = map[string]string{} 158 | } 159 | obj.Annotations[util.GlobalLabel] = "true" 160 | } 161 | -------------------------------------------------------------------------------- /pkg/controllers/utils_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright ©2020. The virtual-kubelet authors 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package controllers 18 | 19 | import ( 20 | "testing" 21 | 22 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 23 | 24 | "github.com/virtual-kubelet/tensile-kube/pkg/util" 25 | ) 26 | 27 | func TestCheckGlobalLabelEqual(t *testing.T) { 28 | type args struct { 29 | obj *metav1.ObjectMeta 30 | clone *metav1.ObjectMeta 31 | } 32 | tests := []struct { 33 | name string 34 | args args 35 | want bool 36 | }{ 37 | { 38 | name: "label nil", 39 | args: args{ 40 | obj: &metav1.ObjectMeta{ 41 | Labels: nil, 42 | Annotations: nil, 43 | }, 44 | clone: &metav1.ObjectMeta{ 45 | Labels: nil, 46 | Annotations: nil, 47 | }, 48 | }, 49 | want: false, 50 | }, 51 | { 52 | name: "one of labels nil", 53 | args: args{ 54 | obj: &metav1.ObjectMeta{ 55 | Annotations: map[string]string{util.GlobalLabel: "true"}, 56 | }, 57 | clone: &metav1.ObjectMeta{ 58 | Annotations: nil, 59 | }, 60 | }, 61 | want: false, 62 | }, 63 | { 64 | name: "both labels not nil", 65 | args: args{ 66 | obj: &metav1.ObjectMeta{ 67 | Annotations: map[string]string{util.GlobalLabel: "true"}, 68 | }, 69 | clone: &metav1.ObjectMeta{ 70 | Annotations: map[string]string{util.GlobalLabel: "true"}, 71 | }, 72 | }, 73 | want: true, 74 | }, 75 | } 76 | for _, tt := range tests { 77 | t.Run(tt.name, func(t *testing.T) { 78 | if got := CheckGlobalLabelEqual(tt.args.obj, tt.args.clone); got != tt.want { 79 | t.Errorf("CheckGlobalLabelEqual() = %v, want %v", got, tt.want) 80 | } 81 | }) 82 | } 83 | } 84 | 85 | func TestIsObjectGlobal(t *testing.T) { 86 | type args struct { 87 | obj *metav1.ObjectMeta 88 | } 89 | tests := []struct { 90 | name string 91 | args args 92 | want bool 93 | }{ 94 | { 95 | name: "label nil", 96 | args: args{ 97 | obj: &metav1.ObjectMeta{ 98 | Annotations: nil, 99 | }, 100 | }, 101 | want: false, 102 | }, 103 | { 104 | name: "can not find annotation", 105 | args: args{ 106 | obj: &metav1.ObjectMeta{ 107 | Annotations: map[string]string{"test": "true"}, 108 | }, 109 | }, 110 | want: false, 111 | }, 112 | { 113 | name: "both labels not nil", 114 | args: args{ 115 | obj: &metav1.ObjectMeta{ 116 | Annotations: map[string]string{util.GlobalLabel: "true"}, 117 | }, 118 | }, 119 | want: true, 120 | }, 121 | } 122 | for _, tt := range tests { 123 | t.Run(tt.name, func(t *testing.T) { 124 | if got := IsObjectGlobal(tt.args.obj); got != tt.want { 125 | t.Errorf("IsObjectGlobal() = %v, want %v", got, tt.want) 126 | } 127 | }) 128 | } 129 | } 130 | 131 | func TestSetObjectGlobal(t *testing.T) { 132 | type args struct { 133 | obj *metav1.ObjectMeta 134 | } 135 | tests := []struct { 136 | name string 137 | args args 138 | }{ 139 | { 140 | name: "set global label", 141 | args: args{ 142 | obj: &metav1.ObjectMeta{ 143 | Labels: nil, 144 | Annotations: nil, 145 | }, 146 | }, 147 | }, 148 | } 149 | for _, tt := range tests { 150 | t.Run(tt.name, func(t *testing.T) { 151 | SetObjectGlobal(tt.args.obj) 152 | if !IsObjectGlobal(tt.args.obj) { 153 | t.Fatal("Set Object Global failed") 154 | } 155 | }) 156 | } 157 | } 158 | -------------------------------------------------------------------------------- /pkg/descheduler/descheduler.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright ©2020. The virtual-kubelet authors 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package descheduler 18 | 19 | import ( 20 | "context" 21 | "fmt" 22 | 23 | v1 "k8s.io/api/core/v1" 24 | "k8s.io/apimachinery/pkg/util/wait" 25 | "k8s.io/client-go/informers" 26 | clientset "k8s.io/client-go/kubernetes" 27 | "k8s.io/client-go/rest" 28 | "k8s.io/klog" 29 | "sigs.k8s.io/descheduler/pkg/api" 30 | "sigs.k8s.io/descheduler/pkg/descheduler" 31 | "sigs.k8s.io/descheduler/pkg/descheduler/evictions/utils" 32 | nodeutil "sigs.k8s.io/descheduler/pkg/descheduler/node" 33 | 34 | "github.com/virtual-kubelet/tensile-kube/cmd/descheduler/app/options" 35 | "github.com/virtual-kubelet/tensile-kube/pkg/descheduler/evictions" 36 | "github.com/virtual-kubelet/tensile-kube/pkg/descheduler/strategies" 37 | "github.com/virtual-kubelet/tensile-kube/pkg/util" 38 | ) 39 | 40 | // Run start a descheduler server 41 | func Run(rs *options.DeschedulerServer) error { 42 | ctx := context.Background() 43 | rsclient, err := util.NewClient(rs.KubeconfigFile, func(c *rest.Config) { 44 | c.QPS = 100 45 | c.Burst = 200 46 | }) 47 | if err != nil { 48 | return err 49 | } 50 | rs.Client = rsclient 51 | 52 | deschedulerPolicy, err := descheduler.LoadPolicyConfig(rs.PolicyConfigFile) 53 | if err != nil { 54 | return err 55 | } 56 | if deschedulerPolicy == nil { 57 | return fmt.Errorf("deschedulerPolicy is nil") 58 | } 59 | 60 | evictionPolicyGroupVersion, err := utils.SupportEviction(rs.Client) 61 | if err != nil || len(evictionPolicyGroupVersion) == 0 { 62 | return err 63 | } 64 | 65 | stopChannel := make(chan struct{}) 66 | return RunDeschedulerStrategies(ctx, rs, deschedulerPolicy, evictionPolicyGroupVersion, stopChannel) 67 | } 68 | 69 | type strategyFunction func(ctx context.Context, client clientset.Interface, strategy api.DeschedulerStrategy, nodes []*v1.Node, evictLocalStoragePods bool, podEvictor *evictions.PodEvictor) 70 | 71 | // RunDeschedulerStrategies runs the strategies 72 | func RunDeschedulerStrategies(ctx context.Context, rs *options.DeschedulerServer, deschedulerPolicy *api.DeschedulerPolicy, evictionPolicyGroupVersion string, stopChannel chan struct{}) error { 73 | sharedInformerFactory := informers.NewSharedInformerFactory(rs.Client, 0) 74 | nodeInformer := sharedInformerFactory.Core().V1().Nodes() 75 | // just trigger sharedInformerFactory add node informers 76 | nodeInformer.Informer() 77 | 78 | sharedInformerFactory.Start(stopChannel) 79 | sharedInformerFactory.WaitForCacheSync(stopChannel) 80 | 81 | strategyFuncs := map[string]strategyFunction{ 82 | "PodLifeTime": strategies.PodLifeTime, 83 | } 84 | 85 | unschedulableCache := util.NewUnschedulableCache() 86 | count := 0 87 | wait.Until(func() { 88 | count++ 89 | nodes, err := nodeutil.ReadyNodes(ctx, rs.Client, nodeInformer, rs.NodeSelector, stopChannel) 90 | if err != nil { 91 | klog.V(1).Infof("Unable to get ready nodes: %v", err) 92 | close(stopChannel) 93 | return 94 | } 95 | 96 | if len(nodes) <= 1 { 97 | klog.V(1).Infof("The cluster size is 0 or 1 meaning eviction causes service disruption or degradation. So aborting..") 98 | close(stopChannel) 99 | return 100 | } 101 | podEvictor := evictions.NewPodEvictor( 102 | rs.Client, 103 | evictionPolicyGroupVersion, 104 | rs.MaxNoOfPodsToEvictPerNode, 105 | nodes, unschedulableCache, 106 | ) 107 | if count%10 == 0 { 108 | count = count % 10 109 | podEvictor.CheckUnschedulablePods = true 110 | } 111 | for name, f := range strategyFuncs { 112 | if strategy := deschedulerPolicy.Strategies[api.StrategyName(name)]; strategy.Enabled { 113 | f(ctx, rs.Client, strategy, nodes, rs.EvictLocalStoragePods, podEvictor) 114 | } 115 | } 116 | 117 | // If there was no interval specified, send a signal to the stopChannel to end the wait.Until loop after 1 iteration 118 | if rs.DeschedulingInterval.Seconds() == 0 { 119 | close(stopChannel) 120 | } 121 | }, rs.DeschedulingInterval, stopChannel) 122 | 123 | return nil 124 | } 125 | -------------------------------------------------------------------------------- /pkg/descheduler/evictions/evictions.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright ©2020. The virtual-kubelet authors 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package evictions 18 | 19 | import ( 20 | "context" 21 | "fmt" 22 | "strconv" 23 | "sync" 24 | "time" 25 | 26 | v1 "k8s.io/api/core/v1" 27 | policy "k8s.io/api/policy/v1beta1" 28 | apierrors "k8s.io/apimachinery/pkg/api/errors" 29 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 30 | mergetypes "k8s.io/apimachinery/pkg/types" 31 | clientset "k8s.io/client-go/kubernetes" 32 | "k8s.io/client-go/kubernetes/scheme" 33 | clientcorev1 "k8s.io/client-go/kubernetes/typed/core/v1" 34 | "k8s.io/client-go/tools/record" 35 | "k8s.io/klog" 36 | "sigs.k8s.io/descheduler/pkg/descheduler/evictions" 37 | eutils "sigs.k8s.io/descheduler/pkg/descheduler/evictions/utils" 38 | 39 | "github.com/virtual-kubelet/tensile-kube/pkg/util" 40 | ) 41 | 42 | const ( 43 | unschedulableNode = "unschedulable-node" 44 | ) 45 | 46 | // nodePodEvictedCount keeps count of pods evicted on node 47 | type nodePodEvictedCount map[*v1.Node]int 48 | 49 | // PodEvictor is used for evicting pods 50 | type PodEvictor struct { 51 | client clientset.Interface 52 | policyGroupVersion string 53 | dryRun bool 54 | maxPodsToEvict int 55 | nodepodCount nodePodEvictedCount 56 | freezeDuration time.Duration 57 | record record.EventRecorder 58 | base evictions.PodEvictor 59 | nodeNum int 60 | *util.UnschedulableCache 61 | CheckUnschedulablePods bool 62 | sync.RWMutex 63 | } 64 | 65 | // NewPodEvictor init a new evictor 66 | func NewPodEvictor( 67 | client clientset.Interface, 68 | policyGroupVersion string, 69 | maxPodsToEvict int, 70 | nodes []*v1.Node, unschedulableCache *util.UnschedulableCache) *PodEvictor { 71 | var nodePodCount = make(nodePodEvictedCount) 72 | for _, node := range nodes { 73 | // Initialize podsEvicted till now with 0. 74 | nodePodCount[node] = 0 75 | } 76 | 77 | eventBroadcaster := record.NewBroadcaster() 78 | eventBroadcaster.StartLogging(klog.V(3).Infof) 79 | eventBroadcaster.StartRecordingToSink(&clientcorev1.EventSinkImpl{Interface: client.CoreV1().Events(v1.NamespaceAll)}) 80 | r := eventBroadcaster.NewRecorder(scheme.Scheme, v1.EventSource{Component: "sigs.k8s.io.descheduler"}) 81 | 82 | virtualCount := 0 83 | for node := range nodePodCount { 84 | if util.IsVirtualNode(node) && !node.Spec.Unschedulable { 85 | virtualCount++ 86 | } 87 | } 88 | 89 | return &PodEvictor{ 90 | client: client, 91 | policyGroupVersion: policyGroupVersion, 92 | dryRun: false, 93 | maxPodsToEvict: maxPodsToEvict, 94 | nodepodCount: nodePodCount, 95 | nodeNum: virtualCount, 96 | freezeDuration: 5 * time.Minute, 97 | record: r, 98 | UnschedulableCache: unschedulableCache, 99 | } 100 | } 101 | 102 | // NodeEvicted gives a number of pods evicted for node 103 | func (pe *PodEvictor) NodeEvicted(node *v1.Node) int { 104 | return pe.nodepodCount[node] 105 | } 106 | 107 | // TotalEvicted gives a number of pods evicted through all nodes 108 | func (pe *PodEvictor) TotalEvicted() int { 109 | var total int 110 | for _, count := range pe.nodepodCount { 111 | total += count 112 | } 113 | return total 114 | } 115 | 116 | // EvictPod returns non-nil error only when evicting a pod on a node is not 117 | // possible (due to maxPodsToEvict constraint). Success is true when the pod 118 | // is evicted on the server side. 119 | func (pe *PodEvictor) EvictPod(ctx context.Context, pod *v1.Pod, node *v1.Node) (bool, error) { 120 | pe.RLock() 121 | if pe.maxPodsToEvict > 0 && pe.nodepodCount[node]+1 > pe.maxPodsToEvict { 122 | pe.RUnlock() 123 | return false, fmt.Errorf("Maximum number %v of evicted pods per %q node reached", pe.maxPodsToEvict, node.Name) 124 | } 125 | pe.RUnlock() 126 | 127 | nodeName := pod.Spec.NodeName 128 | podCopy := pod.DeepCopy() 129 | podCopy.UID = "" 130 | podCopy.ResourceVersion = "" 131 | podCopy.Spec.NodeName = "" 132 | if podCopy.Labels == nil { 133 | podCopy.Labels = map[string]string{} 134 | } 135 | podCopy.Labels[util.CreatedbyDescheduler] = "true" 136 | podCopy.Status = v1.PodStatus{} 137 | 138 | ownerID := pod.Name 139 | ownerCount := len(pod.OwnerReferences) 140 | if ownerCount != 0 { 141 | ownerID = string(pod.OwnerReferences[ownerCount-1].UID) 142 | } 143 | pe.Add(nodeName, ownerID) 144 | ti := pe.GetFreezeTime(nodeName, ownerID) 145 | klog.V(4).Info(ti) 146 | affinity, _ := util.ReplacePodNodeNameNodeAffinity(pod.Spec.Affinity, 147 | ownerID, pe.freezeDuration, pe.isNodeFreeze, nodeName) 148 | 149 | podCopy.Spec.Affinity = affinity 150 | klog.Infof("New pod affinity %+v", podCopy.Spec.Affinity) 151 | propagationPolicy := metav1.DeletePropagationBackground 152 | deleteOptions := &metav1.DeleteOptions{ 153 | GracePeriodSeconds: new(int64), 154 | PropagationPolicy: &propagationPolicy, 155 | } 156 | copy := pod.DeepCopy() 157 | addUnschedulablenode(copy) 158 | patch, err := util.CreateMergePatch(pod, copy) 159 | if err != nil { 160 | return false, err 161 | } 162 | _, err = pe.client.CoreV1().Pods(pod.Namespace).Patch(ctx, copy.Name, 163 | mergetypes.MergePatchType, patch, metav1.PatchOptions{}) 164 | if err != nil && !apierrors.IsNotFound(err) { 165 | klog.Errorf("Error evicting pod: %#v in namespace %#v (%#v)", pod.Name, pod.Namespace, err) 166 | return false, err 167 | } 168 | 169 | err = pe.client.CoreV1().Pods(podCopy.Namespace).Delete(ctx, podCopy.Name, *deleteOptions) 170 | if err != nil && !apierrors.IsNotFound(err) { 171 | // err is used only for logging purposes 172 | klog.Errorf("Error evicting pod: %#v in namespace %#v (%#v)", pod.Name, pod.Namespace, err) 173 | return false, err 174 | 175 | } 176 | addDescheduleCount(podCopy) 177 | 178 | _, err = pe.client.CoreV1().Pods(podCopy.Namespace).Create(ctx, podCopy, metav1.CreateOptions{}) 179 | klog.V(4).Infof("New pod %+v", podCopy) 180 | 181 | if err != nil && !apierrors.IsAlreadyExists(err) { 182 | // err is used only for logging purposes 183 | klog.Errorf("Error re-create pod: %#v in namespace %#v (%#v)", pod.Name, pod.Namespace, err) 184 | return false, nil 185 | } 186 | pe.record.Event(pod, v1.EventTypeNormal, "Rescheduled", "pod re-create by sigs.k8s.io/descheduler") 187 | klog.Infof("Re-create pod: %#v in namespace %#v success", pod.Name, pod.Namespace) 188 | pe.Lock() 189 | pe.nodepodCount[node]++ 190 | pe.Unlock() 191 | if pe.dryRun { 192 | klog.V(1).Infof("Evicted pod in dry run mode: %#v in namespace %#v", pod.Name, pod.Namespace) 193 | } else { 194 | klog.V(1).Infof("Evicted pod: %#v in namespace %#v", pod.Name, pod.Namespace) 195 | } 196 | return true, nil 197 | } 198 | 199 | // replacePodNodeNameNodeAffinity replaces the RequiredDuringSchedulingIgnoredDuringExecution 200 | // NodeAffinity of the given affinity with a new NodeAffinity that selects the given nodeName. 201 | // Note that this function assumes that no NodeAffinity conflicts with the selected nodeName. 202 | func (pe *PodEvictor) replacePodNodeNameNodeAffinity(affinity *v1.Affinity, nodeName, ownerID string) (*v1.Affinity, 203 | int) { 204 | nodeSelReq := v1.NodeSelectorRequirement{ 205 | Key: "kubernetes.io/hostname", 206 | Operator: v1.NodeSelectorOpNotIn, 207 | Values: []string{nodeName}, 208 | } 209 | 210 | nodeSelector := &v1.NodeSelector{ 211 | NodeSelectorTerms: []v1.NodeSelectorTerm{ 212 | { 213 | MatchExpressions: []v1.NodeSelectorRequirement{nodeSelReq}, 214 | }, 215 | }, 216 | } 217 | 218 | count := 1 219 | if affinity == nil { 220 | return &v1.Affinity{ 221 | NodeAffinity: &v1.NodeAffinity{ 222 | RequiredDuringSchedulingIgnoredDuringExecution: nodeSelector, 223 | }, 224 | }, count 225 | } 226 | 227 | if affinity.NodeAffinity == nil { 228 | affinity.NodeAffinity = &v1.NodeAffinity{ 229 | RequiredDuringSchedulingIgnoredDuringExecution: nodeSelector, 230 | } 231 | return affinity, count 232 | } 233 | 234 | nodeAffinity := affinity.NodeAffinity 235 | if nodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution == nil { 236 | nodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution = nodeSelector 237 | return affinity, count 238 | } 239 | 240 | terms := nodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution.NodeSelectorTerms 241 | if terms == nil { 242 | nodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution.NodeSelectorTerms = []v1.NodeSelectorTerm{ 243 | { 244 | MatchFields: []v1.NodeSelectorRequirement{nodeSelReq}, 245 | }, 246 | } 247 | return affinity, count 248 | } 249 | 250 | newTerms := make([]v1.NodeSelectorTerm, 0) 251 | for _, term := range terms { 252 | if term.MatchExpressions == nil { 253 | continue 254 | } 255 | mes, noScheduleCount := pe.getNodeSelectorRequirement(term, nodeName, ownerID, nodeSelReq) 256 | count = noScheduleCount 257 | term.MatchExpressions = mes 258 | newTerms = append(newTerms, term) 259 | } 260 | 261 | // Replace node selector with the new one. 262 | nodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution.NodeSelectorTerms = newTerms 263 | affinity.NodeAffinity = nodeAffinity 264 | return affinity, count 265 | } 266 | 267 | func (pe *PodEvictor) getNodeSelectorRequirement(term v1.NodeSelectorTerm, 268 | nodeName, ownerID string, nodeSelReq v1.NodeSelectorRequirement) ([]v1.NodeSelectorRequirement, int) { 269 | mes := make([]v1.NodeSelectorRequirement, 0) 270 | count := 0 271 | for _, me := range term.MatchExpressions { 272 | if me.Key != nodeSelReq.Key || me.Operator != nodeSelReq.Operator { 273 | mes = append(mes, me) 274 | continue 275 | } 276 | values := make([]string, 0) 277 | for _, v := range me.Values { 278 | klog.V(4).Infof("current term value %v", v) 279 | if v == nodeName { 280 | continue 281 | } 282 | if pe.isNodeFreeze(v, ownerID, pe.freezeDuration) { 283 | values = append(values, v) 284 | } 285 | } 286 | if nodeName != "" { 287 | me.Values = append(values, nodeSelReq.Values...) 288 | } 289 | count = len(values) 290 | mes = append(mes, me) 291 | continue 292 | } 293 | return mes, count 294 | } 295 | 296 | func (pe *PodEvictor) isNodeFreeze(node, ownerID string, 297 | freezeDuration time.Duration) bool { 298 | freezeTime := pe.GetFreezeTime(node, ownerID) 299 | klog.V(4).Infof("OwnerID %v, node %v, time %v", ownerID, node, freezeTime) 300 | if freezeTime == nil { 301 | return false 302 | } 303 | if freezeTime.Add(freezeDuration).After(time.Now()) { 304 | return true 305 | } 306 | return false 307 | } 308 | 309 | func evictPod(ctx context.Context, client clientset.Interface, pod *v1.Pod, policyGroupVersion string, dryRun bool) error { 310 | if dryRun { 311 | return nil 312 | } 313 | 314 | var gracePeriodSeconds int64 = 0 315 | propagationPolicy := metav1.DeletePropagationForeground 316 | deleteOptions := &metav1.DeleteOptions{ 317 | GracePeriodSeconds: &gracePeriodSeconds, 318 | PropagationPolicy: &propagationPolicy, 319 | } 320 | // GracePeriodSeconds ? 321 | eviction := &policy.Eviction{ 322 | TypeMeta: metav1.TypeMeta{ 323 | APIVersion: policyGroupVersion, 324 | Kind: eutils.EvictionKind, 325 | }, 326 | ObjectMeta: metav1.ObjectMeta{ 327 | Name: pod.Name, 328 | Namespace: pod.Namespace, 329 | }, 330 | DeleteOptions: deleteOptions, 331 | } 332 | err := client.PolicyV1beta1().Evictions(eviction.Namespace).Evict(ctx, eviction) 333 | 334 | if err == nil { 335 | return nil 336 | } 337 | if apierrors.IsTooManyRequests(err) { 338 | return fmt.Errorf("error when evicting pod (ignoring) %q: %v", pod.Name, err) 339 | } 340 | if apierrors.IsNotFound(err) { 341 | return fmt.Errorf("pod not found when evicting %q: %v", pod.Name, err) 342 | } 343 | return err 344 | } 345 | 346 | func addDescheduleCount(pod *v1.Pod) { 347 | if pod == nil { 348 | return 349 | } 350 | if pod.Annotations == nil { 351 | pod.Annotations = map[string]string{util.DescheduleCount: strconv.Itoa(1)} 352 | return 353 | } 354 | countStr, ok := pod.Annotations[util.DescheduleCount] 355 | if !ok { 356 | pod.Annotations[util.DescheduleCount] = strconv.Itoa(1) 357 | return 358 | } 359 | count, err := strconv.Atoi(countStr) 360 | if err != nil { 361 | count = 0 362 | } 363 | pod.Annotations = map[string]string{util.DescheduleCount: strconv.Itoa(count + 1)} 364 | } 365 | 366 | func addUnschedulablenode(pod *v1.Pod) { 367 | if pod == nil { 368 | return 369 | } 370 | if pod.Annotations == nil { 371 | pod.Annotations = map[string]string{} 372 | } 373 | if len(pod.Spec.NodeName) > 0 { 374 | pod.Annotations[unschedulableNode] = pod.Spec.NodeName 375 | } 376 | pod.ResourceVersion = "0" 377 | } 378 | -------------------------------------------------------------------------------- /pkg/descheduler/evictions/evictions_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright ©2020. The virtual-kubelet authors 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package evictions 18 | 19 | import ( 20 | "context" 21 | v1 "k8s.io/api/core/v1" 22 | "k8s.io/apimachinery/pkg/runtime" 23 | "k8s.io/client-go/kubernetes/fake" 24 | core "k8s.io/client-go/testing" 25 | "sigs.k8s.io/descheduler/test" 26 | "testing" 27 | ) 28 | 29 | func TestEvictPod(t *testing.T) { 30 | ctx := context.Background() 31 | node1 := test.BuildTestNode("node1", 1000, 2000, 9, nil) 32 | pod1 := test.BuildTestPod("p1", 400, 0, "node1", nil) 33 | tests := []struct { 34 | description string 35 | node *v1.Node 36 | pod *v1.Pod 37 | pods []v1.Pod 38 | want error 39 | }{ 40 | { 41 | description: "test pod eviction - pod present", 42 | node: node1, 43 | pod: pod1, 44 | pods: []v1.Pod{*pod1}, 45 | want: nil, 46 | }, 47 | { 48 | description: "test pod eviction - pod absent", 49 | node: node1, 50 | pod: pod1, 51 | pods: []v1.Pod{*test.BuildTestPod("p2", 400, 0, "node1", nil), *test.BuildTestPod("p3", 450, 0, "node1", nil)}, 52 | want: nil, 53 | }, 54 | } 55 | 56 | for _, test := range tests { 57 | fakeClient := &fake.Clientset{} 58 | fakeClient.Fake.AddReactor("list", "pods", func(action core.Action) (bool, runtime.Object, error) { 59 | return true, &v1.PodList{Items: test.pods}, nil 60 | }) 61 | got := evictPod(ctx, fakeClient, test.pod, "v1", false) 62 | if got != test.want { 63 | t.Errorf("Test error for Desc: %s. Expected %v pod eviction to be %v, got %v", test.description, test.pod.Name, test.want, got) 64 | } 65 | } 66 | } 67 | -------------------------------------------------------------------------------- /pkg/descheduler/pod/pods.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright ©2020. The virtual-kubelet authors 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package pod 18 | 19 | import ( 20 | "context" 21 | v1 "k8s.io/api/core/v1" 22 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 23 | "k8s.io/apimachinery/pkg/fields" 24 | clientset "k8s.io/client-go/kubernetes" 25 | base "sigs.k8s.io/descheduler/pkg/descheduler/pod" 26 | ) 27 | 28 | // IsEvictable checks if a pod is evictable or not. 29 | func IsEvictable(pod *v1.Pod, evictLocalStoragePods bool) bool { 30 | if !base.IsEvictable(pod, evictLocalStoragePods) { 31 | return false 32 | } 33 | if pod.Status.Phase != v1.PodPending { 34 | return false 35 | } 36 | return true 37 | } 38 | 39 | // ListEvictablePodsOnNode returns the list of evictable pods on node. 40 | func ListEvictablePodsOnNode(client clientset.Interface, node *v1.Node, evictLocalStoragePods bool) ([]*v1.Pod, error) { 41 | pods, err := ListPodsOnANode(client, node) 42 | if err != nil { 43 | return []*v1.Pod{}, err 44 | } 45 | evictablePods := make([]*v1.Pod, 0) 46 | for _, pod := range pods { 47 | if !IsEvictable(pod, evictLocalStoragePods) { 48 | continue 49 | } else { 50 | evictablePods = append(evictablePods, pod) 51 | } 52 | } 53 | return evictablePods, nil 54 | } 55 | 56 | // ListPodsOnANode lists pod on some node 57 | func ListPodsOnANode(client clientset.Interface, node *v1.Node) ([]*v1.Pod, error) { 58 | fieldSelector, err := fields.ParseSelector("spec.nodeName=" + node.Name + ",status.phase=" + string(v1.PodPending)) 59 | if err != nil { 60 | return []*v1.Pod{}, err 61 | } 62 | 63 | podList, err := client.CoreV1().Pods(v1.NamespaceAll).List(context.TODO(), 64 | metav1.ListOptions{FieldSelector: fieldSelector.String()}) 65 | if err != nil { 66 | return []*v1.Pod{}, err 67 | } 68 | 69 | pods := make([]*v1.Pod, 0) 70 | for i := range podList.Items { 71 | pods = append(pods, &podList.Items[i]) 72 | } 73 | return pods, nil 74 | } 75 | -------------------------------------------------------------------------------- /pkg/descheduler/pod/pods_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright ©2020. The virtual-kubelet authors 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package pod 18 | 19 | import ( 20 | "testing" 21 | 22 | v1 "k8s.io/api/core/v1" 23 | "k8s.io/apimachinery/pkg/api/resource" 24 | "sigs.k8s.io/descheduler/pkg/utils" 25 | "sigs.k8s.io/descheduler/test" 26 | ) 27 | 28 | func TestIsEvictable(t *testing.T) { 29 | n1 := test.BuildTestNode("node1", 1000, 2000, 13, nil) 30 | type testCase struct { 31 | pod *v1.Pod 32 | runBefore func(*v1.Pod) 33 | evictLocalStoragePods bool 34 | result bool 35 | } 36 | 37 | testCases := []testCase{ 38 | { 39 | pod: test.BuildTestPod("p1", 400, 0, n1.Name, nil), 40 | runBefore: func(pod *v1.Pod) { 41 | pod.ObjectMeta.OwnerReferences = test.GetNormalPodOwnerRefList() 42 | }, 43 | evictLocalStoragePods: false, 44 | result: true, 45 | }, { 46 | pod: test.BuildTestPod("p2", 400, 0, n1.Name, nil), 47 | runBefore: func(pod *v1.Pod) { 48 | pod.Annotations = map[string]string{"descheduler.alpha.kubernetes.io/evict": "true"} 49 | pod.ObjectMeta.OwnerReferences = test.GetNormalPodOwnerRefList() 50 | }, 51 | evictLocalStoragePods: false, 52 | result: true, 53 | }, { 54 | pod: test.BuildTestPod("p3", 400, 0, n1.Name, nil), 55 | runBefore: func(pod *v1.Pod) { 56 | pod.ObjectMeta.OwnerReferences = test.GetReplicaSetOwnerRefList() 57 | }, 58 | evictLocalStoragePods: false, 59 | result: true, 60 | }, { 61 | pod: test.BuildTestPod("p4", 400, 0, n1.Name, nil), 62 | runBefore: func(pod *v1.Pod) { 63 | pod.Annotations = map[string]string{"descheduler.alpha.kubernetes.io/evict": "true"} 64 | pod.ObjectMeta.OwnerReferences = test.GetReplicaSetOwnerRefList() 65 | }, 66 | evictLocalStoragePods: false, 67 | result: true, 68 | }, { 69 | pod: test.BuildTestPod("p5", 400, 0, n1.Name, nil), 70 | runBefore: func(pod *v1.Pod) { 71 | pod.ObjectMeta.OwnerReferences = test.GetNormalPodOwnerRefList() 72 | pod.Spec.Volumes = []v1.Volume{ 73 | { 74 | Name: "sample", 75 | VolumeSource: v1.VolumeSource{ 76 | HostPath: &v1.HostPathVolumeSource{Path: "somePath"}, 77 | EmptyDir: &v1.EmptyDirVolumeSource{ 78 | SizeLimit: resource.NewQuantity(int64(10), resource.BinarySI)}, 79 | }, 80 | }, 81 | } 82 | }, 83 | evictLocalStoragePods: false, 84 | result: false, 85 | }, { 86 | pod: test.BuildTestPod("p6", 400, 0, n1.Name, nil), 87 | runBefore: func(pod *v1.Pod) { 88 | pod.ObjectMeta.OwnerReferences = test.GetNormalPodOwnerRefList() 89 | pod.Spec.Volumes = []v1.Volume{ 90 | { 91 | Name: "sample", 92 | VolumeSource: v1.VolumeSource{ 93 | HostPath: &v1.HostPathVolumeSource{Path: "somePath"}, 94 | EmptyDir: &v1.EmptyDirVolumeSource{ 95 | SizeLimit: resource.NewQuantity(int64(10), resource.BinarySI)}, 96 | }, 97 | }, 98 | } 99 | }, 100 | evictLocalStoragePods: true, 101 | result: true, 102 | }, { 103 | pod: test.BuildTestPod("p7", 400, 0, n1.Name, nil), 104 | runBefore: func(pod *v1.Pod) { 105 | pod.Annotations = map[string]string{"descheduler.alpha.kubernetes.io/evict": "true"} 106 | pod.ObjectMeta.OwnerReferences = test.GetNormalPodOwnerRefList() 107 | pod.Spec.Volumes = []v1.Volume{ 108 | { 109 | Name: "sample", 110 | VolumeSource: v1.VolumeSource{ 111 | HostPath: &v1.HostPathVolumeSource{Path: "somePath"}, 112 | EmptyDir: &v1.EmptyDirVolumeSource{ 113 | SizeLimit: resource.NewQuantity(int64(10), resource.BinarySI)}, 114 | }, 115 | }, 116 | } 117 | }, 118 | evictLocalStoragePods: false, 119 | result: true, 120 | }, { 121 | pod: test.BuildTestPod("p8", 400, 0, n1.Name, nil), 122 | runBefore: func(pod *v1.Pod) { 123 | pod.ObjectMeta.OwnerReferences = test.GetDaemonSetOwnerRefList() 124 | }, 125 | evictLocalStoragePods: false, 126 | result: false, 127 | }, { 128 | pod: test.BuildTestPod("p9", 400, 0, n1.Name, nil), 129 | runBefore: func(pod *v1.Pod) { 130 | pod.Annotations = map[string]string{"descheduler.alpha.kubernetes.io/evict": "true"} 131 | pod.ObjectMeta.OwnerReferences = test.GetDaemonSetOwnerRefList() 132 | }, 133 | evictLocalStoragePods: false, 134 | result: true, 135 | }, { 136 | pod: test.BuildTestPod("p10", 400, 0, n1.Name, nil), 137 | runBefore: func(pod *v1.Pod) { 138 | pod.Annotations = test.GetMirrorPodAnnotation() 139 | }, 140 | evictLocalStoragePods: false, 141 | result: false, 142 | }, { 143 | pod: test.BuildTestPod("p11", 400, 0, n1.Name, nil), 144 | runBefore: func(pod *v1.Pod) { 145 | pod.Annotations = test.GetMirrorPodAnnotation() 146 | pod.Annotations["descheduler.alpha.kubernetes.io/evict"] = "true" 147 | }, 148 | evictLocalStoragePods: false, 149 | result: true, 150 | }, { 151 | pod: test.BuildTestPod("p12", 400, 0, n1.Name, nil), 152 | runBefore: func(pod *v1.Pod) { 153 | priority := utils.SystemCriticalPriority 154 | pod.Spec.Priority = &priority 155 | }, 156 | evictLocalStoragePods: false, 157 | result: false, 158 | }, { 159 | pod: test.BuildTestPod("p13", 400, 0, n1.Name, nil), 160 | runBefore: func(pod *v1.Pod) { 161 | priority := utils.SystemCriticalPriority 162 | pod.Spec.Priority = &priority 163 | pod.Annotations = map[string]string{ 164 | "descheduler.alpha.kubernetes.io/evict": "true", 165 | } 166 | }, 167 | evictLocalStoragePods: false, 168 | result: true, 169 | }, 170 | } 171 | 172 | for _, test := range testCases { 173 | test.pod.Status.Phase = v1.PodPending 174 | test.runBefore(test.pod) 175 | result := IsEvictable(test.pod, test.evictLocalStoragePods) 176 | if result != test.result { 177 | t.Errorf("IsEvictable should return for pod %s %t, but it returns %t", test.pod.Name, test.result, result) 178 | } 179 | 180 | } 181 | } 182 | -------------------------------------------------------------------------------- /pkg/descheduler/strategies/pod_lifetime.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright ©2020. The virtual-kubelet authors 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package strategies 18 | 19 | import ( 20 | "context" 21 | "sync" 22 | 23 | v1 "k8s.io/api/core/v1" 24 | v1meta "k8s.io/apimachinery/pkg/apis/meta/v1" 25 | clientset "k8s.io/client-go/kubernetes" 26 | "k8s.io/client-go/util/workqueue" 27 | "k8s.io/klog" 28 | "sigs.k8s.io/descheduler/pkg/api" 29 | 30 | "github.com/virtual-kubelet/tensile-kube/pkg/descheduler/evictions" 31 | podutil "github.com/virtual-kubelet/tensile-kube/pkg/descheduler/pod" 32 | ) 33 | 34 | // PodLifeTime evicts pods on nodes that were created more than strategy.Params.MaxPodLifeTimeSeconds seconds ago. 35 | func PodLifeTime(ctx context.Context, client clientset.Interface, strategy api.DeschedulerStrategy, nodes []*v1.Node, evictLocalStoragePods bool, podEvictor *evictions.PodEvictor) { 36 | if strategy.Params.MaxPodLifeTimeSeconds == nil { 37 | klog.V(1).Infof("MaxPodLifeTimeSeconds not set") 38 | return 39 | } 40 | var wg sync.WaitGroup 41 | for _, node := range nodes { 42 | go func(node *v1.Node) { 43 | wg.Add(1) 44 | defer wg.Done() 45 | klog.V(1).Infof("Processing node: %#v", node.Name) 46 | pods := listOldPodsOnNode(client, node, *strategy.Params.MaxPodLifeTimeSeconds, evictLocalStoragePods) 47 | 48 | f := func(idx int) { 49 | success, err := podEvictor.EvictPod(ctx, pods[idx], node) 50 | if success { 51 | klog.V(1).Infof("Evicted pod: %#v because it was created more than %v seconds ago", pods[idx].Name, *strategy.Params.MaxPodLifeTimeSeconds) 52 | } 53 | 54 | if err != nil { 55 | klog.Errorf("Error evicting pod: (%#v)", err) 56 | return 57 | } 58 | } 59 | 60 | workqueue.ParallelizeUntil(context.TODO(), 64, len(pods), f) 61 | }(node) 62 | } 63 | wg.Wait() 64 | if podEvictor.CheckUnschedulablePods { 65 | klog.V(1).Info("Processing unschedulabe pods") 66 | pods := listOldPodsOnNode(client, &v1.Node{}, (*strategy.Params.MaxPodLifeTimeSeconds)*3, 67 | evictLocalStoragePods) 68 | f := func(idx int) { 69 | success, err := podEvictor.EvictPod(ctx, pods[idx], &v1.Node{}) 70 | if success { 71 | klog.V(1).Infof("Evicted pod: %#v because it was created more than %v seconds ago", pods[idx].Name, *strategy.Params.MaxPodLifeTimeSeconds) 72 | } 73 | 74 | if err != nil { 75 | klog.Errorf("Error evicting pod: (%#v)", err) 76 | return 77 | } 78 | } 79 | 80 | workqueue.ParallelizeUntil(context.TODO(), 64, len(pods), f) 81 | } 82 | } 83 | 84 | func listOldPodsOnNode(client clientset.Interface, node *v1.Node, maxAge uint, evictLocalStoragePods bool) []*v1.Pod { 85 | pods, err := podutil.ListEvictablePodsOnNode(client, node, evictLocalStoragePods) 86 | if err != nil { 87 | return nil 88 | } 89 | 90 | var oldPods []*v1.Pod 91 | for _, pod := range pods { 92 | podAgeSeconds := uint(v1meta.Now().Sub(pod.GetCreationTimestamp().Local()).Seconds()) 93 | if podAgeSeconds > maxAge { 94 | oldPods = append(oldPods, pod) 95 | continue 96 | } 97 | for _, condition := range pod.Status.Conditions { 98 | if condition.Type == v1.PodScheduled && (condition.Reason == "Unschedulable" || condition.Reason == "SchedulerError") { 99 | oldPods = append(oldPods, pod) 100 | break 101 | } 102 | } 103 | } 104 | 105 | return oldPods 106 | } 107 | -------------------------------------------------------------------------------- /pkg/provider/helper.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright ©2020. The virtual-kubelet authors 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package provider 18 | 19 | import ( 20 | "strings" 21 | 22 | corev1 "k8s.io/api/core/v1" 23 | "k8s.io/klog" 24 | ) 25 | 26 | // getSecrets filters the volumes of a pod to get only the secret volumes, 27 | // excluding the serviceaccount token secret which is automatically added by kuberentes. 28 | func getSecrets(pod *corev1.Pod) []string { 29 | secretNames := []string{} 30 | for _, v := range pod.Spec.Volumes { 31 | switch { 32 | case v.Secret != nil: 33 | if strings.HasPrefix(v.Name, "default-token") { 34 | continue 35 | } 36 | klog.Infof("pod %s depends on secret %s", pod.Name, v.Secret.SecretName) 37 | secretNames = append(secretNames, v.Secret.SecretName) 38 | 39 | case v.CephFS != nil: 40 | klog.Infof("pod %s depends on secret %s", pod.Name, v.CephFS.SecretRef.Name) 41 | secretNames = append(secretNames, v.CephFS.SecretRef.Name) 42 | case v.Cinder != nil: 43 | klog.Infof("pod %s depends on secret %s", pod.Name, v.Cinder.SecretRef.Name) 44 | secretNames = append(secretNames, v.Cinder.SecretRef.Name) 45 | case v.RBD != nil: 46 | klog.Infof("pod %s depends on secret %s", pod.Name, v.RBD.SecretRef.Name) 47 | secretNames = append(secretNames, v.RBD.SecretRef.Name) 48 | default: 49 | klog.Warning("Skip other type volumes") 50 | } 51 | } 52 | if pod.Spec.ImagePullSecrets != nil { 53 | for _, s := range pod.Spec.ImagePullSecrets { 54 | secretNames = append(secretNames, s.Name) 55 | } 56 | } 57 | klog.Infof("pod %s depends on secrets %s", pod.Name, secretNames) 58 | return secretNames 59 | } 60 | 61 | // getConfigmaps filters the volumes of a pod to get only the configmap volumes, 62 | func getConfigmaps(pod *corev1.Pod) []string { 63 | cmNames := []string{} 64 | for _, v := range pod.Spec.Volumes { 65 | if v.ConfigMap == nil { 66 | continue 67 | } 68 | cmNames = append(cmNames, v.ConfigMap.Name) 69 | } 70 | klog.Infof("pod %s depends on configMap %s", pod.Name, cmNames) 71 | return cmNames 72 | } 73 | 74 | // getPVCs filters the volumes of a pod to get only the pvc, 75 | func getPVCs(pod *corev1.Pod) []string { 76 | cmNames := []string{} 77 | for _, v := range pod.Spec.Volumes { 78 | if v.PersistentVolumeClaim == nil { 79 | continue 80 | } 81 | cmNames = append(cmNames, v.PersistentVolumeClaim.ClaimName) 82 | } 83 | klog.Infof("pod %s depends on pvc %v", pod.Name, cmNames) 84 | return cmNames 85 | } 86 | 87 | func checkNodeStatusReady(node *corev1.Node) bool { 88 | for _, condition := range node.Status.Conditions { 89 | if condition.Type != corev1.NodeReady { 90 | continue 91 | } 92 | if condition.Status == corev1.ConditionTrue { 93 | return true 94 | } 95 | } 96 | return false 97 | } 98 | 99 | func compareNodeStatusReady(old, new *corev1.Node) (bool, bool) { 100 | return checkNodeStatusReady(old), checkNodeStatusReady(new) 101 | } 102 | 103 | func podStopped(pod *corev1.Pod) bool { 104 | return (pod.Status.Phase == corev1.PodSucceeded || pod.Status.Phase == corev1.PodFailed) && pod.Spec. 105 | RestartPolicy == corev1.RestartPolicyNever 106 | } 107 | 108 | // nodeCustomLabel adds an additional node label. 109 | // The label can be any customised meaningful label specified from user. 110 | func nodeCustomLabel(node *corev1.Node, label string) { 111 | nodelabel := strings.Split(label, ":") 112 | if len(nodelabel) == 2 { 113 | node.ObjectMeta.Labels[strings.TrimSpace(nodelabel[0])] = strings.TrimSpace(nodelabel[1]) 114 | } 115 | } 116 | -------------------------------------------------------------------------------- /pkg/provider/helper_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright ©2020. The virtual-kubelet authors 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package provider 18 | 19 | import ( 20 | "reflect" 21 | "testing" 22 | 23 | v1 "k8s.io/api/core/v1" 24 | 25 | "github.com/virtual-kubelet/tensile-kube/pkg/testbase" 26 | ) 27 | 28 | func TestGetSecret(t *testing.T) { 29 | pod := testbase.PodForTestWithSecret() 30 | secrets := []string{"test1", "testbase"} 31 | real := getSecrets(pod) 32 | if !reflect.DeepEqual(secrets, real) { 33 | t.Fatalf("desire %v real %v", secrets, real) 34 | } 35 | } 36 | 37 | func TestGetConfimap(t *testing.T) { 38 | pod := testbase.PodForTestWithConfigmap() 39 | secrets := []string{"testbase"} 40 | real := getConfigmaps(pod) 41 | if !reflect.DeepEqual(secrets, real) { 42 | t.Fatalf("desire %v real %v", secrets, real) 43 | } 44 | } 45 | 46 | func TestGetPVC(t *testing.T) { 47 | pod := testbase.PodForTestWithPVC() 48 | desire := []string{"testbase-pvc", "testbase-pvc1"} 49 | real := getPVCs(pod) 50 | if !reflect.DeepEqual(desire, real) { 51 | t.Fatalf("desire %v real %v", desire, real) 52 | } 53 | } 54 | 55 | func TestCheckNodeStatusReady(t *testing.T) { 56 | node := testbase.NodeForTest() 57 | node1 := node.DeepCopy() 58 | node1.Status.Conditions = []v1.NodeCondition{ 59 | { 60 | Type: v1.NodeReady, 61 | Status: v1.ConditionTrue, 62 | }, 63 | } 64 | node2 := node.DeepCopy() 65 | node2.Status.Conditions = []v1.NodeCondition{ 66 | { 67 | Type: v1.NodeDiskPressure, 68 | Status: v1.ConditionTrue, 69 | }, 70 | } 71 | 72 | cases := []struct { 73 | node *v1.Node 74 | ready bool 75 | }{ 76 | { 77 | node: node, 78 | ready: false, 79 | }, 80 | { 81 | node: node1, 82 | ready: true, 83 | }, 84 | { 85 | node: node2, 86 | ready: false, 87 | }, 88 | } 89 | 90 | for i, c := range cases { 91 | if c.ready != checkNodeStatusReady(c.node) { 92 | t.Fatalf("case %v unexpected", i) 93 | } 94 | } 95 | } 96 | 97 | func TestPodStopped(t *testing.T) { 98 | pod := testbase.PodForTest() 99 | pod1 := pod.DeepCopy() 100 | pod1.Status.Phase = v1.PodSucceeded 101 | pod2 := pod.DeepCopy() 102 | pod2.Status.Phase = v1.PodFailed 103 | pod3 := pod.DeepCopy() 104 | pod3.Status.Phase = v1.PodFailed 105 | pod3.Spec.RestartPolicy = v1.RestartPolicyNever 106 | pod4 := pod.DeepCopy() 107 | pod4.Status.Phase = v1.PodSucceeded 108 | pod4.Spec.RestartPolicy = v1.RestartPolicyNever 109 | cases := []struct { 110 | pod *v1.Pod 111 | stopped bool 112 | }{ 113 | { 114 | pod: pod, 115 | stopped: false, 116 | }, 117 | { 118 | pod: pod1, 119 | stopped: false, 120 | }, 121 | { 122 | pod: pod2, 123 | stopped: false, 124 | }, 125 | { 126 | pod: pod3, 127 | stopped: true, 128 | }, 129 | { 130 | pod: pod4, 131 | stopped: true, 132 | }, 133 | } 134 | for _, c := range cases { 135 | if c.stopped != podStopped(c.pod) { 136 | t.Errorf("Desire %v, but not", c.stopped) 137 | } 138 | } 139 | } 140 | -------------------------------------------------------------------------------- /pkg/provider/metrics.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright ©2020. The virtual-kubelet authors 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package provider 18 | 19 | import ( 20 | "context" 21 | "time" 22 | 23 | corev1 "k8s.io/api/core/v1" 24 | "k8s.io/apimachinery/pkg/api/resource" 25 | v1 "k8s.io/apimachinery/pkg/apis/meta/v1" 26 | "k8s.io/apimachinery/pkg/labels" 27 | stats "k8s.io/kubernetes/pkg/kubelet/apis/stats/v1alpha1" 28 | "k8s.io/metrics/pkg/apis/metrics/v1beta1" 29 | 30 | "github.com/virtual-kubelet/tensile-kube/pkg/util" 31 | ) 32 | 33 | // GetStatsSummary summaries the cluster metrics which represented by the provider 34 | func (v *VirtualK8S) GetStatsSummary(ctx context.Context) (*stats.Summary, error) { 35 | var summary stats.Summary 36 | selector := labels.SelectorFromSet(map[string]string{ 37 | util.VirtualPodLabel: "true"}, 38 | ) 39 | metrics, err := v.metricClient.MetricsV1beta1().PodMetricses(corev1.NamespaceAll).List(ctx, 40 | v1.ListOptions{ 41 | LabelSelector: selector.String(), 42 | }) 43 | if err != nil { 44 | return nil, err 45 | } 46 | var cpuAll, memoryAll uint64 47 | var time time.Time 48 | for _, metric := range metrics.Items { 49 | podStats := convert2PodStats(&metric) 50 | summary.Pods = append(summary.Pods, *podStats) 51 | cpuAll += *podStats.CPU.UsageNanoCores 52 | memoryAll += *podStats.Memory.WorkingSetBytes 53 | if time.IsZero() { 54 | time = podStats.StartTime.Time 55 | } 56 | } 57 | summary.Node = stats.NodeStats{ 58 | NodeName: v.providerNode.Name, 59 | StartTime: v1.Time{Time: time}, 60 | CPU: &stats.CPUStats{ 61 | Time: v1.Time{Time: time}, 62 | UsageNanoCores: &cpuAll, 63 | }, 64 | Memory: &stats.MemoryStats{ 65 | Time: v1.Time{Time: time}, 66 | WorkingSetBytes: &memoryAll, 67 | }, 68 | } 69 | return &summary, nil 70 | } 71 | 72 | func convert2PodStats(metric *v1beta1.PodMetrics) *stats.PodStats { 73 | stat := &stats.PodStats{} 74 | if metric == nil { 75 | return nil 76 | } 77 | stat.PodRef.Namespace = metric.Namespace 78 | stat.PodRef.Name = metric.Name 79 | stat.StartTime = metric.Timestamp 80 | 81 | containerStats := stats.ContainerStats{} 82 | var cpuAll, memoryAll uint64 83 | for _, c := range metric.Containers { 84 | containerStats.StartTime = metric.Timestamp 85 | containerStats.Name = c.Name 86 | nanoCore := uint64(c.Usage.Cpu().ScaledValue(resource.Nano)) 87 | memory := uint64(c.Usage.Memory().Value()) 88 | containerStats.CPU = &stats.CPUStats{ 89 | Time: metric.Timestamp, 90 | UsageNanoCores: &nanoCore, 91 | } 92 | containerStats.Memory = &stats.MemoryStats{ 93 | Time: metric.Timestamp, 94 | WorkingSetBytes: &memory, 95 | } 96 | cpuAll += nanoCore 97 | memoryAll += memory 98 | stat.Containers = append(stat.Containers, containerStats) 99 | } 100 | stat.CPU = &stats.CPUStats{ 101 | Time: metric.Timestamp, 102 | UsageNanoCores: &cpuAll, 103 | } 104 | stat.Memory = &stats.MemoryStats{ 105 | Time: metric.Timestamp, 106 | WorkingSetBytes: &memoryAll, 107 | } 108 | return stat 109 | } 110 | -------------------------------------------------------------------------------- /pkg/provider/node.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright ©2020. The virtual-kubelet authors 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package provider 18 | 19 | import ( 20 | "context" 21 | "fmt" 22 | "os" 23 | 24 | corev1 "k8s.io/api/core/v1" 25 | "k8s.io/apimachinery/pkg/api/resource" 26 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 27 | "k8s.io/apimachinery/pkg/fields" 28 | "k8s.io/apimachinery/pkg/labels" 29 | "k8s.io/klog" 30 | 31 | "github.com/virtual-kubelet/tensile-kube/pkg/common" 32 | "github.com/virtual-kubelet/tensile-kube/pkg/util" 33 | ) 34 | 35 | // ConfigureNode enables a provider to configure the node object that 36 | // will be used for Kubernetes. 37 | func (v *VirtualK8S) ConfigureNode(ctx context.Context, node *corev1.Node) { 38 | nodes, err := v.clientCache.nodeLister.List(labels.Everything()) 39 | if err != nil { 40 | return 41 | } 42 | 43 | nodeResource := common.NewResource() 44 | 45 | for _, n := range nodes { 46 | if n.Spec.Unschedulable { 47 | continue 48 | } 49 | if !checkNodeStatusReady(n) { 50 | klog.Infof("Node %v not ready", node.Name) 51 | continue 52 | } 53 | nc := common.ConvertResource(n.Status.Capacity) 54 | nodeResource.Add(nc) 55 | } 56 | podResource := v.getResourceFromPods() 57 | nodeResource.Sub(podResource) 58 | nodeResource.SetCapacityToNode(node) 59 | node.Status.NodeInfo.KubeletVersion = v.version 60 | node.Status.NodeInfo.OperatingSystem = "linux" 61 | node.Status.NodeInfo.Architecture = "amd64" 62 | node.ObjectMeta.Labels[corev1.LabelArchStable] = "amd64" 63 | node.ObjectMeta.Labels[corev1.LabelOSStable] = "linux" 64 | node.ObjectMeta.Labels[util.LabelOSBeta] = "linux" 65 | if label := os.Getenv("VKUBELET_NODE_LABEL"); label != "" { 66 | nodeCustomLabel(node, label) 67 | } 68 | node.Status.Addresses = []corev1.NodeAddress{{Type: corev1.NodeInternalIP, Address: os.Getenv("VKUBELET_POD_IP")}} 69 | if externalIP := os.Getenv("VKUBELET_EXTERNAL_POD_IP"); externalIP != "" { 70 | node.Status.Addresses = append(node.Status.Addresses, corev1.NodeAddress{Type: corev1.NodeExternalIP, Address: externalIP}) 71 | } 72 | node.Status.Conditions = nodeConditions() 73 | node.Status.DaemonEndpoints = v.nodeDaemonEndpoints() 74 | v.providerNode.Node = node 75 | v.configured = true 76 | return 77 | } 78 | 79 | // Ping tries to connect to client cluster 80 | // implement node.NodeProvider 81 | func (v *VirtualK8S) Ping(ctx context.Context) error { 82 | // If node or master ping fail, we should it as a failed ping 83 | _, err := v.master.Discovery().ServerVersion() 84 | if err != nil { 85 | klog.Error("Failed ping") 86 | return fmt.Errorf("could not list master apiserver statuses: %v", err) 87 | } 88 | _, err = v.client.Discovery().ServerVersion() 89 | if err != nil { 90 | klog.Error("Failed ping") 91 | return fmt.Errorf("could not list client apiserver statuses: %v", err) 92 | } 93 | return nil 94 | } 95 | 96 | // NotifyNodeStatus is used to asynchronously monitor the node. 97 | // The passed in callback should be called any time there is a change to the 98 | // node's status. 99 | // This will generally trigger a call to the Kubernetes API server to update 100 | // the status. 101 | // 102 | // NotifyNodeStatus should not block callers. 103 | func (v *VirtualK8S) NotifyNodeStatus(ctx context.Context, f func(*corev1.Node)) { 104 | klog.Info("Called NotifyNodeStatus") 105 | go func() { 106 | for { 107 | select { 108 | case node := <-v.updatedNode: 109 | klog.Infof("Enqueue updated node %v", node.Name) 110 | f(node) 111 | case <-v.stopCh: 112 | return 113 | case <-ctx.Done(): 114 | return 115 | } 116 | } 117 | }() 118 | } 119 | 120 | // nodeDaemonEndpoints returns NodeDaemonEndpoints for the node status 121 | // within Kubernetes. 122 | func (v *VirtualK8S) nodeDaemonEndpoints() corev1.NodeDaemonEndpoints { 123 | return corev1.NodeDaemonEndpoints{ 124 | KubeletEndpoint: corev1.DaemonEndpoint{ 125 | Port: v.daemonPort, 126 | }, 127 | } 128 | } 129 | 130 | // getResourceFromPods summary the resource already used by pods. 131 | func (v *VirtualK8S) getResourceFromPods() *common.Resource { 132 | podResource := common.NewResource() 133 | pods, err := v.clientCache.podLister.List(labels.Everything()) 134 | if err != nil { 135 | return podResource 136 | } 137 | for _, pod := range pods { 138 | if pod.Status.Phase == corev1.PodPending && pod.Spec.NodeName != "" || 139 | pod.Status.Phase == corev1.PodRunning { 140 | nodeName := pod.Spec.NodeName 141 | node, err := v.clientCache.nodeLister.Get(nodeName) 142 | if err != nil { 143 | klog.Infof("get node %v failed err: %v", nodeName, err) 144 | continue 145 | } 146 | if node.Spec.Unschedulable || !checkNodeStatusReady(node) { 147 | continue 148 | } 149 | res := util.GetRequestFromPod(pod) 150 | res.Pods = resource.MustParse("1") 151 | podResource.Add(res) 152 | } 153 | } 154 | return podResource 155 | } 156 | 157 | // getResourceFromPodsByNodeName summary the resource already used by pods according to nodeName 158 | func (v *VirtualK8S) getResourceFromPodsByNodeName(nodeName string) *common.Resource { 159 | podResource := common.NewResource() 160 | fieldSelector, err := fields.ParseSelector("spec.nodeName=" + nodeName) 161 | pods, err := v.client.CoreV1().Pods(corev1.NamespaceAll).List(context.TODO(), 162 | metav1.ListOptions{ 163 | FieldSelector: fieldSelector.String(), 164 | }) 165 | if err != nil { 166 | return podResource 167 | } 168 | for _, pod := range pods.Items { 169 | if util.IsVirtualPod(&pod) { 170 | continue 171 | } 172 | if pod.Status.Phase == corev1.PodPending || 173 | pod.Status.Phase == corev1.PodRunning { 174 | res := util.GetRequestFromPod(&pod) 175 | res.Pods = resource.MustParse("1") 176 | podResource.Add(res) 177 | } 178 | } 179 | return podResource 180 | } 181 | 182 | // nodeConditions creates a slice of node conditions representing a 183 | // kubelet in perfect health. These four conditions are the ones which virtual-kubelet 184 | // sets as Unknown when a Ping fails. 185 | func nodeConditions() []corev1.NodeCondition { 186 | return []corev1.NodeCondition{ 187 | { 188 | Type: "Ready", 189 | Status: corev1.ConditionTrue, 190 | LastHeartbeatTime: metav1.Now(), 191 | LastTransitionTime: metav1.Now(), 192 | Reason: "KubeletReady", 193 | Message: "kubelet is posting ready status", 194 | }, 195 | { 196 | Type: "MemoryPressure", 197 | Status: corev1.ConditionFalse, 198 | LastHeartbeatTime: metav1.Now(), 199 | LastTransitionTime: metav1.Now(), 200 | Reason: "KubeletHasSufficientMemory", 201 | Message: "kubelet has sufficient memory available", 202 | }, 203 | { 204 | Type: "DiskPressure", 205 | Status: corev1.ConditionFalse, 206 | LastHeartbeatTime: metav1.Now(), 207 | LastTransitionTime: metav1.Now(), 208 | Reason: "KubeletHasNoDiskPressure", 209 | Message: "kubelet has no disk pressure", 210 | }, 211 | { 212 | Type: "PIDPressure", 213 | Status: corev1.ConditionFalse, 214 | LastHeartbeatTime: metav1.Now(), 215 | LastTransitionTime: metav1.Now(), 216 | Reason: "KubeletHasSufficientPID", 217 | Message: "kubelet has sufficient PID available", 218 | }, 219 | } 220 | } 221 | -------------------------------------------------------------------------------- /pkg/provider/node_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright ©2020. The virtual-kubelet authors 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package provider 18 | 19 | import ( 20 | "context" 21 | "testing" 22 | 23 | corev1 "k8s.io/api/core/v1" 24 | "k8s.io/apimachinery/pkg/api/resource" 25 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 26 | "k8s.io/client-go/informers" 27 | v1 "k8s.io/client-go/informers/core/v1" 28 | "k8s.io/client-go/kubernetes/fake" 29 | "k8s.io/kubernetes/pkg/controller" 30 | 31 | "github.com/virtual-kubelet/tensile-kube/pkg/common" 32 | ) 33 | 34 | func TestConfigureNode(t *testing.T) { 35 | ctx := context.Background() 36 | res := corev1.ResourceList{ 37 | corev1.ResourceCPU: resource.MustParse("10"), 38 | } 39 | res1 := corev1.ResourceList{ 40 | corev1.ResourceCPU: resource.MustParse("20"), 41 | } 42 | res2 := corev1.ResourceList{ 43 | corev1.ResourceCPU: resource.MustParse("17"), 44 | } 45 | nodeNotReady := &corev1.Node{ 46 | ObjectMeta: metav1.ObjectMeta{Name: "node1"}, 47 | Status: corev1.NodeStatus{ 48 | Capacity: res, 49 | Allocatable: res, 50 | }, 51 | } 52 | node1 := nodeNotReady.DeepCopy() 53 | node1.Status.Conditions = []corev1.NodeCondition{ 54 | { 55 | Type: corev1.NodeReady, 56 | Status: corev1.ConditionTrue, 57 | }, 58 | } 59 | node2 := node1.DeepCopy() 60 | node2.Name = "node2" 61 | for _, c := range []struct { 62 | name string 63 | nodes []*corev1.Node 64 | pod *corev1.Pod 65 | desiredVirtualNode *corev1.Node 66 | }{ 67 | { 68 | name: "one node, notReady", 69 | nodes: []*corev1.Node{nodeNotReady}, 70 | desiredVirtualNode: &corev1.Node{ 71 | ObjectMeta: metav1.ObjectMeta{Name: "fake"}, 72 | Status: corev1.NodeStatus{}, 73 | }, 74 | }, 75 | { 76 | name: "one node, ready", 77 | nodes: []*corev1.Node{node1}, 78 | desiredVirtualNode: &corev1.Node{ 79 | ObjectMeta: metav1.ObjectMeta{Name: "fake"}, 80 | Status: corev1.NodeStatus{ 81 | Capacity: res, 82 | Allocatable: res, 83 | }, 84 | }, 85 | }, 86 | { 87 | name: "two nodes", 88 | nodes: []*corev1.Node{node1, node2}, 89 | desiredVirtualNode: &corev1.Node{ 90 | ObjectMeta: metav1.ObjectMeta{Name: "fake"}, 91 | Status: corev1.NodeStatus{ 92 | Capacity: res1, 93 | Allocatable: res1, 94 | }, 95 | }, 96 | }, 97 | { 98 | name: "nodes with pod running", 99 | nodes: []*corev1.Node{node1, node2}, 100 | pod: fakeNodeWithReq(), 101 | desiredVirtualNode: &corev1.Node{ 102 | ObjectMeta: metav1.ObjectMeta{Name: "fake"}, 103 | Status: corev1.NodeStatus{ 104 | Capacity: res2, 105 | Allocatable: res2, 106 | }, 107 | }, 108 | }, 109 | } { 110 | t.Run(c.name, func(t *testing.T) { 111 | node := corev1.Node{} 112 | node.Labels = map[string]string{} 113 | vk, nodeInformer, podInformer := newFakeVirtualK8SWithNodePod() 114 | for _, node := range c.nodes { 115 | nodeInformer.Informer().GetStore().Add(node) 116 | } 117 | if c.pod != nil { 118 | podInformer.Informer().GetStore().Add(c.pod) 119 | } 120 | vk.ConfigureNode(ctx, &node) 121 | if node.Status.Allocatable.Cpu().Sign() != 122 | c.desiredVirtualNode.Status.Allocatable.Cpu().Sign() { 123 | t.Errorf("Desired: %v, get: %v", 124 | c.desiredVirtualNode.Status.Allocatable, node.Status.Allocatable) 125 | } 126 | }) 127 | } 128 | } 129 | 130 | func newFakeVirtualK8SWithNodePod() (*VirtualK8S, v1.NodeInformer, v1.PodInformer) { 131 | client := fake.NewSimpleClientset() 132 | master := fake.NewSimpleClientset() 133 | 134 | clientInformer := informers.NewSharedInformerFactory(client, controller.NoResyncPeriodFunc()) 135 | 136 | nodeInformer := clientInformer.Core().V1().Nodes() 137 | podInformer := clientInformer.Core().V1().Pods() 138 | return &VirtualK8S{ 139 | master: master, 140 | client: client, 141 | clientCache: clientCache{ 142 | nodeLister: nodeInformer.Lister(), 143 | podLister: podInformer.Lister(), 144 | }, 145 | providerNode: &common.ProviderNode{ 146 | Node: &corev1.Node{}, 147 | }, 148 | }, nodeInformer, podInformer 149 | } 150 | 151 | func fakeNodeWithReq() *corev1.Pod { 152 | pod := fakePod("ns") 153 | res := corev1.ResourceList{ 154 | corev1.ResourceCPU: resource.MustParse("3"), 155 | } 156 | pod.Spec.NodeName = "node1" 157 | pod.Spec.Containers = []corev1.Container{ 158 | { 159 | Resources: corev1.ResourceRequirements{ 160 | Requests: res, 161 | }, 162 | }, 163 | } 164 | return pod 165 | } 166 | -------------------------------------------------------------------------------- /pkg/provider/pod_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright ©2020. The virtual-kubelet authors 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package provider 18 | 19 | import ( 20 | "context" 21 | "reflect" 22 | "testing" 23 | 24 | corev1 "k8s.io/api/core/v1" 25 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 26 | "k8s.io/client-go/informers" 27 | v1 "k8s.io/client-go/informers/core/v1" 28 | "k8s.io/client-go/kubernetes/fake" 29 | "k8s.io/kubernetes/pkg/controller" 30 | 31 | "github.com/virtual-kubelet/tensile-kube/pkg/util" 32 | ) 33 | 34 | func TestCreatePod(t *testing.T) { 35 | vk, _, podInformer := newFakeVirtualK8S() 36 | ctx := context.Background() 37 | for _, c := range []struct { 38 | name string 39 | pod *corev1.Pod 40 | error bool 41 | existPod *corev1.Pod 42 | }{ 43 | { 44 | name: "kube-system pod, do not create", 45 | pod: fakePod("kube-system"), 46 | error: false, 47 | }, 48 | { 49 | name: "ns not exist", 50 | pod: fakePod("test"), 51 | error: false, 52 | }, 53 | { 54 | name: "pod exists", 55 | pod: fakePod("test"), 56 | existPod: fakePod("test"), 57 | error: true, 58 | }, 59 | } { 60 | t.Run(c.name, func(t *testing.T) { 61 | if c.existPod != nil { 62 | podInformer.Informer().GetStore().Add(c.pod) 63 | } 64 | err := vk.CreatePod(ctx, c.pod) 65 | if (err != nil) != c.error { 66 | t.Errorf("Desire: %v, get: %v", c.error, err != nil) 67 | } 68 | }) 69 | } 70 | } 71 | 72 | func TestUpdatePod(t *testing.T) { 73 | vk, _, podInformer := newFakeVirtualK8S() 74 | ctx := context.Background() 75 | notVirtualPod := fakePod("test") 76 | updatedNotVirtualPod := notVirtualPod.DeepCopy() 77 | updatedNotVirtualPod.Annotations = map[string]string{"virtual-pod": "true"} 78 | 79 | virtualPod := notVirtualPod.DeepCopy() 80 | virtualPod.Labels = map[string]string{"virtual-pod": "true"} 81 | updatedVirtualPod := notVirtualPod.DeepCopy() 82 | updatedVirtualPod.Annotations = map[string]string{"virtual-pod": "true"} 83 | for _, c := range []struct { 84 | name string 85 | pod *corev1.Pod 86 | error bool 87 | existPod *corev1.Pod 88 | }{ 89 | { 90 | name: "kube-system pod, do not update", 91 | pod: fakePod("kube-system"), 92 | error: false, 93 | }, 94 | { 95 | name: "pod not exist", 96 | pod: fakePod("test"), 97 | error: true, 98 | }, 99 | { 100 | name: "not virtual pod", 101 | pod: updatedNotVirtualPod, 102 | existPod: notVirtualPod, 103 | error: false, 104 | }, 105 | { 106 | name: "pod exists", 107 | pod: updatedVirtualPod, 108 | existPod: virtualPod, 109 | error: false, 110 | }, 111 | } { 112 | t.Run(c.name, func(t *testing.T) { 113 | if c.existPod != nil { 114 | podInformer.Informer().GetStore().Add(c.pod) 115 | } 116 | err := vk.UpdatePod(ctx, c.pod) 117 | if (err != nil) != c.error { 118 | t.Errorf("Desire: %v, get: %v", c.error, err != nil) 119 | } 120 | if c.existPod == nil { 121 | return 122 | } 123 | if !util.IsVirtualPod(c.existPod) { 124 | return 125 | } 126 | pod, err := vk.GetPod(ctx, c.existPod.Namespace, c.existPod.Name) 127 | if err != nil { 128 | t.Error(err) 129 | } 130 | if !reflect.DeepEqual(pod, c.pod) { 131 | t.Error("update pod failed") 132 | } 133 | 134 | }) 135 | } 136 | } 137 | 138 | func TestDeletePod(t *testing.T) { 139 | vk, _, podInformer := newFakeVirtualK8S() 140 | ctx := context.Background() 141 | for _, c := range []struct { 142 | name string 143 | pod *corev1.Pod 144 | error bool 145 | existPod *corev1.Pod 146 | }{ 147 | { 148 | name: "kube-system pod, do not delete", 149 | pod: fakePod("kube-system"), 150 | error: false, 151 | }, 152 | { 153 | name: "not virtual pod, do not delete", 154 | pod: fakePod("kube-system"), 155 | error: false, 156 | }, 157 | { 158 | name: "pod not exist", 159 | pod: fakePod("test"), 160 | error: false, 161 | }, 162 | { 163 | name: "pod exists", 164 | pod: fakePod("test"), 165 | existPod: fakePod("test"), 166 | error: false, 167 | }, 168 | } { 169 | t.Run(c.name, func(t *testing.T) { 170 | if c.existPod != nil { 171 | podInformer.Informer().GetStore().Add(c.pod) 172 | } 173 | err := vk.DeletePod(ctx, c.pod) 174 | if (err != nil) != c.error { 175 | t.Errorf("Desire: %v, get: %v", c.error, err != nil) 176 | } 177 | }) 178 | } 179 | } 180 | 181 | func TestGetPod(t *testing.T) { 182 | vk, _, podInformer := newFakeVirtualK8S() 183 | ctx := context.Background() 184 | for _, c := range []struct { 185 | name string 186 | pod *corev1.Pod 187 | error bool 188 | existPod *corev1.Pod 189 | }{ 190 | { 191 | name: "pod not exist", 192 | pod: fakePod("test"), 193 | error: true, 194 | }, 195 | { 196 | name: "pod exists", 197 | pod: fakePod("test"), 198 | existPod: fakePod("test"), 199 | error: false, 200 | }, 201 | } { 202 | t.Run(c.name, func(t *testing.T) { 203 | if c.existPod != nil { 204 | podInformer.Informer().GetStore().Add(c.pod) 205 | } 206 | _, err := vk.GetPod(ctx, c.pod.Namespace, c.pod.Name) 207 | if (err != nil) != c.error { 208 | t.Errorf("Desire: %v, get: %v", c.error, err != nil) 209 | } 210 | }) 211 | } 212 | } 213 | 214 | func newFakeVirtualK8S() (*VirtualK8S, v1.NamespaceInformer, v1.PodInformer) { 215 | client := fake.NewSimpleClientset() 216 | master := fake.NewSimpleClientset() 217 | 218 | clientInformer := informers.NewSharedInformerFactory(client, controller.NoResyncPeriodFunc()) 219 | 220 | nsInformer := clientInformer.Core().V1().Namespaces() 221 | podInformer := clientInformer.Core().V1().Pods() 222 | return &VirtualK8S{ 223 | master: master, 224 | client: client, 225 | clientCache: clientCache{ 226 | nsLister: nsInformer.Lister(), 227 | podLister: podInformer.Lister(), 228 | }, 229 | }, nsInformer, podInformer 230 | } 231 | 232 | func fakePod(ns string) *corev1.Pod { 233 | pod := &corev1.Pod{ 234 | ObjectMeta: metav1.ObjectMeta{ 235 | Name: "test", 236 | }, 237 | Spec: corev1.PodSpec{}, 238 | Status: corev1.PodStatus{}, 239 | } 240 | if ns != "" { 241 | pod.Namespace = ns 242 | } 243 | return pod 244 | } 245 | -------------------------------------------------------------------------------- /pkg/testbase/test.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright ©2020. The virtual-kubelet authors 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package testbase 18 | 19 | import ( 20 | v1 "k8s.io/api/core/v1" 21 | "k8s.io/apimachinery/pkg/api/resource" 22 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 23 | ) 24 | 25 | const ( 26 | // TaintNodeNotReady is not ready taint 27 | TaintNodeNotReady = "node.kubernetes.io/not-ready" 28 | // TaintNodeUnreachable is unreachable taint 29 | TaintNodeUnreachable = "node.kubernetes.io/unreachable" 30 | ) 31 | 32 | // PodForTest return a basic pod for test 33 | func PodForTest() *v1.Pod { 34 | defaultMode := int32(420) 35 | return &v1.Pod{ 36 | ObjectMeta: metav1.ObjectMeta{ 37 | Name: "testbase", 38 | }, 39 | Spec: v1.PodSpec{ 40 | Containers: []v1.Container{ 41 | { 42 | Resources: v1.ResourceRequirements{ 43 | Limits: map[v1.ResourceName]resource.Quantity{ 44 | "cpu": resource.MustParse("10"), 45 | }, 46 | Requests: map[v1.ResourceName]resource.Quantity{ 47 | "cpu": resource.MustParse("10"), 48 | }, 49 | }, 50 | VolumeMounts: []v1.VolumeMount{ 51 | { 52 | MountPath: "/var/run/secrets/kubernetes.io/serviceaccount", 53 | ReadOnly: true, 54 | Name: "default-token-mvzcf", 55 | }, 56 | }, 57 | }, 58 | }, 59 | NodeName: "test", 60 | Volumes: []v1.Volume{ 61 | { 62 | Name: "default-token-mvzcf", 63 | VolumeSource: v1.VolumeSource{ 64 | Secret: &v1.SecretVolumeSource{ 65 | SecretName: "default-token-mvzcf", 66 | DefaultMode: &defaultMode, 67 | }, 68 | }, 69 | }, 70 | }, 71 | }, 72 | } 73 | } 74 | 75 | // PodForTestWithSystemTolerations return a basic pod for test with system Tolerations 76 | func PodForTestWithSystemTolerations() *v1.Pod { 77 | pod := PodForTest() 78 | var tolerationTime int64 = 10 79 | pod.Spec.Tolerations = []v1.Toleration{ 80 | { 81 | Key: TaintNodeNotReady, 82 | Operator: v1.TolerationOpExists, 83 | Effect: v1.TaintEffectNoExecute, 84 | TolerationSeconds: &tolerationTime, 85 | }, 86 | { 87 | Key: TaintNodeUnreachable, 88 | Operator: v1.TolerationOpExists, 89 | Effect: v1.TaintEffectNoExecute, 90 | TolerationSeconds: &tolerationTime, 91 | }, 92 | } 93 | return pod 94 | } 95 | 96 | // PodForTestWithOtherTolerations return a basic pod for test with other Tolerations 97 | func PodForTestWithOtherTolerations() *v1.Pod { 98 | pod := PodForTest() 99 | pod.Spec.Tolerations = []v1.Toleration{ 100 | { 101 | Key: "testbase", 102 | Operator: v1.TolerationOpExists, 103 | Effect: v1.TaintEffectNoExecute, 104 | }, 105 | } 106 | return pod 107 | } 108 | 109 | // PodForTestWithSecret return a basic pod for test with secret 110 | func PodForTestWithSecret() *v1.Pod { 111 | pod := PodForTest() 112 | pod.Spec.ImagePullSecrets = []v1.LocalObjectReference{{Name: "testbase"}} 113 | pod.Spec.Volumes = []v1.Volume{ 114 | { 115 | Name: "test1", 116 | VolumeSource: v1.VolumeSource{ 117 | Secret: &v1.SecretVolumeSource{ 118 | SecretName: "test1", 119 | }, 120 | }, 121 | }, 122 | } 123 | return pod 124 | } 125 | 126 | // PodForTestWithConfigmap return a basic pod for test with configmap 127 | func PodForTestWithConfigmap() *v1.Pod { 128 | pod := PodForTest() 129 | pod.Spec.Volumes = []v1.Volume{ 130 | { 131 | Name: "testbase", 132 | VolumeSource: v1.VolumeSource{ 133 | ConfigMap: &v1.ConfigMapVolumeSource{ 134 | LocalObjectReference: v1.LocalObjectReference{ 135 | Name: "testbase", 136 | }, 137 | }, 138 | }, 139 | }, 140 | } 141 | return pod 142 | } 143 | 144 | // PodForTestWithPVC return a basic pod for test with PVC 145 | func PodForTestWithPVC() *v1.Pod { 146 | pod := PodForTest() 147 | pod.Spec.Volumes = []v1.Volume{ 148 | { 149 | Name: "testbase", 150 | VolumeSource: v1.VolumeSource{ 151 | PersistentVolumeClaim: &v1.PersistentVolumeClaimVolumeSource{ 152 | ClaimName: "testbase-pvc", 153 | ReadOnly: false, 154 | }, 155 | }, 156 | }, 157 | { 158 | Name: "test1", 159 | VolumeSource: v1.VolumeSource{ 160 | PersistentVolumeClaim: &v1.PersistentVolumeClaimVolumeSource{ 161 | ClaimName: "testbase-pvc1", 162 | ReadOnly: false, 163 | }, 164 | }, 165 | }, 166 | } 167 | return pod 168 | } 169 | 170 | // PodForTestWithNodeSelector return a basic pod for test with node selector 171 | func PodForTestWithNodeSelector() *v1.Pod { 172 | pod := PodForTest() 173 | pod.Spec.NodeSelector = map[string]string{"testbase": "testbase"} 174 | return pod 175 | } 176 | 177 | // PodForTestWithNodeSelectorClusterID return a basic pod for test with NodeSelectorClusterID 178 | func PodForTestWithNodeSelectorClusterID() *v1.Pod { 179 | pod := PodForTest() 180 | pod.Spec.NodeSelector = map[string]string{"testbase": "testbase", "clusterID": "1"} 181 | return pod 182 | } 183 | 184 | // PodForTestWithNodeSelectorAndAffinityClusterID return a basic pod for test with NodeSelectorClusterID 185 | func PodForTestWithNodeSelectorAndAffinityClusterID() *v1.Pod { 186 | pod := PodForTest() 187 | pod.Spec.NodeSelector = map[string]string{"testbase": "testbase", "clusterID": "1"} 188 | pod.Spec.Affinity = &v1.Affinity{ 189 | NodeAffinity: &v1.NodeAffinity{ 190 | RequiredDuringSchedulingIgnoredDuringExecution: &v1.NodeSelector{NodeSelectorTerms: []v1. 191 | NodeSelectorTerm{{ 192 | MatchExpressions: []v1.NodeSelectorRequirement{ 193 | { 194 | Key: "test0", 195 | Operator: v1.NodeSelectorOpIn, 196 | Values: []string{ 197 | "aa", 198 | }, 199 | }, 200 | { 201 | Key: "clusterID", 202 | Operator: v1.NodeSelectorOpIn, 203 | Values: []string{ 204 | "aa", 205 | }, 206 | }, 207 | { 208 | Key: "test", 209 | Operator: v1.NodeSelectorOpIn, 210 | Values: []string{ 211 | "aa", 212 | }, 213 | }, 214 | { 215 | Key: "test1", 216 | Operator: v1.NodeSelectorOpIn, 217 | Values: []string{ 218 | "aa", 219 | }, 220 | }, 221 | }, 222 | }}}, 223 | PreferredDuringSchedulingIgnoredDuringExecution: nil, 224 | }, 225 | } 226 | return pod 227 | } 228 | 229 | // NodeForTest return a basic node 230 | func NodeForTest() *v1.Node { 231 | return &v1.Node{ 232 | TypeMeta: metav1.TypeMeta{}, 233 | ObjectMeta: metav1.ObjectMeta{ 234 | Name: "testbase", 235 | }, 236 | Status: v1.NodeStatus{ 237 | Capacity: map[v1.ResourceName]resource.Quantity{ 238 | "cpu": resource.MustParse("50"), 239 | "memory": resource.MustParse("50Gi"), 240 | }, 241 | Allocatable: map[v1.ResourceName]resource.Quantity{ 242 | "cpu": resource.MustParse("50"), 243 | "memory": resource.MustParse("50Gi"), 244 | }, 245 | }, 246 | } 247 | } 248 | 249 | // PodForTestWithAffinity return a basic pod for test with Affinity 250 | func PodForTestWithAffinity() *v1.Pod { 251 | pod := PodForTest() 252 | pod.Spec.NodeSelector = map[string]string{"testbase": "testbase"} 253 | pod.Spec.Affinity = &v1.Affinity{ 254 | NodeAffinity: &v1.NodeAffinity{ 255 | RequiredDuringSchedulingIgnoredDuringExecution: &v1.NodeSelector{NodeSelectorTerms: []v1. 256 | NodeSelectorTerm{{ 257 | MatchExpressions: []v1.NodeSelectorRequirement{ 258 | { 259 | Key: "testbase", 260 | Operator: v1.NodeSelectorOpIn, 261 | Values: []string{ 262 | "aa", 263 | }, 264 | }, 265 | }, 266 | }}}, 267 | PreferredDuringSchedulingIgnoredDuringExecution: nil, 268 | }, 269 | } 270 | return pod 271 | } 272 | -------------------------------------------------------------------------------- /pkg/util/cache.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright ©2020. The virtual-kubelet authors 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package util 18 | 19 | import ( 20 | "sync" 21 | "time" 22 | 23 | v1 "k8s.io/api/core/v1" 24 | "k8s.io/klog" 25 | 26 | gochache "github.com/patrickmn/go-cache" 27 | ) 28 | 29 | // UnschedulableCache contaiens cache ownerid/node/freezeTime 30 | type UnschedulableCache struct { 31 | cache map[string]*gochache.Cache 32 | sync.RWMutex 33 | } 34 | 35 | // NewUnschedulableCache init a new Unschedulable 36 | func NewUnschedulableCache() *UnschedulableCache { 37 | return &UnschedulableCache{cache: map[string]*gochache.Cache{}} 38 | } 39 | 40 | // Add add node/ownerID to cache 41 | func (c *UnschedulableCache) Add(node, ownerID string) { 42 | c.Lock() 43 | defer c.Unlock() 44 | now := time.Now() 45 | freezeCache := c.cache[ownerID] 46 | if freezeCache == nil { 47 | freezeCache = gochache.New(3*time.Minute, 6*time.Minute) 48 | } 49 | freezeCache.Add(node, &now, 0) 50 | c.cache[ownerID] = freezeCache 51 | } 52 | 53 | // GetFreezeNodes return the freezed nodes 54 | func (c *UnschedulableCache) GetFreezeNodes(ownerID string) []string { 55 | c.Lock() 56 | defer c.Unlock() 57 | freezeCache := c.cache[ownerID] 58 | if freezeCache == nil { 59 | return nil 60 | } 61 | nodes := make([]string, 0) 62 | for key := range freezeCache.Items() { 63 | nodes = append(nodes, key) 64 | } 65 | return nodes 66 | } 67 | 68 | // GetFreezeTime returns node/ownerID freeze time 69 | func (c *UnschedulableCache) GetFreezeTime(node, ownerID string) *time.Time { 70 | c.RLock() 71 | defer c.RUnlock() 72 | if c.cache[ownerID] == nil { 73 | return nil 74 | } 75 | timePtr, found := c.cache[ownerID].Get(node) 76 | if !found { 77 | return nil 78 | } 79 | 80 | return timePtr.(*time.Time) 81 | } 82 | 83 | // CheckValidFunc defines the check func 84 | type CheckValidFunc func(string, string, time.Duration) bool 85 | 86 | // ReplacePodNodeNameNodeAffinity replaces the RequiredDuringSchedulingIgnoredDuringExecution 87 | // NodeAffinity of the given affinity with a new NodeAffinity that selects the given nodeName. 88 | // Note that this function assumes that no NodeAffinity conflicts with the selected nodeName. 89 | func ReplacePodNodeNameNodeAffinity(affinity *v1.Affinity, ownerID string, expireTime time.Duration, 90 | checkFuc CheckValidFunc, nodeNames ...string) (*v1.Affinity, 91 | int) { 92 | nodeSelReq := v1.NodeSelectorRequirement{ 93 | // Key: "metadata.name", 94 | Key: "kubernetes.io/hostname", 95 | Operator: v1.NodeSelectorOpNotIn, 96 | Values: nodeNames, 97 | } 98 | 99 | nodeSelector := &v1.NodeSelector{ 100 | NodeSelectorTerms: []v1.NodeSelectorTerm{ 101 | { 102 | MatchExpressions: []v1.NodeSelectorRequirement{nodeSelReq}, 103 | }, 104 | }, 105 | } 106 | 107 | count := 1 108 | 109 | if affinity == nil { 110 | return &v1.Affinity{ 111 | NodeAffinity: &v1.NodeAffinity{ 112 | RequiredDuringSchedulingIgnoredDuringExecution: nodeSelector, 113 | }, 114 | }, count 115 | } 116 | 117 | if affinity.NodeAffinity == nil { 118 | affinity.NodeAffinity = &v1.NodeAffinity{ 119 | RequiredDuringSchedulingIgnoredDuringExecution: nodeSelector, 120 | } 121 | return affinity, count 122 | } 123 | 124 | nodeAffinity := affinity.NodeAffinity 125 | 126 | if nodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution == nil { 127 | nodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution = nodeSelector 128 | return affinity, count 129 | } 130 | 131 | terms := nodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution.NodeSelectorTerms 132 | if terms == nil { 133 | nodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution.NodeSelectorTerms = []v1.NodeSelectorTerm{ 134 | { 135 | MatchFields: []v1.NodeSelectorRequirement{nodeSelReq}, 136 | }, 137 | } 138 | return affinity, count 139 | } 140 | 141 | newTerms := make([]v1.NodeSelectorTerm, 0) 142 | for _, term := range terms { 143 | if term.MatchExpressions == nil { 144 | continue 145 | } 146 | mes, noScheduleCount := getNodeSelectorRequirement(term, ownerID, nodeSelReq, checkFuc, expireTime) 147 | count = noScheduleCount 148 | term.MatchExpressions = mes 149 | newTerms = append(newTerms, term) 150 | } 151 | 152 | // Replace node selector with the new one. 153 | nodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution.NodeSelectorTerms = newTerms 154 | affinity.NodeAffinity = nodeAffinity 155 | return affinity, count 156 | } 157 | 158 | func getNodeSelectorRequirement(term v1.NodeSelectorTerm, 159 | ownerID string, nodeSelReq v1.NodeSelectorRequirement, checkFuc CheckValidFunc, expireTime time.Duration) ([]v1.NodeSelectorRequirement, int) { 160 | mes := make([]v1.NodeSelectorRequirement, 0) 161 | count := 0 162 | for _, me := range term.MatchExpressions { 163 | if me.Key != nodeSelReq.Key || me.Operator != nodeSelReq.Operator { 164 | mes = append(mes, me) 165 | continue 166 | } 167 | values := make([]string, 0) 168 | for _, v := range me.Values { 169 | klog.V(4).Infof("current term value %v", v) 170 | if checkContains(nodeSelReq.Values, v) { 171 | continue 172 | } 173 | if checkFuc == nil { 174 | values = append(values, v) 175 | continue 176 | } 177 | if !checkFuc(v, ownerID, expireTime) && len(v) > 0 { 178 | values = append(values, v) 179 | } 180 | } 181 | me.Values = append(values, nodeSelReq.Values...) 182 | count = len(values) 183 | mes = append(mes, me) 184 | } 185 | return mes, count 186 | } 187 | 188 | func checkContains(exists []string, value string) bool { 189 | if len(exists) == 0 { 190 | return false 191 | } 192 | for _, v := range exists { 193 | if v == value { 194 | return true 195 | } 196 | } 197 | return false 198 | } 199 | -------------------------------------------------------------------------------- /pkg/util/cache_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright ©2020. The virtual-kubelet authors 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package util 18 | 19 | import ( 20 | "testing" 21 | "time" 22 | 23 | "github.com/patrickmn/go-cache" 24 | ) 25 | 26 | func TestCache(t *testing.T) { 27 | uc := UnschedulableCache{cache: map[string]*cache.Cache{}} 28 | uc.Add("test", "1") 29 | time.Sleep(5 * time.Second) 30 | uc.Add("test", "2") 31 | ft := uc.GetFreezeTime("test", "1") 32 | if ft == nil { 33 | t.Fatal("Unexpected results") 34 | } 35 | t.Log(ft) 36 | ft1 := uc.GetFreezeTime("test", "2") 37 | if ft1 == nil { 38 | t.Fatal("Unexpected results") 39 | } 40 | t.Log(ft1) 41 | } 42 | -------------------------------------------------------------------------------- /pkg/util/conversions.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright ©2020. The virtual-kubelet authors 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package util 18 | 19 | import ( 20 | "encoding/json" 21 | "strings" 22 | 23 | corev1 "k8s.io/api/core/v1" 24 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 25 | ) 26 | 27 | // TrimPod filter some fields that should not be contained when created in 28 | // subClusters for example: ownerReference, serviceLink and Uid 29 | // we should also add some fields back for scheduling. 30 | func TrimPod(pod *corev1.Pod, ignoreLabels []string) *corev1.Pod { 31 | vols := []corev1.Volume{} 32 | for _, v := range pod.Spec.Volumes { 33 | if strings.HasPrefix(v.Name, "default-token") { 34 | continue 35 | } 36 | vols = append(vols, v) 37 | } 38 | 39 | podCopy := pod.DeepCopy() 40 | TrimObjectMeta(&podCopy.ObjectMeta) 41 | if podCopy.Labels == nil { 42 | podCopy.Labels = make(map[string]string) 43 | } 44 | if podCopy.Annotations == nil { 45 | podCopy.Annotations = make(map[string]string) 46 | } 47 | podCopy.Labels[VirtualPodLabel] = "true" 48 | cns := ConvertAnnotations(pod.Annotations) 49 | recoverSelectors(podCopy, cns) 50 | podCopy.Spec.Containers = trimContainers(pod.Spec.Containers) 51 | podCopy.Spec.InitContainers = trimContainers(pod.Spec.InitContainers) 52 | podCopy.Spec.Volumes = vols 53 | podCopy.Spec.NodeName = "" 54 | podCopy.Status = corev1.PodStatus{} 55 | // remove labels should be removed, which would influence schedule in client cluster 56 | tripped := TrimLabels(podCopy.ObjectMeta.Labels, ignoreLabels) 57 | if tripped != nil { 58 | trippedStr, err := json.Marshal(tripped) 59 | if err != nil { 60 | return podCopy 61 | } 62 | podCopy.Annotations[TrippedLabels] = string(trippedStr) 63 | } 64 | 65 | return podCopy 66 | } 67 | 68 | // trimContainers remove 'default-token' crated automatically by k8s 69 | func trimContainers(containers []corev1.Container) []corev1.Container { 70 | var newContainers []corev1.Container 71 | 72 | for _, c := range containers { 73 | var volMounts []corev1.VolumeMount 74 | for _, v := range c.VolumeMounts { 75 | if strings.HasPrefix(v.Name, "default-token") { 76 | continue 77 | } 78 | volMounts = append(volMounts, v) 79 | } 80 | c.VolumeMounts = volMounts 81 | newContainers = append(newContainers, c) 82 | } 83 | 84 | return newContainers 85 | } 86 | 87 | // GetUpdatedPod allows user to update image, label, annotations 88 | // for tolerations, we can only add some more. 89 | func GetUpdatedPod(orig, update *corev1.Pod, ignoreLabels []string) { 90 | for i := range orig.Spec.InitContainers { 91 | orig.Spec.InitContainers[i].Image = update.Spec.InitContainers[i].Image 92 | } 93 | for i := range orig.Spec.Containers { 94 | orig.Spec.Containers[i].Image = update.Spec.Containers[i].Image 95 | } 96 | if update.Annotations == nil { 97 | update.Annotations = make(map[string]string) 98 | } 99 | if orig.Annotations[SelectorKey] != update.Annotations[SelectorKey] { 100 | if cns := ConvertAnnotations(update.Annotations); cns != nil { 101 | // we assume tolerations would only add not remove 102 | orig.Spec.Tolerations = cns.Tolerations 103 | } 104 | } 105 | orig.Labels = update.Labels 106 | orig.Annotations = update.Annotations 107 | orig.Spec.ActiveDeadlineSeconds = update.Spec.ActiveDeadlineSeconds 108 | if orig.Labels != nil { 109 | TrimLabels(orig.ObjectMeta.Labels, ignoreLabels) 110 | } 111 | return 112 | } 113 | 114 | // TrimObjectMeta removes some fields of ObjectMeta 115 | func TrimObjectMeta(meta *metav1.ObjectMeta) { 116 | meta.UID = "" 117 | meta.ResourceVersion = "" 118 | meta.SelfLink = "" 119 | meta.OwnerReferences = nil 120 | } 121 | 122 | // RecoverLabels recover some label that have been removed 123 | func RecoverLabels(labels map[string]string, annotations map[string]string) { 124 | trippedLabels := annotations[TrippedLabels] 125 | if trippedLabels == "" { 126 | return 127 | } 128 | trippedLabelsMap := make(map[string]string) 129 | if err := json.Unmarshal([]byte(trippedLabels), &trippedLabelsMap); err != nil { 130 | return 131 | } 132 | for k, v := range trippedLabelsMap { 133 | labels[k] = v 134 | } 135 | } 136 | 137 | // recoverSelectors recover some affinity, tolerations and nodeSelector from 138 | // ClusterSelector 139 | func recoverSelectors(pod *corev1.Pod, cns *ClustersNodeSelection) { 140 | if cns != nil { 141 | pod.Spec.NodeSelector = cns.NodeSelector 142 | pod.Spec.Tolerations = cns.Tolerations 143 | if pod.Spec.Affinity == nil { 144 | pod.Spec.Affinity = cns.Affinity 145 | } else { 146 | if cns.Affinity != nil && cns.Affinity.NodeAffinity != nil { 147 | if pod.Spec.Affinity.NodeAffinity != nil { 148 | pod.Spec.Affinity.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution = cns.Affinity.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution 149 | } else { 150 | pod.Spec.Affinity.NodeAffinity = cns.Affinity.NodeAffinity 151 | } 152 | } else { 153 | pod.Spec.Affinity.NodeAffinity = nil 154 | } 155 | } 156 | } else { 157 | pod.Spec.NodeSelector = nil 158 | pod.Spec.Tolerations = nil 159 | if pod.Spec.Affinity != nil && pod.Spec.Affinity.NodeAffinity != nil { 160 | pod.Spec.Affinity.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution = nil 161 | } 162 | } 163 | if pod.Spec.Affinity != nil { 164 | if pod.Spec.Affinity.NodeAffinity != nil { 165 | if pod.Spec.Affinity.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution == nil && 166 | pod.Spec.Affinity.NodeAffinity.PreferredDuringSchedulingIgnoredDuringExecution == nil { 167 | pod.Spec.Affinity.NodeAffinity = nil 168 | } 169 | } 170 | if pod.Spec.Affinity.NodeAffinity == nil && pod.Spec.Affinity.PodAffinity == nil && 171 | pod.Spec.Affinity.PodAntiAffinity == nil { 172 | pod.Spec.Affinity = nil 173 | } 174 | } 175 | } 176 | 177 | // TrimLabels removes label from labels according to ignoreLabels 178 | func TrimLabels(labels map[string]string, ignoreLabels []string) map[string]string { 179 | if ignoreLabels == nil { 180 | return nil 181 | } 182 | trippedLabels := make(map[string]string, len(ignoreLabels)) 183 | for _, key := range ignoreLabels { 184 | if labels[key] == "" { 185 | continue 186 | } 187 | trippedLabels[key] = labels[key] 188 | delete(labels, key) 189 | } 190 | return trippedLabels 191 | } 192 | 193 | // ConvertAnnotations converts annotations to ClustersNodeSelection 194 | func ConvertAnnotations(annotation map[string]string) *ClustersNodeSelection { 195 | if annotation == nil { 196 | return nil 197 | } 198 | val := annotation[SelectorKey] 199 | if len(val) == 0 { 200 | return nil 201 | } 202 | 203 | var cns ClustersNodeSelection 204 | err := json.Unmarshal([]byte(val), &cns) 205 | if err != nil { 206 | return nil 207 | } 208 | return &cns 209 | } 210 | -------------------------------------------------------------------------------- /pkg/util/conversions_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright ©2020. The virtual-kubelet authors 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package util 18 | 19 | import ( 20 | "reflect" 21 | "testing" 22 | 23 | corev1 "k8s.io/api/core/v1" 24 | "k8s.io/apimachinery/pkg/api/resource" 25 | v1 "k8s.io/apimachinery/pkg/apis/meta/v1" 26 | 27 | "github.com/virtual-kubelet/tensile-kube/pkg/testbase" 28 | ) 29 | 30 | func TestTrimObjectMeta(t *testing.T) { 31 | meta := v1.ObjectMeta{ 32 | UID: "test", 33 | ResourceVersion: "test", 34 | SelfLink: "http://test.com", 35 | OwnerReferences: []v1.OwnerReference{ 36 | { 37 | APIVersion: "Apps/v1", 38 | Kind: "StatefulSet", 39 | Name: "test", 40 | }, 41 | }, 42 | } 43 | TrimObjectMeta(&meta) 44 | if meta.UID != "" || meta.SelfLink != "" || meta.ResourceVersion != "" || meta.OwnerReferences != nil { 45 | t.Fatal("Unexpected") 46 | } 47 | } 48 | 49 | func TestTrimPod(t *testing.T) { 50 | 51 | desired := &corev1.Pod{ 52 | ObjectMeta: v1.ObjectMeta{ 53 | Name: "testbase", 54 | Labels: map[string]string{"virtual-pod": "true"}, 55 | }, 56 | Spec: corev1.PodSpec{ 57 | Containers: []corev1.Container{ 58 | { 59 | Resources: corev1.ResourceRequirements{ 60 | Limits: map[corev1.ResourceName]resource.Quantity{ 61 | "cpu": resource.MustParse("10"), 62 | }, 63 | Requests: map[corev1.ResourceName]resource.Quantity{ 64 | "cpu": resource.MustParse("10"), 65 | }, 66 | }, 67 | VolumeMounts: []corev1.VolumeMount{}, 68 | }, 69 | }, 70 | NodeName: "", 71 | Volumes: []corev1.Volume{}, 72 | }, 73 | Status: corev1.PodStatus{}, 74 | } 75 | 76 | desired1 := desired.DeepCopy() 77 | desired2 := desired.DeepCopy() 78 | 79 | desired1.Annotations = map[string]string{"clusterSelector": `{"nodeSelector":{"test": "test"}}`} 80 | desired1.Spec.NodeSelector = map[string]string{"test": "test"} 81 | desired2.Annotations = map[string]string{"tripped-labels": `{"test":"test"}`} 82 | 83 | basePod := testbase.PodForTest() 84 | basePod1 := basePod.DeepCopy() 85 | basePod2 := basePod.DeepCopy() 86 | 87 | basePod1.Annotations = map[string]string{"clusterSelector": `{"nodeSelector":{"test": "test"}}`} 88 | 89 | basePod2.Labels = map[string]string{"test": "test"} 90 | 91 | cases := []struct { 92 | name string 93 | pod *corev1.Pod 94 | desire *corev1.Pod 95 | trimLabel []string 96 | }{ 97 | { 98 | name: "base test", 99 | pod: basePod, 100 | desire: desired, 101 | trimLabel: nil, 102 | }, 103 | { 104 | name: "base test with annotation clusterSelector", 105 | pod: basePod1, 106 | desire: desired1, 107 | trimLabel: nil, 108 | }, 109 | { 110 | name: "base test with label", 111 | pod: basePod2, 112 | desire: desired2, 113 | trimLabel: []string{"test"}, 114 | }, 115 | { 116 | name: "base test toleration", 117 | pod: testbase.PodForTestWithOtherTolerations(), 118 | desire: desired, 119 | trimLabel: nil, 120 | }, 121 | { 122 | name: "base test node selector", 123 | pod: testbase.PodForTestWithNodeSelector(), 124 | desire: desired, 125 | trimLabel: nil, 126 | }, 127 | { 128 | name: "base affinity", 129 | pod: testbase.PodForTestWithAffinity(), 130 | desire: desired, 131 | trimLabel: nil, 132 | }, 133 | } 134 | for _, d := range cases { 135 | t.Log(d.name) 136 | new := TrimPod(d.pod, d.trimLabel) 137 | if new.String() != d.desire.String() { 138 | t.Fatalf("Desired:\n %v\n, get:\n %v", d.desire, new) 139 | } 140 | } 141 | } 142 | 143 | func TestRecoverLabels(t *testing.T) { 144 | annotations := map[string]string{"tripped-labels": `{"test":"test"}`} 145 | oldLabels := map[string]string{} 146 | labels := map[string]string{"test": "test"} 147 | RecoverLabels(annotations, oldLabels) 148 | if reflect.DeepEqual(oldLabels, labels) { 149 | t.Fatal("Unexpected") 150 | } 151 | t.Logf("%v %v", oldLabels, labels) 152 | 153 | } 154 | -------------------------------------------------------------------------------- /pkg/util/k8s.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright ©2020. The virtual-kubelet authors 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package util 18 | 19 | import ( 20 | "encoding/json" 21 | "fmt" 22 | "os" 23 | "os/signal" 24 | "syscall" 25 | 26 | jsonpatch "github.com/evanphx/json-patch" 27 | jsonpatch1 "github.com/mattbaird/jsonpatch" 28 | corev1 "k8s.io/api/core/v1" 29 | "k8s.io/client-go/kubernetes" 30 | "k8s.io/client-go/rest" 31 | "k8s.io/client-go/tools/clientcmd" 32 | "k8s.io/metrics/pkg/client/clientset/versioned" 33 | ) 34 | 35 | const ( 36 | // GlobalLabel make object global 37 | GlobalLabel = "global" 38 | // SelectorKey is the key of ClusterSelector 39 | SelectorKey = "clusterSelector" 40 | // SelectedNodeKey is the node selected by a scheduler 41 | SelectedNodeKey = "volume.kubernetes.io/selected-node" 42 | // HostNameKey is the label of HostNameKey 43 | HostNameKey = "kubernetes.io/hostname" 44 | // BetaHostNameKey is the label of HostNameKey 45 | BetaHostNameKey = "beta.kubernetes.io/hostname" 46 | // LabelOSBeta is the label of os 47 | LabelOSBeta = "beta.kubernetes.io/os" 48 | // VirtualPodLabel is the label of virtual pod 49 | VirtualPodLabel = "virtual-pod" 50 | // VirtualKubeletLabel is the label of virtual kubelet 51 | VirtualKubeletLabel = "virtual-kubelet" 52 | // TrippedLabels is the label of tripped labels 53 | TrippedLabels = "tripped-labels" 54 | // ClusterID marks the id of a cluster 55 | ClusterID = "clusterID" 56 | // NodeType is define the node type key 57 | NodeType = "type" 58 | // BatchPodLabel is the label of batch pod 59 | BatchPodLabel = "pod-group.scheduling.sigs.k8s.io" 60 | // TaintNodeNotReady will be added when node is not ready 61 | // and feature-gate for TaintBasedEvictions flag is enabled, 62 | // and removed when node becomes ready. 63 | TaintNodeNotReady = "node.kubernetes.io/not-ready" 64 | 65 | // TaintNodeUnreachable will be added when node becomes unreachable 66 | // (corresponding to NodeReady status ConditionUnknown) 67 | // and feature-gate for TaintBasedEvictions flag is enabled, 68 | // and removed when node becomes reachable (NodeReady status ConditionTrue). 69 | TaintNodeUnreachable = "node.kubernetes.io/unreachable" 70 | // CreatedbyDescheduler is used to mark if a pod is re-created by descheduler 71 | CreatedbyDescheduler = "create-by-descheduler" 72 | // DescheduleCount is used for recording deschedule count 73 | DescheduleCount = "sigs.k8s.io/deschedule-count" 74 | ) 75 | 76 | // ClustersNodeSelection is a struct including some scheduling parameters 77 | type ClustersNodeSelection struct { 78 | NodeSelector map[string]string `json:"nodeSelector,omitempty"` 79 | Affinity *corev1.Affinity `json:"affinity,omitempty"` 80 | Tolerations []corev1.Toleration `json:"tolerations,omitempty"` 81 | } 82 | 83 | // CreateMergePatch return patch generated from original and new interfaces 84 | func CreateMergePatch(original, new interface{}) ([]byte, error) { 85 | pvByte, err := json.Marshal(original) 86 | if err != nil { 87 | return nil, err 88 | } 89 | cloneByte, err := json.Marshal(new) 90 | if err != nil { 91 | return nil, err 92 | } 93 | patch, err := jsonpatch.CreateMergePatch(pvByte, cloneByte) 94 | if err != nil { 95 | return nil, err 96 | } 97 | return patch, nil 98 | } 99 | 100 | // CreateJSONPatch return patch generated from original and new interfaces 101 | func CreateJSONPatch(original, new interface{}) ([]byte, error) { 102 | pvByte, err := json.Marshal(original) 103 | if err != nil { 104 | return nil, err 105 | } 106 | cloneByte, err := json.Marshal(new) 107 | if err != nil { 108 | return nil, err 109 | } 110 | patchs, err := jsonpatch1.CreatePatch(pvByte, cloneByte) 111 | if err != nil { 112 | return nil, err 113 | } 114 | patchBytes, err := json.Marshal(patchs) 115 | if err != nil { 116 | return nil, err 117 | } 118 | return patchBytes, nil 119 | } 120 | 121 | var shutdownSignals = []os.Signal{os.Interrupt, syscall.SIGTERM} 122 | var onlyOneSignalHandler = make(chan struct{}) 123 | var shutdownHandler chan os.Signal 124 | 125 | // SetupSignalHandler registered for SIGTERM and SIGINT. A stop channel is returned 126 | // which is closed on one of these signals. If a second signal is caught, the program 127 | // is terminated with exit code 1. 128 | func SetupSignalHandler() <-chan struct{} { 129 | close(onlyOneSignalHandler) // panics when called twice 130 | shutdownHandler = make(chan os.Signal, 2) 131 | stop := make(chan struct{}) 132 | signal.Notify(shutdownHandler, shutdownSignals...) 133 | go func() { 134 | <-shutdownHandler 135 | close(stop) 136 | <-shutdownHandler 137 | os.Exit(1) // second signal. Exit directly. 138 | }() 139 | return stop 140 | } 141 | 142 | // Opts define the ops parameter functions 143 | type Opts func(*rest.Config) 144 | 145 | // NewClient returns a new client for k8s 146 | func NewClient(configPath string, opts ...Opts) (kubernetes.Interface, error) { 147 | // master config, maybe a real node or a pod 148 | var ( 149 | config *rest.Config 150 | err error 151 | ) 152 | config, err = clientcmd.BuildConfigFromFlags("", configPath) 153 | if err != nil { 154 | config, err = rest.InClusterConfig() 155 | if err != nil { 156 | return nil, fmt.Errorf("could not read config file for cluster: %v", err) 157 | } 158 | } 159 | 160 | for _, opt := range opts { 161 | if opt == nil { 162 | continue 163 | } 164 | opt(config) 165 | } 166 | 167 | client, err := kubernetes.NewForConfig(config) 168 | if err != nil { 169 | return nil, fmt.Errorf("could not create client for master cluster: %v", err) 170 | } 171 | return client, nil 172 | } 173 | 174 | // NewMetricClient returns a new client for k8s 175 | func NewMetricClient(configPath string, opts ...Opts) (versioned.Interface, error) { 176 | // master config, maybe a real node or a pod 177 | var ( 178 | config *rest.Config 179 | err error 180 | ) 181 | config, err = clientcmd.BuildConfigFromFlags("", configPath) 182 | if err != nil { 183 | config, err = rest.InClusterConfig() 184 | if err != nil { 185 | return nil, fmt.Errorf("could not read config file for cluster: %v", err) 186 | } 187 | } 188 | 189 | for _, opt := range opts { 190 | if opt == nil { 191 | continue 192 | } 193 | opt(config) 194 | } 195 | 196 | metricClient, err := versioned.NewForConfig(config) 197 | if err != nil { 198 | return nil, fmt.Errorf("could not create client for master cluster: %v", err) 199 | } 200 | return metricClient, nil 201 | } 202 | 203 | // IsVirtualNode defines if a node is virtual node 204 | func IsVirtualNode(node *corev1.Node) bool { 205 | if node == nil { 206 | return false 207 | } 208 | valStr, exist := node.ObjectMeta.Labels[NodeType] 209 | if !exist { 210 | return false 211 | } 212 | return valStr == VirtualKubeletLabel 213 | } 214 | 215 | // IsVirtualPod defines if a pod is virtual pod 216 | func IsVirtualPod(pod *corev1.Pod) bool { 217 | if pod.Labels != nil && pod.Labels[VirtualPodLabel] == "true" { 218 | return true 219 | } 220 | return false 221 | } 222 | 223 | // GetClusterID return the cluster in node label 224 | func GetClusterID(node *corev1.Node) string { 225 | if node == nil { 226 | return "" 227 | } 228 | clusterName, exist := node.ObjectMeta.Labels[ClusterID] 229 | if !exist { 230 | return "" 231 | } 232 | return clusterName 233 | } 234 | 235 | // UpdateConfigMap updates the configMap data 236 | func UpdateConfigMap(old, new *corev1.ConfigMap) { 237 | old.Labels = new.Labels 238 | old.Data = new.Data 239 | old.BinaryData = new.BinaryData 240 | } 241 | 242 | // UpdateSecret updates the secret data 243 | func UpdateSecret(old, new *corev1.Secret) { 244 | old.Labels = new.Labels 245 | old.Data = new.Data 246 | old.StringData = new.StringData 247 | old.Type = new.Type 248 | } 249 | -------------------------------------------------------------------------------- /pkg/util/k8s_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright ©2020. The virtual-kubelet authors 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package util 18 | 19 | import ( 20 | "os" 21 | "testing" 22 | 23 | v1 "k8s.io/api/core/v1" 24 | 25 | "github.com/virtual-kubelet/tensile-kube/pkg/testbase" 26 | ) 27 | 28 | func TestIsVirtualNode(t *testing.T) { 29 | node := testbase.NodeForTest() 30 | node1 := node.DeepCopy() 31 | node1.Labels = map[string]string{"type": "virtual-kubelet"} 32 | cases := []struct { 33 | name string 34 | node *v1.Node 35 | vn bool 36 | }{ 37 | { 38 | "not vk node", 39 | node, 40 | false, 41 | }, 42 | { 43 | "vk node", 44 | node1, 45 | true, 46 | }, 47 | } 48 | for _, c := range cases { 49 | if c.vn != IsVirtualNode(c.node) { 50 | t.Fatalf("case %v failed", c.name) 51 | } 52 | } 53 | } 54 | 55 | func TestIsVirtualPod(t *testing.T) { 56 | pod := testbase.PodForTest() 57 | pod1 := pod.DeepCopy() 58 | pod1.Labels = map[string]string{VirtualPodLabel: "true"} 59 | cases := []struct { 60 | name string 61 | pod *v1.Pod 62 | vn bool 63 | }{ 64 | { 65 | "not vk pod", 66 | pod, 67 | false, 68 | }, 69 | { 70 | "vk node", 71 | pod1, 72 | true, 73 | }, 74 | } 75 | for _, c := range cases { 76 | if c.vn != IsVirtualPod(c.pod) { 77 | t.Fatalf("case %v failed", c.name) 78 | } 79 | } 80 | } 81 | 82 | func TestGetClusterID(t *testing.T) { 83 | node := testbase.NodeForTest() 84 | node1 := node.DeepCopy() 85 | node1.Labels = map[string]string{"clusterID": "vk"} 86 | cases := []struct { 87 | name string 88 | node *v1.Node 89 | id string 90 | }{ 91 | { 92 | "cluster Id empty", 93 | node, 94 | "", 95 | }, 96 | { 97 | "cluster Id not empty", 98 | node1, 99 | "vk", 100 | }, 101 | } 102 | for _, c := range cases { 103 | if c.id != GetClusterID(c.node) { 104 | t.Fatalf("case %v failed", c.name) 105 | } 106 | } 107 | } 108 | 109 | func TestNewClient(t *testing.T) { 110 | path := "/tmp/test.config" 111 | f, err := os.Create(path) 112 | if err != nil && !os.IsExist(err) { 113 | t.Fatal(err) 114 | } 115 | //defer os.Remove(path) 116 | cases := []struct { 117 | name string 118 | path string 119 | opt Opts 120 | configExist bool 121 | err bool 122 | }{ 123 | { 124 | "empty path", 125 | "", 126 | nil, 127 | false, 128 | true, 129 | }, 130 | { 131 | "not empty", 132 | path, 133 | nil, 134 | false, 135 | true, 136 | }, 137 | { 138 | "not empty, cfg exist", 139 | path, 140 | nil, 141 | true, 142 | true, 143 | }, 144 | } 145 | for _, c := range cases { 146 | _, retErr := NewClient(c.path, c.opt) 147 | 148 | if c.configExist { 149 | cfg := `apiVersion: v1 150 | clusters: 151 | - cluster: 152 | server: http://localhost:80 153 | name: kubernetes 154 | contexts: 155 | - context: 156 | cluster: kubernetes 157 | user: kubernetes-admin 158 | name: kubernetes-admin@kubernetes 159 | current-context: kubernetes-admin@kubernetes 160 | kind: Config 161 | preferences: {} 162 | users: 163 | - name: kubernetes-admin` 164 | len, err := f.Write([]byte(cfg)) 165 | if err != nil { 166 | t.Fatal(err) 167 | } 168 | if len == 0 { 169 | t.Fatal("write failed") 170 | } 171 | f.Close() 172 | } 173 | if retErr != nil && !c.err { 174 | t.Fatalf("case: %v, err :%v", c.name, retErr) 175 | } 176 | if retErr == nil && c.err { 177 | t.Fatalf(c.name) 178 | } 179 | } 180 | } 181 | 182 | func TestCreateMergePatch(t *testing.T) { 183 | basePod := testbase.PodForTest() 184 | basePodWithSelector := testbase.PodForTestWithNodeSelector() 185 | desiredPatch := `{"spec":{"nodeSelector":{"testbase":"testbase"}}}` 186 | patch, err := CreateMergePatch(basePod, basePodWithSelector) 187 | if err != nil { 188 | t.Fatal(err) 189 | } 190 | if string(patch) != desiredPatch { 191 | t.Fatalf("desired path: \n%v\n, get: \n%v\n", desiredPatch, string(patch)) 192 | } 193 | } 194 | 195 | func TestCreateJSONPatch(t *testing.T) { 196 | basePod := testbase.PodForTest() 197 | basePodWithSelector := testbase.PodForTestWithNodeSelector() 198 | desiredPatch := `[{"op":"add","path":"/spec/nodeSelector","value":{"testbase":"testbase"}}]` 199 | patch, err := CreateJSONPatch(basePod, basePodWithSelector) 200 | if err != nil { 201 | t.Fatal(err) 202 | } 203 | if string(patch) != desiredPatch { 204 | t.Fatalf("desired path: \n%v\n, get: \n%v\n", desiredPatch, string(patch)) 205 | } 206 | } 207 | -------------------------------------------------------------------------------- /pkg/util/util.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright ©2020. The virtual-kubelet authors 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package util 18 | 19 | import ( 20 | "github.com/virtual-kubelet/tensile-kube/pkg/common" 21 | corev1 "k8s.io/api/core/v1" 22 | resourcehelper "k8s.io/kubernetes/pkg/api/v1/resource" 23 | ) 24 | 25 | // GetRequestFromPod get resources required by pod 26 | func GetRequestFromPod(pod *corev1.Pod) *common.Resource { 27 | if pod == nil { 28 | return nil 29 | } 30 | reqs, _ := resourcehelper.PodRequestsAndLimits(pod) 31 | capacity := common.ConvertResource(reqs) 32 | return capacity 33 | } 34 | -------------------------------------------------------------------------------- /pkg/util/util_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright ©2020. The virtual-kubelet authors 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package util 18 | 19 | import ( 20 | "github.com/virtual-kubelet/tensile-kube/pkg/common" 21 | "testing" 22 | 23 | "github.com/virtual-kubelet/tensile-kube/pkg/testbase" 24 | v1 "k8s.io/api/core/v1" 25 | "k8s.io/apimachinery/pkg/api/resource" 26 | ) 27 | 28 | func TestRequestFromPod(t *testing.T) { 29 | pod := testbase.PodForTest() 30 | pod1 := pod.DeepCopy() 31 | pod2 := pod.DeepCopy() 32 | pod2.Spec.Containers[0].Resources.Limits = nil 33 | pod3 := pod.DeepCopy() 34 | pod3.Spec.Containers[0].Resources.Limits = nil 35 | pod3.Spec.Containers[0].Resources.Requests = nil 36 | pod4 := pod.DeepCopy() 37 | pod4.Spec.Containers[0].Resources.Limits["ip"] = resource.MustParse("1") 38 | pod4.Spec.Containers[0].Resources.Requests["ip"] = resource.MustParse("1") 39 | 40 | desired := &common.Resource{CPU: resource.MustParse("10"), Memory: resource.MustParse("0"), 41 | Pods: resource.MustParse("0"), EphemeralStorage: resource.MustParse("0"), Custom: common.CustomResources{}} 42 | desiredNil := &common.Resource{Custom: common.CustomResources{}} 43 | 44 | desiredCustom := &common.Resource{CPU: resource.MustParse("10"), Memory: resource.MustParse("0"), 45 | Pods: resource.MustParse("0"), EphemeralStorage: resource.MustParse("0"), 46 | Custom: common.CustomResources{"ip": resource.MustParse("1")}} 47 | 48 | cases := []struct { 49 | pod *v1.Pod 50 | desire *common.Resource 51 | }{ 52 | { 53 | pod: pod1, 54 | desire: desired, 55 | }, 56 | { 57 | pod: pod2, 58 | desire: desired, 59 | }, 60 | { 61 | pod: pod3, 62 | desire: desiredNil, 63 | }, 64 | { 65 | pod: pod4, 66 | desire: desiredCustom, 67 | }, 68 | } 69 | for _, c := range cases { 70 | capacity := GetRequestFromPod(c.pod) 71 | if !c.desire.Equal(capacity) { 72 | t.Fatalf("desired: \n%v\n, get: \n%v\n", c.desire, capacity) 73 | } 74 | t.Logf("desired: \n%v\n, get: \n%v\n", c.desire, capacity) 75 | } 76 | } 77 | -------------------------------------------------------------------------------- /pkg/webhook/hook.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright ©2020. The virtual-kubelet authors 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package webhook 18 | 19 | import ( 20 | "encoding/json" 21 | "fmt" 22 | "io/ioutil" 23 | "net/http" 24 | 25 | "k8s.io/api/admission/v1beta1" 26 | admissionregistrationv1beta1 "k8s.io/api/admissionregistration/v1beta1" 27 | corev1 "k8s.io/api/core/v1" 28 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 29 | "k8s.io/apimachinery/pkg/runtime" 30 | "k8s.io/apimachinery/pkg/runtime/serializer" 31 | v1 "k8s.io/client-go/listers/core/v1" 32 | "k8s.io/klog" 33 | 34 | "github.com/virtual-kubelet/tensile-kube/pkg/util" 35 | ) 36 | 37 | var ( 38 | freezeCache = util.NewUnschedulableCache() 39 | runtimeScheme = runtime.NewScheme() 40 | codecs = serializer.NewCodecFactory(runtimeScheme) 41 | deserializer = codecs.UniversalDeserializer() 42 | // (https://github.com/kubernetes/kubernetes/issues/57982) 43 | defaulter = runtime.ObjectDefaulter(runtimeScheme) 44 | desiredMap = map[string]corev1.Toleration{ 45 | util.TaintNodeNotReady: { 46 | Key: util.TaintNodeNotReady, 47 | Operator: corev1.TolerationOpExists, 48 | Effect: corev1.TaintEffectNoExecute, 49 | }, 50 | util.TaintNodeUnreachable: { 51 | Key: util.TaintNodeUnreachable, 52 | Operator: corev1.TolerationOpExists, 53 | Effect: corev1.TaintEffectNoExecute, 54 | }, 55 | } 56 | ) 57 | 58 | // HookServer is an interface defines a server 59 | type HookServer interface { 60 | // Serve starts a server 61 | Serve(http.ResponseWriter, *http.Request) 62 | } 63 | 64 | // webhookServer is a sever for webhook 65 | type webhookServer struct { 66 | ignoreSelectorKeys []string 67 | pvcLister v1.PersistentVolumeClaimLister 68 | Server *http.Server 69 | } 70 | 71 | func init() { 72 | _ = corev1.AddToScheme(runtimeScheme) 73 | _ = admissionregistrationv1beta1.AddToScheme(runtimeScheme) 74 | } 75 | 76 | // NewWebhookServer start a new webhook server 77 | func NewWebhookServer(pvcLister v1.PersistentVolumeClaimLister, ignoreKeys []string) HookServer { 78 | return &webhookServer{ 79 | ignoreSelectorKeys: ignoreKeys, 80 | pvcLister: pvcLister, 81 | } 82 | } 83 | 84 | // mutate k8s pod annotations, Affinity, nodeSelector and etc. 85 | func (whsvr *webhookServer) mutate(ar *v1beta1.AdmissionReview) *v1beta1.AdmissionResponse { 86 | req := ar.Request 87 | var ( 88 | err error 89 | pod corev1.Pod 90 | ) 91 | switch req.Kind.Kind { 92 | case "Pod": 93 | rawBytes := req.Object.Raw 94 | klog.V(4).Infof("Raw request %v", string(rawBytes)) 95 | if err := json.Unmarshal(rawBytes, &pod); err != nil { 96 | klog.Errorf("Could not unmarshal raw object %v err: %v", req, err) 97 | return &v1beta1.AdmissionResponse{ 98 | Result: &metav1.Status{ 99 | Message: err.Error(), 100 | }, 101 | } 102 | } 103 | default: 104 | return &v1beta1.AdmissionResponse{ 105 | Allowed: false, 106 | } 107 | } 108 | if shouldSkip(&pod) { 109 | return &v1beta1.AdmissionResponse{ 110 | Allowed: true, 111 | } 112 | } 113 | ref := getOwnerRef(&pod) 114 | clone := pod.DeepCopy() 115 | switch req.Operation { 116 | case v1beta1.Update: 117 | setUnschedulableNodes(ref, clone) 118 | return &v1beta1.AdmissionResponse{ 119 | Allowed: true, 120 | } 121 | case v1beta1.Create: 122 | nodes := getUnschedulableNodes(ref, clone) 123 | if len(nodes) > 0 { 124 | klog.Infof("Create pod %v Not nodes %+v", clone.Name, nodes) 125 | clone.Spec.Affinity, _ = util.ReplacePodNodeNameNodeAffinity(clone.Spec.Affinity, ref, 0, nil, nodes...) 126 | } 127 | default: 128 | klog.Warningf("Skip operation: %v", req.Operation) 129 | } 130 | 131 | whsvr.trySetNodeName(clone) 132 | inject(clone, whsvr.ignoreSelectorKeys) 133 | patch, err := util.CreateJSONPatch(pod, clone) 134 | klog.Infof("Final patch %+v", string(patch)) 135 | var result metav1.Status 136 | if err != nil { 137 | result.Code = 403 138 | result.Message = err.Error() 139 | } 140 | jsonPatch := v1beta1.PatchTypeJSONPatch 141 | return &v1beta1.AdmissionResponse{ 142 | Allowed: true, 143 | Result: &result, 144 | Patch: patch, 145 | PatchType: &jsonPatch, 146 | } 147 | } 148 | 149 | // Serve method for webhook server 150 | func (whsvr *webhookServer) Serve(w http.ResponseWriter, r *http.Request) { 151 | admissionReview, err := getRequestReview(r) 152 | if err != nil { 153 | klog.Error(err) 154 | http.Error(w, err.Error(), http.StatusBadRequest) 155 | return 156 | } 157 | admissionResponse := whsvr.mutate(admissionReview) 158 | if admissionResponse != nil { 159 | admissionReview.Response = admissionResponse 160 | admissionReview.Response.UID = admissionReview.Request.UID 161 | } 162 | resp, err := json.Marshal(admissionReview) 163 | if err != nil { 164 | klog.Errorf("Can't encode response: %v", err) 165 | http.Error(w, fmt.Sprintf("could not encode response: %v", err), http.StatusInternalServerError) 166 | return 167 | } 168 | if _, err := w.Write(resp); err != nil { 169 | klog.Errorf("Can't write response: %v", err) 170 | http.Error(w, fmt.Sprintf("could not write response: %v", err), http.StatusInternalServerError) 171 | } 172 | } 173 | 174 | func (whsvr *webhookServer) trySetNodeName(pod *corev1.Pod) { 175 | if pod.Spec.Volumes == nil { 176 | return 177 | } 178 | nodeName := "" 179 | for _, volume := range pod.Spec.Volumes { 180 | pvcSource := volume.PersistentVolumeClaim 181 | if pvcSource == nil { 182 | continue 183 | } 184 | nodeName = whsvr.getNodeNameFromPVC(pod.Namespace, pvcSource.ClaimName) 185 | if len(nodeName) != 0 { 186 | pod.Spec.NodeName = nodeName 187 | klog.Infof("Set desired node name to %v ", nodeName) 188 | return 189 | } 190 | } 191 | return 192 | } 193 | 194 | func (whsvr *webhookServer) getNodeNameFromPVC(ns, pvcName string) string { 195 | var nodeName string 196 | pvc, err := whsvr.pvcLister.PersistentVolumeClaims(ns).Get(pvcName) 197 | if err != nil { 198 | return nodeName 199 | } 200 | if pvc.Annotations == nil { 201 | return nodeName 202 | } 203 | return pvc.Annotations[util.SelectedNodeKey] 204 | } 205 | 206 | func inject(pod *corev1.Pod, ignoreKeys []string) { 207 | nodeSelector := make(map[string]string) 208 | var affinity *corev1.Affinity 209 | 210 | if skipInject(pod) { 211 | return 212 | } 213 | 214 | if pod.Spec.Affinity != nil { 215 | affinity = injectAffinity(pod.Spec.Affinity, ignoreKeys) 216 | } 217 | 218 | if pod.Spec.NodeSelector != nil { 219 | nodeSelector = injectNodeSelector(pod.Spec.NodeSelector, ignoreKeys) 220 | } 221 | 222 | cns := util.ClustersNodeSelection{ 223 | NodeSelector: nodeSelector, 224 | Affinity: affinity, 225 | Tolerations: pod.Spec.Tolerations, 226 | } 227 | cnsByte, err := json.Marshal(cns) 228 | if err != nil { 229 | return 230 | } 231 | if pod.Annotations == nil { 232 | pod.Annotations = make(map[string]string) 233 | } 234 | pod.Annotations[util.SelectorKey] = string(cnsByte) 235 | 236 | pod.Spec.Tolerations = getPodTolerations(pod) 237 | } 238 | 239 | func getPodTolerations(pod *corev1.Pod) []corev1.Toleration { 240 | var notReady, unSchedulable bool 241 | tolerations := make([]corev1.Toleration, 0) 242 | for _, toleration := range pod.Spec.Tolerations { 243 | if toleration.Key == util.TaintNodeNotReady { 244 | notReady = true 245 | } 246 | if toleration.Key == util.TaintNodeUnreachable { 247 | unSchedulable = true 248 | } 249 | 250 | if _, found := desiredMap[toleration.Key]; found { 251 | tolerations = append(tolerations, desiredMap[toleration.Key]) 252 | continue 253 | } 254 | tolerations = append(tolerations, toleration) 255 | } 256 | return addDefaultPodTolerations(tolerations, notReady, unSchedulable) 257 | } 258 | 259 | func addDefaultPodTolerations(tolerations []corev1.Toleration, notReady, unSchedulable bool) []corev1.Toleration { 260 | if !notReady { 261 | tolerations = append(tolerations, desiredMap[util.TaintNodeNotReady]) 262 | } 263 | if !unSchedulable { 264 | tolerations = append(tolerations, desiredMap[util.TaintNodeUnreachable]) 265 | } 266 | return tolerations 267 | } 268 | 269 | // injectNodeSelector reserve ignoreLabels in nodeSelector, others would be removed 270 | func injectNodeSelector(nodeSelector map[string]string, ignoreLabels []string) map[string]string { 271 | finalNodeSelector := make(map[string]string) 272 | labelMap := make(map[string]string) 273 | for _, v := range ignoreLabels { 274 | labelMap[v] = v 275 | } 276 | for k, v := range nodeSelector { 277 | // not found in label, delete 278 | if labelMap[k] != "" { 279 | continue 280 | } 281 | delete(nodeSelector, k) 282 | finalNodeSelector[k] = v 283 | } 284 | return finalNodeSelector 285 | } 286 | 287 | func injectAffinity(affinity *corev1.Affinity, ignoreLabels []string) *corev1.Affinity { 288 | if affinity.NodeAffinity == nil { 289 | return nil 290 | } 291 | if affinity.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution == nil { 292 | return nil 293 | } 294 | required := affinity.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution 295 | if required == nil { 296 | return nil 297 | } 298 | labelMap := make(map[string]string) 299 | for _, v := range ignoreLabels { 300 | labelMap[v] = v 301 | } 302 | requiredCopy := affinity.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution.DeepCopy() 303 | var nodeSelectorTerm []corev1.NodeSelectorTerm 304 | for termIdx, term := range requiredCopy.NodeSelectorTerms { 305 | var mes, mfs []corev1.NodeSelectorRequirement 306 | var mesDeleteCount, mfsDeleteCount int 307 | for meIdx, me := range term.MatchExpressions { 308 | if labelMap[me.Key] != "" { 309 | // found key, do not delete 310 | continue 311 | } 312 | mes = append(mes, *me.DeepCopy()) 313 | 314 | required. 315 | NodeSelectorTerms[termIdx].MatchExpressions = append(required. 316 | NodeSelectorTerms[termIdx].MatchExpressions[:meIdx-mesDeleteCount], required. 317 | NodeSelectorTerms[termIdx].MatchExpressions[meIdx-mesDeleteCount+1:]...) 318 | mesDeleteCount++ 319 | } 320 | 321 | for mfIdx, mf := range term.MatchFields { 322 | if labelMap[mf.Key] != "" { 323 | // found key, do not delete 324 | continue 325 | } 326 | 327 | mfs = append(mfs, *mf.DeepCopy()) 328 | required. 329 | NodeSelectorTerms[termIdx].MatchFields = append(required. 330 | NodeSelectorTerms[termIdx].MatchFields[:mfIdx-mesDeleteCount], 331 | required.NodeSelectorTerms[termIdx].MatchFields[mfIdx-mfsDeleteCount+1:]...) 332 | mfsDeleteCount++ 333 | } 334 | if len(mfs) != 0 || len(mes) != 0 { 335 | nodeSelectorTerm = append(nodeSelectorTerm, corev1.NodeSelectorTerm{MatchFields: mfs, MatchExpressions: mes}) 336 | } 337 | } 338 | 339 | filteredTerms := make([]corev1.NodeSelectorTerm, 0) 340 | for _, term := range required.NodeSelectorTerms { 341 | if len(term.MatchFields) == 0 && len(term.MatchExpressions) == 0 { 342 | continue 343 | } 344 | filteredTerms = append(filteredTerms, term) 345 | } 346 | if len(filteredTerms) == 0 { 347 | required = nil 348 | } else { 349 | required.NodeSelectorTerms = filteredTerms 350 | } 351 | affinity.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution = required 352 | if len(nodeSelectorTerm) == 0 { 353 | return nil 354 | } 355 | return &corev1.Affinity{NodeAffinity: &corev1.NodeAffinity{ 356 | RequiredDuringSchedulingIgnoredDuringExecution: &corev1.NodeSelector{NodeSelectorTerms: nodeSelectorTerm}, 357 | }} 358 | } 359 | 360 | func shouldSkip(pod *corev1.Pod) bool { 361 | if pod.Namespace == "kube-system" { 362 | return true 363 | } 364 | if pod.Labels != nil { 365 | if pod.Labels[util.CreatedbyDescheduler] == "true" { 366 | return true 367 | } 368 | if !util.IsVirtualPod(pod) { 369 | return true 370 | } 371 | } 372 | return false 373 | } 374 | 375 | func skipInject(pod *corev1.Pod) bool { 376 | return len(pod.Spec.NodeSelector) == 0 && 377 | pod.Spec.Affinity == nil && 378 | pod.Spec.Tolerations == nil 379 | } 380 | 381 | func getOwnerRef(pod *corev1.Pod) string { 382 | ref := "" 383 | if len(pod.OwnerReferences) > 0 { 384 | ref = string(pod.OwnerReferences[0].UID) 385 | } 386 | return ref 387 | } 388 | 389 | func setUnschedulableNodes(ref string, pod *corev1.Pod) { 390 | node := "" 391 | if len(ref) == 0 { 392 | return 393 | } 394 | if pod.Annotations != nil { 395 | node = pod.Annotations["unschedulable-node"] 396 | } 397 | if len(node) > 0 { 398 | klog.Infof("Unschedulable nodes %+v ref %v to cache", node, ref) 399 | freezeCache.Add(node, ref) 400 | } 401 | } 402 | 403 | func getUnschedulableNodes(ref string, pod *corev1.Pod) []string { 404 | var nodes []string 405 | if len(ref) == 0 { 406 | return nodes 407 | } 408 | if len(pod.Spec.NodeName) != 0 { 409 | return nodes 410 | } 411 | nodes = freezeCache.GetFreezeNodes(ref) 412 | klog.Infof("Not in nodes %v for %v", nodes, ref) 413 | return nodes 414 | } 415 | 416 | func getRequestReview(r *http.Request) (*v1beta1.AdmissionReview, error) { 417 | if r.Body == nil { 418 | return nil, fmt.Errorf("empty body") 419 | } 420 | body, err := ioutil.ReadAll(r.Body) 421 | if err != nil { 422 | return nil, err 423 | } 424 | klog.V(5).Infof("Receive request: %+v", *r) 425 | if len(body) == 0 { 426 | return nil, fmt.Errorf("empty body") 427 | } 428 | ar := v1beta1.AdmissionReview{} 429 | if deserializer.Decode(body, nil, &ar); err != nil { 430 | return nil, fmt.Errorf("Can't decode body: %v", err) 431 | } 432 | return &ar, nil 433 | } 434 | -------------------------------------------------------------------------------- /pkg/webhook/hook_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright ©2020. The virtual-kubelet authors 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package webhook 18 | 19 | import ( 20 | "encoding/json" 21 | "reflect" 22 | "testing" 23 | 24 | test "github.com/virtual-kubelet/tensile-kube/pkg/testbase" 25 | "github.com/virtual-kubelet/tensile-kube/pkg/util" 26 | v1 "k8s.io/api/core/v1" 27 | ) 28 | 29 | func TestGetPodTolerations(t *testing.T) { 30 | pod1 := test.PodForTestWithSystemTolerations() 31 | pod2 := test.PodForTestWithOtherTolerations() 32 | cases := []struct { 33 | pod *v1.Pod 34 | desireTolerations []v1.Toleration 35 | }{ 36 | { 37 | pod: pod1, 38 | desireTolerations: []v1.Toleration{ 39 | { 40 | Key: util.TaintNodeNotReady, 41 | Operator: v1.TolerationOpExists, 42 | Effect: v1.TaintEffectNoExecute, 43 | }, 44 | { 45 | Key: util.TaintNodeUnreachable, 46 | Operator: v1.TolerationOpExists, 47 | Effect: v1.TaintEffectNoExecute, 48 | }, 49 | }, 50 | }, 51 | { 52 | pod: pod2, 53 | desireTolerations: []v1.Toleration{ 54 | { 55 | Key: "testbase", 56 | Operator: v1.TolerationOpExists, 57 | Effect: v1.TaintEffectNoExecute, 58 | }, 59 | { 60 | Key: util.TaintNodeNotReady, 61 | Operator: v1.TolerationOpExists, 62 | Effect: v1.TaintEffectNoExecute, 63 | }, 64 | { 65 | Key: util.TaintNodeUnreachable, 66 | Operator: v1.TolerationOpExists, 67 | Effect: v1.TaintEffectNoExecute, 68 | }, 69 | }, 70 | }, 71 | } 72 | for _, c := range cases { 73 | tolerations := getPodTolerations(c.pod) 74 | if !reflect.DeepEqual(tolerations, c.desireTolerations) { 75 | t.Fatalf("Desire %v, Get %v", tolerations, c.desireTolerations) 76 | } 77 | } 78 | } 79 | 80 | func TestInject(t *testing.T) { 81 | cases := []struct { 82 | name string 83 | pod *v1.Pod 84 | desireCNS util.ClustersNodeSelection 85 | keys []string 86 | }{ 87 | { 88 | name: "Pod ForTest With Node Selector", 89 | pod: test.PodForTestWithNodeSelector(), 90 | desireCNS: util.ClustersNodeSelection{ 91 | NodeSelector: map[string]string{"testbase": "testbase"}, 92 | }, 93 | }, 94 | { 95 | name: "Pod For Test With Node Selector with clusterID", 96 | pod: test.PodForTestWithNodeSelectorClusterID(), 97 | desireCNS: util.ClustersNodeSelection{ 98 | NodeSelector: map[string]string{"testbase": "testbase"}, 99 | }, 100 | keys: []string{util.ClusterID}, 101 | }, 102 | { 103 | name: "Pod For Test With Node Selector and Affinity with clusterID", 104 | pod: test.PodForTestWithNodeSelectorAndAffinityClusterID(), 105 | desireCNS: util.ClustersNodeSelection{ 106 | NodeSelector: map[string]string{"testbase": "testbase"}, 107 | Affinity: &v1.Affinity{ 108 | NodeAffinity: &v1.NodeAffinity{ 109 | RequiredDuringSchedulingIgnoredDuringExecution: &v1.NodeSelector{NodeSelectorTerms: []v1. 110 | NodeSelectorTerm{{ 111 | MatchExpressions: []v1.NodeSelectorRequirement{ 112 | { 113 | Key: "test0", 114 | Operator: v1.NodeSelectorOpIn, 115 | Values: []string{ 116 | "aa", 117 | }, 118 | }, 119 | { 120 | Key: "test", 121 | Operator: v1.NodeSelectorOpIn, 122 | Values: []string{ 123 | "aa", 124 | }, 125 | }, 126 | { 127 | Key: "test1", 128 | Operator: v1.NodeSelectorOpIn, 129 | Values: []string{ 130 | "aa", 131 | }, 132 | }, 133 | }, 134 | }}}, 135 | PreferredDuringSchedulingIgnoredDuringExecution: nil, 136 | }, 137 | }, 138 | }, 139 | keys: []string{util.ClusterID}, 140 | }, 141 | { 142 | name: "Pod For Test With Node Selector and Affinity without match labels", 143 | pod: test.PodForTestWithNodeSelectorAndAffinityClusterID(), 144 | desireCNS: util.ClustersNodeSelection{ 145 | NodeSelector: map[string]string{"testbase": "testbase", "clusterID": "1"}, 146 | Affinity: &v1.Affinity{ 147 | NodeAffinity: &v1.NodeAffinity{ 148 | RequiredDuringSchedulingIgnoredDuringExecution: &v1.NodeSelector{NodeSelectorTerms: []v1. 149 | NodeSelectorTerm{{ 150 | MatchExpressions: []v1.NodeSelectorRequirement{ 151 | { 152 | Key: "test0", 153 | Operator: v1.NodeSelectorOpIn, 154 | Values: []string{ 155 | "aa", 156 | }, 157 | }, 158 | { 159 | Key: "clusterID", 160 | Operator: v1.NodeSelectorOpIn, 161 | Values: []string{ 162 | "aa", 163 | }, 164 | }, 165 | { 166 | Key: "test", 167 | Operator: v1.NodeSelectorOpIn, 168 | Values: []string{ 169 | "aa", 170 | }, 171 | }, 172 | { 173 | Key: "test1", 174 | Operator: v1.NodeSelectorOpIn, 175 | Values: []string{ 176 | "aa", 177 | }, 178 | }, 179 | }, 180 | }}}, 181 | PreferredDuringSchedulingIgnoredDuringExecution: nil, 182 | }, 183 | }, 184 | }, 185 | }, 186 | { 187 | name: "Pod For Test With Node Selector and Affinity with multi match labels", 188 | pod: test.PodForTestWithNodeSelectorAndAffinityClusterID(), 189 | desireCNS: util.ClustersNodeSelection{ 190 | NodeSelector: map[string]string{"testbase": "testbase"}, 191 | Affinity: &v1.Affinity{ 192 | NodeAffinity: &v1.NodeAffinity{ 193 | RequiredDuringSchedulingIgnoredDuringExecution: &v1.NodeSelector{NodeSelectorTerms: []v1. 194 | NodeSelectorTerm{{ 195 | MatchExpressions: []v1.NodeSelectorRequirement{ 196 | { 197 | Key: "test", 198 | Operator: v1.NodeSelectorOpIn, 199 | Values: []string{ 200 | "aa", 201 | }, 202 | }, 203 | { 204 | Key: "test1", 205 | Operator: v1.NodeSelectorOpIn, 206 | Values: []string{ 207 | "aa", 208 | }, 209 | }, 210 | }, 211 | }}}, 212 | PreferredDuringSchedulingIgnoredDuringExecution: nil, 213 | }, 214 | }, 215 | }, 216 | keys: []string{"clusterID", "test0"}, 217 | }, 218 | { 219 | name: "Pod For Test With Affinity", 220 | pod: test.PodForTestWithAffinity(), 221 | desireCNS: util.ClustersNodeSelection{ 222 | NodeSelector: map[string]string{"testbase": "testbase"}, 223 | Affinity: &v1.Affinity{ 224 | NodeAffinity: &v1.NodeAffinity{ 225 | RequiredDuringSchedulingIgnoredDuringExecution: &v1.NodeSelector{NodeSelectorTerms: []v1. 226 | NodeSelectorTerm{{ 227 | MatchExpressions: []v1.NodeSelectorRequirement{ 228 | { 229 | Key: "testbase", 230 | Operator: v1.NodeSelectorOpIn, 231 | Values: []string{ 232 | "aa", 233 | }, 234 | }, 235 | }, 236 | }}}, 237 | PreferredDuringSchedulingIgnoredDuringExecution: nil, 238 | }, 239 | }, 240 | }, 241 | }, 242 | } 243 | for _, c := range cases { 244 | t.Logf("Running %v", c.name) 245 | inject(c.pod, c.keys) 246 | str := c.pod.Annotations[util.SelectorKey] 247 | cns := util.ClustersNodeSelection{} 248 | err := json.Unmarshal([]byte(str), &cns) 249 | if err != nil { 250 | t.Fatal(err) 251 | } 252 | t.Logf("ann: %v", str) 253 | if !reflect.DeepEqual(cns, c.desireCNS) { 254 | t.Fatalf("Desire: %v, Get: %v", c.desireCNS, cns) 255 | } 256 | t.Logf("Desire: %v, Get: %v", c.desireCNS, cns) 257 | } 258 | } 259 | --------------------------------------------------------------------------------