├── .github ├── CODEOWNERS ├── dependabot.yml └── workflows │ └── ci.yml ├── .gitignore ├── Dockerfile ├── LICENSE ├── Makefile ├── README.md ├── cmd └── kucero │ ├── lock.go │ └── main.go ├── controllers ├── certificatesigningrequest_controller.go ├── helper.go └── helper_test.go ├── demo ├── kubeadm.cast └── kubelet.cast ├── go.mod ├── go.sum ├── integration └── test.sh ├── logo.png ├── manifest ├── daemonset.yaml └── privileged.yaml └── pkg ├── host ├── command.go ├── kubectl.go └── kubelet.go └── pki ├── authority ├── authority.go └── policies.go ├── cert ├── certificate.go ├── kubeadm │ ├── exec.go │ ├── exec_test.go │ └── kubeadm.go ├── null │ └── null.go └── parser.go ├── clock └── clock.go ├── conf ├── conf.go └── kubelet │ ├── exec.go │ ├── exec_test.go │ ├── kubelet.go │ └── tests │ ├── 1.16-kubelet.conf │ ├── 1.17-kubelet.conf │ ├── client-kubelet.yaml │ └── server-kubelet.yaml ├── node └── node.go └── signer ├── ca_provider.go └── signer.go /.github/CODEOWNERS: -------------------------------------------------------------------------------- 1 | * @jenting 2 | -------------------------------------------------------------------------------- /.github/dependabot.yml: -------------------------------------------------------------------------------- 1 | # To get started with Dependabot version updates, you'll need to specify which 2 | # package ecosystems to update and where the package manifests are located. 3 | # Please see the documentation for all configuration options: 4 | # https://docs.github.com/github/administering-a-repository/configuration-options-for-dependency-updates 5 | 6 | version: 2 7 | updates: 8 | - package-ecosystem: "gomod" # See documentation for possible values 9 | directory: "/" # Location of package manifests 10 | schedule: 11 | interval: "weekly" 12 | -------------------------------------------------------------------------------- /.github/workflows/ci.yml: -------------------------------------------------------------------------------- 1 | name: CI 2 | permissions: 3 | contents: read 4 | pull-requests: write 5 | 6 | on: 7 | pull_request: 8 | push: 9 | branches: 10 | - main 11 | 12 | jobs: 13 | build: 14 | strategy: 15 | matrix: 16 | go: 17 | - '1.21' 18 | - '1.22' 19 | - '1.23' 20 | - '1.24' 21 | 22 | runs-on: ubuntu-latest 23 | steps: 24 | - name: Check out code into the Go module directory 25 | uses: actions/checkout@v3 26 | 27 | - name: Set up Go ${{ matrix.go }} 28 | uses: actions/setup-go@v4 29 | with: 30 | go-version: ${{ matrix.go }} 31 | 32 | - name: Go build 33 | run: make 34 | 35 | test: 36 | runs-on: ubuntu-latest 37 | strategy: 38 | max-parallel: 1 39 | matrix: 40 | k8s: 41 | - v1.19.16 42 | - v1.20.15 43 | - v1.21.12 44 | - v1.22.15 45 | - v1.23.13 46 | - v1.24.12 47 | - v1.25.8 48 | - v1.26.3 49 | - v1.27.0 50 | - v1.28.9 51 | - v1.29.4 52 | - v1.30.0 53 | - v1.31.0 54 | - v1.32.0 55 | - v1.33.0 56 | steps: 57 | - name: Check out code into the Go module directory 58 | uses: actions/checkout@v3 59 | 60 | - name: Set up KIND cluster 61 | uses: engineerd/setup-kind@v0.5.0 62 | with: 63 | version: "v0.27.0" 64 | image: "kindest/node:${{ matrix.k8s }}" 65 | 66 | - name: Install kustomize 67 | env: 68 | GITHUB_TOKEN: ${{ secrets.GHCR_TOKEN }} 69 | run: | 70 | curl -s "https://raw.githubusercontent.com/kubernetes-sigs/kustomize/kustomize/v5.0.3/hack/install_kustomize.sh" | bash -s -- 4.5.7 71 | sudo mv kustomize /usr/local/bin/kustomize 72 | 73 | - name: E2E Test 74 | run: | 75 | make e2e-test 76 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Binaries for programs and plugins 2 | *.exe 3 | *.exe~ 4 | *.dll 5 | *.so 6 | *.dylib 7 | 8 | # Test binary, built with `go test -c` 9 | *.test 10 | 11 | # Output of the go coverage tool, specifically when used with LiteIDE 12 | *.out 13 | 14 | # Dependency directories (remove the comment below to include it) 15 | vendor/ 16 | 17 | # Binary 18 | cmd/kucero/kucero 19 | 20 | # kustomize 21 | manifest/kustomization.yaml 22 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM golang:1.24-bookworm as build 2 | WORKDIR /src 3 | 4 | ARG VERSION=latest 5 | 6 | COPY . . 7 | RUN go mod download && \ 8 | CGO_ENABLED=0 GOOS=linux go build -ldflags "-s -w -X main.version=${VERSION}" -o kucero cmd/kucero/*.go 9 | 10 | FROM cgr.dev/chainguard/wolfi-base 11 | WORKDIR /usr/bin 12 | COPY --from=build /src/kucero . 13 | ENTRYPOINT ["/usr/bin/kucero"] 14 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright [yyyy] [name of copyright owner] 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | MAKEFLAGS += --warn-undefined-variables 2 | SHELL := bash 3 | .SHELLFLAGS := -eu -o pipefail -c 4 | .DELETE_ON_ERROR: 5 | .SUFFIXES: 6 | 7 | # The semver version number which will be used as the Docker image tag 8 | # Defaults to the output of git describe. 9 | VERSION ?= $(shell git describe --tags --dirty --always) 10 | 11 | # Docker image name parameters 12 | IMG_NAME ?= jenting/kucero 13 | IMG_TAG ?= ${VERSION} 14 | IMG ?= ${IMG_NAME}:${IMG_TAG} 15 | 16 | # Get the currently used golang install path (in GOPATH/bin, unless GOBIN is set) 17 | ifeq (,$(shell go env GOBIN)) 18 | GOBIN=$(shell go env GOPATH)/bin 19 | else 20 | GOBIN=$(shell go env GOBIN) 21 | endif 22 | 23 | BIN := ${CURDIR}/bin 24 | export PATH := ${BIN}:${PATH} 25 | 26 | all: kucero 27 | 28 | verify: 29 | go mod tidy 30 | go mod verify 31 | 32 | test: verify 33 | go test -count=1 ./... 34 | 35 | e2e-test: docker-build 36 | bash +x ./integration/test.sh ${IMG} 37 | cd manifest && kustomize create --autodetect 2>/dev/null || true 38 | kustomize build manifest | kubectl delete -f - 39 | 40 | kucero: test 41 | CGO_ENABLED=0 go build -ldflags "-s -w -X main.version=$(VERSION)" -o cmd/kucero/kucero cmd/kucero/*.go 42 | 43 | # Run go fmt against code 44 | fmt: 45 | go fmt ./... 46 | 47 | # Run go vet against code 48 | vet: 49 | go vet ./... 50 | 51 | # Build the docker image 52 | docker-build: 53 | docker build --build-arg VERSION=${VERSION} -t ${IMG} . 54 | 55 | # Push the docker image 56 | docker-push: 57 | docker push ${IMG} 58 | 59 | # Deploy manifest 60 | deploy-manifest: 61 | cd manifest && kustomize create --autodetect 2>/dev/null || true 62 | cd manifest && kustomize edit set image kucero=${IMG} 63 | kustomize build manifest | kubectl apply -f - 64 | 65 | # Destroy manifest 66 | destroy-manifest: 67 | cd manifest && kustomize create --autodetect 2>/dev/null || true 68 | kustomize build manifest | kubectl delete -f - 69 | 70 | clean: 71 | go clean -x -i ./... 72 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | ![CI](https://github.com/SUSE/kucero/workflows/CI/badge.svg) 2 | 3 | ![kucero](logo.png) 4 | 5 | ## Introduction 6 | 7 | Kucero (KUbernetes CErtificate ROtation) is a Kubernetes daemonset that 8 | performs _automatic_ Kubernetes control plane certificate rotation. 9 | 10 | Kucero takes care both: 11 | - kubeadm certificates and kubeconfigs: kucero periodically watches the kubeadm generated certificates and kubeconfigs on host system, and renews certificates/kubeconfigs when the certificates/kubeconfigs residual time is below than user configured time period. 12 | - kubelet certificates: 13 | - kubelet.conf: kucero helps on auto-update the `/etc/kubernetes/kubelet.conf` from embedded base64 encoded client cert/key to using the local file `/var/lib/kubelet/kubelet-client-current.pem` (this is a bug if you bootstrap a cluster with kubeadm version < 1.17). 14 | - client certificate: kucero helps on configuring `rotateCertificates: true` or `rotateCertificates: false` in `/var/lib/kubelet/config.yaml` which controls to auto rotates the kubelet client certificate or not. When configures `rotateCertificates: true`, the kubelet sends out the client CSR at approximately 70%-90% of the total lifetime of the certificate, then the kube-controler-manager watches kubelet client CSR, and then auto signs and approves kubelet client certificates with Kubernetes cluster CA cert/key pair. 15 | - server certificate: kucero helps on configuring `serverTLSBootstrap: true` or `serverTLSBootstrap: false` in `/var/lib/kubelet/config.yaml` which controls to auto rotates the kubelet server certificate or not. When configures `serverTLSBootstrap: true`, the kubelet sends out the server CSR at approximately 70%-90% of the total lifetime of the certificate, then the kucero controller watches kubelet server CSR, and then auto signs and approves kubelet server certificates with user-specified CA cert/key pair. 16 | 17 | ## Kubelet Configuration 18 | 19 | By default, kucero enables kubelet client `rotateCertificates: true` and server certificates `serverTLSBootstrap: true` auto rotation, you could disable it by passing flags to kucero: 20 | - `--enable-kubelet-client-cert-rotation=false` 21 | - `--enable-kubelet-server-cert-rotation=false` 22 | 23 | ## Build Requirements 24 | 25 | - Golang >= 1.17 26 | - Docker 27 | - Kustomize 28 | 29 | ## Container Requirement Package 30 | 31 | - /usr/bin/nsenter 32 | 33 | ## Kubeadm Compatibility 34 | 35 | - kubeadm >= 1.15.0 36 | 37 | ## Installation 38 | 39 | ``` 40 | make docker-build IMG= 41 | make docker-push IMG= 42 | make deploy-manifest IMG= 43 | ``` 44 | 45 | ## Configuration 46 | 47 | The following arguments can be passed to kucero via the daemonset pod template: 48 | 49 | ``` 50 | Flags: 51 | --ca-cert-path string sign CSR with this certificate file (default "/etc/kubernetes/pki/ca.crt") 52 | --ca-key-path string sign CSR with this private key file (default "/etc/kubernetes/pki/ca.key") 53 | --ds-name string name of daemonset on which to place lock (default "kucero") 54 | --ds-namespace string namespace containing daemonset on which to place lock (default "kube-system") 55 | --enable-kucero-controller enable kucero controller (default true) 56 | -h, --help help for kucero 57 | --leader-election-id string the name of the configmap used to coordinate leader election between kucero-controllers (default "kucero-leader-election") 58 | --lock-annotation string annotation in which to record locking node (default "caasp.suse.com/kucero-node-lock") 59 | --metrics-addr string the address the metric endpoint binds to (default ":8080") 60 | --polling-period duration certificate rotation check period (default 1h0m0s) 61 | --renew-before duration rotates certificate before expiry is below (default 720h0m0s) 62 | ``` 63 | 64 | ## Uninstallation 65 | 66 | ``` 67 | make destroy-manifest 68 | ``` 69 | 70 | ## Demo 71 | 72 | - kubeadm 73 | [![asciicast](https://asciinema.org/a/340053.svg)](https://asciinema.org/a/340053) 74 | - kubelet 75 | [![asciicast](https://asciinema.org/a/340054.svg)](https://asciinema.org/a/340054) 76 | -------------------------------------------------------------------------------- /cmd/kucero/lock.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright (c) 2020 SUSE LLC. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package main 18 | 19 | import ( 20 | "time" 21 | 22 | "github.com/sirupsen/logrus" 23 | "github.com/weaveworks/kured/pkg/daemonsetlock" 24 | ) 25 | 26 | func holding(lock *daemonsetlock.DaemonSetLock, metadata interface{}) bool { 27 | holding, err := lock.Test(metadata) 28 | if err != nil { 29 | logrus.Errorf("Error testing lock: %v", err) 30 | } 31 | if holding { 32 | logrus.Info("Holding lock") 33 | } 34 | return holding 35 | } 36 | 37 | func acquire(lock *daemonsetlock.DaemonSetLock, metadata interface{}) bool { 38 | holding, holder, err := lock.Acquire(metadata, time.Minute) 39 | switch { 40 | case err != nil: 41 | logrus.Errorf("Error acquiring lock: %v", err) 42 | return false 43 | case !holding: 44 | logrus.Warnf("Lock already held: %v", holder) 45 | return false 46 | default: 47 | logrus.Info("Acquired kucero lock") 48 | return true 49 | } 50 | } 51 | 52 | func release(lock *daemonsetlock.DaemonSetLock) { 53 | logrus.Info("Releasing lock") 54 | if err := lock.Release(); err != nil { 55 | logrus.Errorf("Error releasing lock: %v", err) 56 | } 57 | } 58 | -------------------------------------------------------------------------------- /cmd/kucero/main.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright (c) 2020 SUSE LLC. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package main 18 | 19 | import ( 20 | "context" 21 | "math/rand" 22 | "os" 23 | "os/signal" 24 | "syscall" 25 | "time" 26 | 27 | corev1 "k8s.io/api/core/v1" 28 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 29 | "k8s.io/apimachinery/pkg/runtime" 30 | utilruntime "k8s.io/apimachinery/pkg/util/runtime" 31 | "k8s.io/client-go/kubernetes" 32 | clientgoscheme "k8s.io/client-go/kubernetes/scheme" 33 | "k8s.io/client-go/tools/clientcmd" 34 | ctrl "sigs.k8s.io/controller-runtime" 35 | metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server" 36 | 37 | "github.com/sirupsen/logrus" 38 | "github.com/spf13/cobra" 39 | "github.com/weaveworks/kured/pkg/daemonsetlock" 40 | 41 | "github.com/jenting/kucero/controllers" 42 | "github.com/jenting/kucero/pkg/host" 43 | "github.com/jenting/kucero/pkg/pki/node" 44 | "github.com/jenting/kucero/pkg/pki/signer" 45 | //+kubebuilder:scaffold:imports 46 | ) 47 | 48 | var ( 49 | version = "unreleased" 50 | 51 | // Command line flags 52 | apiServerHost, kubeconfig string 53 | pollingPeriod, expiryTimeToRotate, duration time.Duration 54 | dsNamespace, dsName, lockAnnotation string 55 | enableKubeletCSRController bool 56 | metricsAddr string 57 | leaderElectionID string 58 | caCertPath, caKeyPath string 59 | enableKubeletClientCertRotation bool 60 | enableKubeletServerCertRotation bool 61 | 62 | scheme = runtime.NewScheme() 63 | ) 64 | 65 | func init() { 66 | utilruntime.Must(clientgoscheme.AddToScheme(scheme)) 67 | //+kubebuilder:scaffold:scheme 68 | } 69 | 70 | func main() { 71 | rootCmd := &cobra.Command{ 72 | Use: "kucero", 73 | Short: "KUbernetes CErtificate ROtation", 74 | Run: root, 75 | } 76 | 77 | // general 78 | rootCmd.PersistentFlags().StringVar(&apiServerHost, "master", "", 79 | "Optional apiserver host address to connect to") 80 | rootCmd.PersistentFlags().StringVar(&kubeconfig, "kubeconfig", "", 81 | "Paths to a kubeconfig. Only required if out-of-cluster.") 82 | 83 | // kubeadm 84 | rootCmd.PersistentFlags().DurationVar(&pollingPeriod, "polling-period", time.Hour, 85 | "Certificate rotation check period") 86 | rootCmd.PersistentFlags().DurationVar(&expiryTimeToRotate, "renew-before", time.Hour*24*30, 87 | "Rotates certificate if certificate not after is below") 88 | rootCmd.PersistentFlags().StringVar(&dsNamespace, "ds-namespace", "kube-system", 89 | "The namespace containing daemonset on which to place lock") 90 | rootCmd.PersistentFlags().StringVar(&dsName, "ds-name", "kucero", 91 | "The name of daemonset on which to place lock") 92 | rootCmd.PersistentFlags().StringVar(&lockAnnotation, "lock-annotation", "caasp.suse.com/kucero-node-lock", 93 | "The annotation in which to record locking node") 94 | 95 | // kubelet CSR controller 96 | rootCmd.PersistentFlags().BoolVar(&enableKubeletCSRController, "enable-kubelet-csr-controller", true, 97 | "Enable kubelet CSR controller") 98 | rootCmd.PersistentFlags().StringVar(&metricsAddr, "metrics-addr", ":8080", 99 | "The address the metric endpoint binds to") 100 | rootCmd.PersistentFlags().StringVar(&leaderElectionID, "leader-election-id", "kucero-leader-election", 101 | "The name of the configmap used to coordinate leader election between kucero-controllers") 102 | rootCmd.PersistentFlags().StringVar(&caCertPath, "ca-cert-path", "/etc/kubernetes/pki/ca.crt", 103 | "To sign CSR with this certificate file") 104 | rootCmd.PersistentFlags().StringVar(&caKeyPath, "ca-key-path", "/etc/kubernetes/pki/ca.key", 105 | "To sign CSR with this private key file") 106 | rootCmd.PersistentFlags().DurationVar(&duration, "duration", time.Hour*24*365, 107 | "Kubelet certificate duration") 108 | 109 | // kubelet configuration 110 | rootCmd.PersistentFlags().BoolVar(&enableKubeletClientCertRotation, "enable-kubelet-client-cert-rotation", true, 111 | "Enable kubelet client cert rotation") 112 | rootCmd.PersistentFlags().BoolVar(&enableKubeletServerCertRotation, "enable-kubelet-server-cert-rotation", true, 113 | "Enable kubelet server cert rotation") 114 | 115 | if err := rootCmd.Execute(); err != nil { 116 | logrus.Error(err) 117 | } 118 | } 119 | 120 | func root(cmd *cobra.Command, args []string) { 121 | logrus.Infof("KUbernetes CErtificate ROtation Daemon: %s", version) 122 | 123 | nodeName := os.Getenv("KUCERO_NODE_NAME") 124 | if nodeName == "" { 125 | logrus.Fatal("KUCERO_NODE_NAME environment variable required") 126 | } 127 | 128 | // shifting certificate check polling period 129 | rand.Seed(time.Now().UnixNano()) 130 | extra := rand.Intn(int(pollingPeriod.Seconds())) 131 | pollingPeriod = pollingPeriod + time.Duration(extra)*time.Second 132 | 133 | // check it's a control plane node or worker node 134 | config, err := clientcmd.BuildConfigFromFlags(apiServerHost, kubeconfig) 135 | if err != nil { 136 | logrus.Fatal(err) 137 | } 138 | 139 | client, err := kubernetes.NewForConfig(config) 140 | if err != nil { 141 | logrus.Fatal(err) 142 | } 143 | 144 | corev1Node, err := client.CoreV1().Nodes().Get(context.TODO(), nodeName, metav1.GetOptions{}) 145 | if err != nil { 146 | logrus.Fatal(err) 147 | } 148 | 149 | _, master := corev1Node.GetLabels()["node-role.kubernetes.io/master"] 150 | _, controlPlane := corev1Node.GetLabels()["node-role.kubernetes.io/control-plane"] 151 | isControlPlaneNode := master || controlPlane 152 | 153 | logrus.Infof("Node Name: %s", nodeName) 154 | logrus.Infof("Lock Annotation: %s/%s:%s", dsNamespace, dsName, lockAnnotation) 155 | logrus.Infof("Shifted Certificate Check Polling Period %v", pollingPeriod) 156 | logrus.Infof("Rotates Certificate If Expiry Time Less Than %v", expiryTimeToRotate) 157 | logrus.Infof("Kubelet client cert rotation enabled: %t", enableKubeletClientCertRotation) 158 | logrus.Infof("Kubelet server cert rotation enabled: %t", enableKubeletServerCertRotation) 159 | if enableKubeletCSRController && isControlPlaneNode { 160 | logrus.Infof("Kubelet CSR controller leader election ID: %s", leaderElectionID) 161 | logrus.Infof("Kubelet CSR controller CA cert: %s", caCertPath) 162 | logrus.Infof("Kubelet CSR controller CA key: %s", caKeyPath) 163 | } 164 | 165 | rotateCertificateWhenNeeded(corev1Node, isControlPlaneNode, client) 166 | } 167 | 168 | // nodeMeta is used to remember information across nodes 169 | // whom is doing certificate rotation 170 | type nodeMeta struct { 171 | Unschedulable bool `json:"unschedulable"` 172 | } 173 | 174 | func rotateCertificateWhenNeeded(corev1Node *corev1.Node, isControlPlaneNode bool, client *kubernetes.Clientset) { 175 | nodeName := corev1Node.GetName() 176 | certNode := node.New(isControlPlaneNode, nodeName, expiryTimeToRotate, enableKubeletClientCertRotation, enableKubeletServerCertRotation) 177 | 178 | lock := daemonsetlock.New(client, nodeName, dsNamespace, dsName, lockAnnotation) 179 | nodeMeta := nodeMeta{} 180 | if holding(lock, &nodeMeta) { 181 | release(lock) 182 | } 183 | 184 | if enableKubeletCSRController && isControlPlaneNode { 185 | go func() { 186 | mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{ 187 | Scheme: scheme, 188 | Metrics: metricsserver.Options{ 189 | BindAddress: metricsAddr, 190 | }, 191 | LeaderElection: true, 192 | LeaderElectionNamespace: dsNamespace, 193 | LeaderElectionID: leaderElectionID, 194 | }) 195 | if err != nil { 196 | logrus.Fatal(err) 197 | } 198 | 199 | signer, err := signer.NewSigner(caCertPath, caKeyPath, duration) 200 | if err != nil { 201 | logrus.Fatal(err) 202 | } 203 | 204 | if err := (&controllers.CertificateSigningRequestSigningReconciler{ 205 | Client: mgr.GetClient(), 206 | ClientSet: kubernetes.NewForConfigOrDie(mgr.GetConfig()), 207 | Scheme: mgr.GetScheme(), 208 | Signer: signer, 209 | EventRecorder: mgr.GetEventRecorderFor("CSRSigningReconciler"), 210 | }).SetupWithManager(mgr); err != nil { 211 | logrus.Fatal(err) 212 | } 213 | //+kubebuilder:scaffold:builder 214 | 215 | logrus.Info("Starting manager") 216 | if err := mgr.Start(ctrl.SetupSignalHandler()); err != nil { 217 | logrus.Fatal(err) 218 | } 219 | }() 220 | } 221 | 222 | quit := make(chan os.Signal, 1) 223 | signal.Notify(quit, syscall.SIGINT, syscall.SIGTERM) 224 | 225 | ch := time.Tick(pollingPeriod) 226 | for { 227 | select { 228 | case <-quit: 229 | logrus.Info("Quitting") 230 | return 231 | case <-ch: 232 | logrus.Info("Check certificate expiration") 233 | 234 | // check the configuration needs to be update 235 | configsToBeUpdate, err := certNode.CheckConfig() 236 | if err != nil { 237 | logrus.Error(err) 238 | } 239 | 240 | // check the certificate needs expiration 241 | expiryCerts, err := certNode.CheckExpiration() 242 | if err != nil { 243 | logrus.Error(err) 244 | } 245 | 246 | // rotates the certificate if there are certificates going to expire 247 | // and the lock can be acquired. 248 | // if the lock cannot be acquired, it will wait `pollingPeriod` time 249 | // and try to acquire the lock again. 250 | if (len(configsToBeUpdate) > 0 || len(expiryCerts) > 0) && acquire(lock, &nodeMeta) { 251 | if !nodeMeta.Unschedulable { 252 | _ = host.Cordon(client, corev1Node) 253 | _ = host.Drain(client, corev1Node) 254 | } 255 | 256 | if len(configsToBeUpdate) > 0 { 257 | logrus.Infof("The configuration need to be updates are %v", configsToBeUpdate) 258 | 259 | logrus.Info("Waiting for configuration to be update") 260 | if err := certNode.UpdateConfig(configsToBeUpdate); err != nil { 261 | logrus.Error(err) 262 | } 263 | logrus.Info("Update configuration done") 264 | } 265 | 266 | if len(expiryCerts) > 0 { 267 | logrus.Infof("The expiry certificiates are %v", expiryCerts) 268 | 269 | logrus.Info("Waiting for certificate rotation") 270 | if err := certNode.Rotate(expiryCerts); err != nil { 271 | logrus.Error(err) 272 | } 273 | logrus.Info("Certificate rotation done") 274 | } 275 | 276 | if !nodeMeta.Unschedulable { 277 | _ = host.Uncordon(client, corev1Node) 278 | } 279 | 280 | release(lock) 281 | } 282 | } 283 | } 284 | } 285 | -------------------------------------------------------------------------------- /controllers/certificatesigningrequest_controller.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2020 The cert-manager authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package controllers 18 | 19 | import ( 20 | "context" 21 | "crypto/x509" 22 | "fmt" 23 | 24 | authorization "k8s.io/api/authorization/v1" 25 | capi "k8s.io/api/certificates/v1" 26 | capiv1beta1 "k8s.io/api/certificates/v1beta1" 27 | corev1 "k8s.io/api/core/v1" 28 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 29 | "k8s.io/apimachinery/pkg/runtime" 30 | "k8s.io/apimachinery/pkg/runtime/schema" 31 | k8sclient "k8s.io/client-go/kubernetes" 32 | "k8s.io/client-go/tools/record" 33 | ctrl "sigs.k8s.io/controller-runtime" 34 | "sigs.k8s.io/controller-runtime/pkg/client" 35 | 36 | "github.com/sirupsen/logrus" 37 | velerodiscovery "github.com/vmware-tanzu/velero/pkg/discovery" 38 | 39 | "github.com/jenting/kucero/pkg/pki/cert" 40 | "github.com/jenting/kucero/pkg/pki/signer" 41 | ) 42 | 43 | // CertificateSigningRequestSigningReconciler reconciles a CertificateSigningRequest object 44 | type CertificateSigningRequestSigningReconciler struct { 45 | Client client.Client 46 | ClientSet k8sclient.Interface 47 | Scheme *runtime.Scheme 48 | Signer *signer.Signer 49 | EventRecorder record.EventRecorder 50 | } 51 | 52 | // Tries to recognize CSRs that are specific to this use case 53 | type csrRecognizer struct { 54 | recognize func(csr *capi.CertificateSigningRequest, x509cr *x509.CertificateRequest) bool 55 | permission authorization.ResourceAttributes 56 | successMessage string 57 | } 58 | 59 | func recognizers() []csrRecognizer { 60 | recognizers := []csrRecognizer{ 61 | { 62 | recognize: isNodeServingCert, 63 | permission: authorization.ResourceAttributes{Group: "certificates.k8s.io", Resource: "certificatesigningrequests", Verb: "create"}, 64 | successMessage: "Auto approving kubelet serving certificate after SubjectAccessReview.", 65 | }, 66 | } 67 | return recognizers 68 | } 69 | 70 | // +kubebuilder:rbac:groups=certificates.k8s.io,resources=certificatesigningrequests,verbs=get;list;watch 71 | // +kubebuilder:rbac:groups=certificates.k8s.io,resources=certificatesigningrequests/status,verbs=patch 72 | // +kubebuilder:rbac:groups=core,resources=events,verbs=create;patch 73 | 74 | func (r *CertificateSigningRequestSigningReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { 75 | var csr capi.CertificateSigningRequest 76 | if err := r.Client.Get(ctx, req.NamespacedName, &csr); client.IgnoreNotFound(err) != nil { 77 | return ctrl.Result{}, fmt.Errorf("error %q getting CSR", err) 78 | } 79 | switch { 80 | case !csr.DeletionTimestamp.IsZero(): 81 | logrus.Info("CSR has been deleted. Ignoring") 82 | case csr.Status.Certificate != nil: 83 | logrus.Info("CSR has already been signed. Ignoring") 84 | default: 85 | logrus.Info("Signing") 86 | x509cr, err := cert.ParseCSR(csr.Spec.Request) 87 | if err != nil { 88 | logrus.Errorf("Unable to parse csr: %v", err) 89 | r.EventRecorder.Event(&csr, corev1.EventTypeWarning, "SigningFailed", "Unable to parse the CSR request") 90 | return ctrl.Result{}, nil 91 | } 92 | 93 | tried := []string{} 94 | for _, recognizer := range recognizers() { 95 | tried = append(tried, recognizer.permission.Resource) 96 | 97 | if !recognizer.recognize(&csr, x509cr) { 98 | continue 99 | } 100 | 101 | approved, err := r.authorize(&csr, recognizer.permission) 102 | if err != nil { 103 | logrus.Errorf("SubjectAccessReview failed: %v", err) 104 | return ctrl.Result{}, fmt.Errorf("error SubjectAccessReview: %v", err) 105 | } 106 | 107 | if approved { 108 | logrus.Infof("CSR: %v", csr.ObjectMeta.Name) 109 | logrus.Infof("X509v3 SAN DNS: %v", x509cr.DNSNames) 110 | logrus.Infof("X509v3 SAN IP: %v", x509cr.IPAddresses) 111 | logrus.Info("Approving csr") 112 | 113 | // sign the csr before approve 114 | // otherwise, the kube-controller-manager will sign the csr 115 | cert, err := r.Signer.Sign(x509cr, csr.Spec) 116 | if err != nil { 117 | return ctrl.Result{}, fmt.Errorf("error auto signing csr: %v", err) 118 | } 119 | patch := client.MergeFrom(csr.DeepCopy()) 120 | csr.Status.Certificate = cert 121 | if err := r.Client.Status().Patch(ctx, &csr, patch); err != nil { 122 | return ctrl.Result{}, fmt.Errorf("error patching CSR: %v", err) 123 | } 124 | 125 | // approve the csr 126 | appendApprovalCondition(&csr, recognizer.successMessage) 127 | _, err = r.ClientSet.CertificatesV1().CertificateSigningRequests().UpdateApproval(context.TODO(), csr.Name, &csr, metav1.UpdateOptions{}) 128 | if err != nil { 129 | return ctrl.Result{}, fmt.Errorf("error updating approval for csr: %v", err) 130 | } 131 | 132 | r.EventRecorder.Event(&csr, corev1.EventTypeNormal, "Signed", "The CSR has been signed") 133 | } else { 134 | return ctrl.Result{}, fmt.Errorf("SubjectAccessReview failed") 135 | } 136 | } 137 | } 138 | return ctrl.Result{}, nil 139 | } 140 | 141 | // Validate that the given node has authorization to actualy create CSRs 142 | func (r *CertificateSigningRequestSigningReconciler) authorize(csr *capi.CertificateSigningRequest, rattrs authorization.ResourceAttributes) (bool, error) { 143 | extra := make(map[string]authorization.ExtraValue) 144 | for k, v := range csr.Spec.Extra { 145 | extra[k] = authorization.ExtraValue(v) 146 | } 147 | 148 | sar := &authorization.SubjectAccessReview{ 149 | Spec: authorization.SubjectAccessReviewSpec{ 150 | User: csr.Spec.Username, 151 | UID: csr.Spec.UID, 152 | Groups: csr.Spec.Groups, 153 | Extra: extra, 154 | ResourceAttributes: &rattrs, 155 | }, 156 | } 157 | sar, err := r.ClientSet.AuthorizationV1().SubjectAccessReviews().Create(context.TODO(), sar, metav1.CreateOptions{}) 158 | if err != nil { 159 | return false, err 160 | } 161 | return sar.Status.Allowed, nil 162 | } 163 | 164 | func appendApprovalCondition(csr *capi.CertificateSigningRequest, message string) { 165 | csr.Status.Conditions = append(csr.Status.Conditions, capi.CertificateSigningRequestCondition{ 166 | Type: capi.CertificateApproved, 167 | Status: corev1.ConditionTrue, 168 | Reason: "AutoApproved by kucero", 169 | Message: message, 170 | LastUpdateTime: metav1.Now(), 171 | }) 172 | } 173 | 174 | func (r *CertificateSigningRequestSigningReconciler) SetupWithManager(mgr ctrl.Manager) error { 175 | discoveryHelper, err := velerodiscovery.NewHelper(r.ClientSet.Discovery(), &logrus.Logger{}) 176 | if err != nil { 177 | return err 178 | } 179 | gvr, _, err := discoveryHelper.ResourceFor(schema.GroupVersionResource{ 180 | Group: "certificates.k8s.io", 181 | Resource: "CertificateSigningRequest", 182 | }) 183 | if err != nil { 184 | return err 185 | } 186 | 187 | switch gvr.Version { 188 | case "v1beta1": 189 | return ctrl.NewControllerManagedBy(mgr). 190 | For(&capiv1beta1.CertificateSigningRequest{}). 191 | Complete(r) 192 | case "v1": 193 | return ctrl.NewControllerManagedBy(mgr). 194 | For(&capi.CertificateSigningRequest{}). 195 | Complete(r) 196 | default: 197 | return fmt.Errorf("unsupported certificates.k8s.io/%s", gvr.Version) 198 | } 199 | } 200 | -------------------------------------------------------------------------------- /controllers/helper.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2020 The cert-manager authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package controllers 18 | 19 | import ( 20 | "crypto/x509" 21 | "reflect" 22 | "strings" 23 | 24 | capi "k8s.io/api/certificates/v1" 25 | 26 | "github.com/sirupsen/logrus" 27 | ) 28 | 29 | func hasExactUsages(csr *capi.CertificateSigningRequest, usages []capi.KeyUsage) bool { 30 | if len(usages) != len(csr.Spec.Usages) { 31 | return false 32 | } 33 | 34 | usageMap := map[capi.KeyUsage]struct{}{} 35 | for _, u := range usages { 36 | usageMap[u] = struct{}{} 37 | } 38 | 39 | for _, u := range csr.Spec.Usages { 40 | if _, ok := usageMap[u]; !ok { 41 | return false 42 | } 43 | } 44 | 45 | return true 46 | } 47 | 48 | var kubeletServerUsages = []capi.KeyUsage{ 49 | capi.UsageKeyEncipherment, 50 | capi.UsageDigitalSignature, 51 | capi.UsageServerAuth, 52 | } 53 | 54 | func isNodeServingCert(csr *capi.CertificateSigningRequest, x509cr *x509.CertificateRequest) bool { 55 | if !reflect.DeepEqual([]string{"system:nodes"}, x509cr.Subject.Organization) { 56 | logrus.Warningf("Org does not match: %s", x509cr.Subject.Organization) 57 | return false 58 | } 59 | if (len(x509cr.DNSNames) < 1) || (len(x509cr.IPAddresses) < 1) { 60 | return false 61 | } 62 | if !hasExactUsages(csr, kubeletServerUsages) { 63 | logrus.Info("Usage does not match") 64 | return false 65 | } 66 | if !strings.HasPrefix(x509cr.Subject.CommonName, "system:node:") { 67 | logrus.Warningf("CN does not start with 'system:node': %s", x509cr.Subject.CommonName) 68 | return false 69 | } 70 | if csr.Spec.Username != x509cr.Subject.CommonName { 71 | logrus.Warningf("X509 CN %q doesn't match CSR username %q", x509cr.Subject.CommonName, csr.Spec.Username) 72 | return false 73 | } 74 | return true 75 | } 76 | -------------------------------------------------------------------------------- /controllers/helper_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2020 The cert-manager authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package controllers 18 | 19 | import ( 20 | "crypto/x509" 21 | "crypto/x509/pkix" 22 | "net" 23 | "testing" 24 | 25 | capi "k8s.io/api/certificates/v1" 26 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 27 | ) 28 | 29 | func TestNodeServingCert_org(t *testing.T) { 30 | var csr = capi.CertificateSigningRequest{ 31 | ObjectMeta: metav1.ObjectMeta{Name: "test-csr"}, 32 | Spec: capi.CertificateSigningRequestSpec{ 33 | Usages: kubeletServerUsages, 34 | }, 35 | } 36 | 37 | var x509cr = x509.CertificateRequest{ 38 | Subject: pkix.Name{ 39 | Organization: []string{"foobar"}, 40 | CommonName: "system:node:node-01", 41 | }, 42 | DNSNames: []string{"foobar"}, 43 | IPAddresses: []net.IP{net.ParseIP("1.2.3.4")}, 44 | } 45 | 46 | v := isNodeServingCert(&csr, &x509cr) 47 | 48 | if v != false { 49 | t.Error("Only 'system:nodes' accepted as org") 50 | } 51 | } 52 | 53 | func TestNodeServingCert_NoDNSOrIP(t *testing.T) { 54 | var csr = capi.CertificateSigningRequest{ 55 | ObjectMeta: metav1.ObjectMeta{Name: "test-csr"}, 56 | Spec: capi.CertificateSigningRequestSpec{ 57 | Usages: kubeletServerUsages, 58 | }, 59 | } 60 | 61 | var x509cr = x509.CertificateRequest{ 62 | Subject: pkix.Name{ 63 | Organization: []string{"foobar"}, 64 | CommonName: "system:node:node-01", 65 | }, 66 | } 67 | 68 | v := isNodeServingCert(&csr, &x509cr) 69 | 70 | if v != false { 71 | t.Error("Need at least one DNS name and IPAddress") 72 | } 73 | } 74 | 75 | func TestNodeServingCert_OnlyIP(t *testing.T) { 76 | var csr = capi.CertificateSigningRequest{ 77 | ObjectMeta: metav1.ObjectMeta{Name: "test-csr"}, 78 | Spec: capi.CertificateSigningRequestSpec{ 79 | Usages: kubeletServerUsages, 80 | }, 81 | } 82 | 83 | var x509cr = x509.CertificateRequest{ 84 | Subject: pkix.Name{ 85 | Organization: []string{"foobar"}, 86 | CommonName: "system:node:node-01", 87 | }, 88 | IPAddresses: []net.IP{net.ParseIP("1.2.3.4")}, 89 | } 90 | 91 | v := isNodeServingCert(&csr, &x509cr) 92 | 93 | if v != false { 94 | t.Error("Need at least one DNS name and IPAddress") 95 | } 96 | } 97 | 98 | func TestNodeServingCert_OnlyDNS(t *testing.T) { 99 | var csr = capi.CertificateSigningRequest{ 100 | ObjectMeta: metav1.ObjectMeta{Name: "test-csr"}, 101 | Spec: capi.CertificateSigningRequestSpec{ 102 | Usages: kubeletServerUsages, 103 | }, 104 | } 105 | 106 | var x509cr = x509.CertificateRequest{ 107 | Subject: pkix.Name{ 108 | Organization: []string{"foobar"}, 109 | CommonName: "system:node:node-01", 110 | }, 111 | DNSNames: []string{"foobar"}, 112 | } 113 | 114 | v := isNodeServingCert(&csr, &x509cr) 115 | 116 | if v != false { 117 | t.Error("Need at least one DNS name and IPAddress") 118 | } 119 | } 120 | 121 | func TestNodeServingCert_unmatchingUSages(t *testing.T) { 122 | var csr = capi.CertificateSigningRequest{ 123 | ObjectMeta: metav1.ObjectMeta{Name: "test-csr"}, 124 | Spec: capi.CertificateSigningRequestSpec{ 125 | Usages: []capi.KeyUsage{"foo", "bar"}, 126 | }, 127 | } 128 | 129 | var x509cr = x509.CertificateRequest{ 130 | Subject: pkix.Name{ 131 | Organization: []string{"foobar"}, 132 | CommonName: "system:node:node-01", 133 | }, 134 | DNSNames: []string{"foobar"}, 135 | } 136 | 137 | v := isNodeServingCert(&csr, &x509cr) 138 | 139 | if v != false { 140 | t.Errorf("Usages need to match %v\n", kubeletServerUsages) 141 | } 142 | } 143 | 144 | func TestNodeServingCert_unmatchingCN(t *testing.T) { 145 | var csr = capi.CertificateSigningRequest{ 146 | ObjectMeta: metav1.ObjectMeta{Name: "test-csr"}, 147 | Spec: capi.CertificateSigningRequestSpec{ 148 | Usages: []capi.KeyUsage{"foo", "bar"}, 149 | }, 150 | } 151 | 152 | var x509cr = x509.CertificateRequest{ 153 | Subject: pkix.Name{ 154 | Organization: []string{"foobar"}, 155 | CommonName: "system:foo:node-01", 156 | }, 157 | DNSNames: []string{"foobar"}, 158 | } 159 | 160 | v := isNodeServingCert(&csr, &x509cr) 161 | 162 | if v != false { 163 | t.Errorf("CN need to match 'system:node:*'") 164 | } 165 | } 166 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/jenting/kucero 2 | 3 | go 1.24.0 4 | 5 | toolchain go1.24.1 6 | 7 | require ( 8 | github.com/sirupsen/logrus v1.9.3 9 | github.com/spf13/cobra v1.9.1 10 | github.com/vmware-tanzu/velero v1.15.2 11 | github.com/weaveworks/kured v0.0.0-20220810042013-9d4ebfc1f82a 12 | k8s.io/api v0.33.1 13 | k8s.io/apimachinery v0.33.1 14 | k8s.io/apiserver v0.33.1 15 | k8s.io/client-go v0.33.1 16 | k8s.io/kubectl v0.32.0 17 | sigs.k8s.io/controller-runtime v0.20.4 18 | sigs.k8s.io/yaml v1.4.0 19 | ) 20 | 21 | require ( 22 | github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161 // indirect 23 | github.com/MakeNowJust/heredoc v1.0.0 // indirect 24 | github.com/beorn7/perks v1.0.1 // indirect 25 | github.com/blang/semver/v4 v4.0.0 // indirect 26 | github.com/cespare/xxhash/v2 v2.3.0 // indirect 27 | github.com/chai2010/gettext-go v1.0.3 // indirect 28 | github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect 29 | github.com/emicklei/go-restful/v3 v3.12.1 // indirect 30 | github.com/evanphx/json-patch/v5 v5.9.11 // indirect 31 | github.com/exponent-io/jsonpath v0.0.0-20210407135951-1de76d718b3f // indirect 32 | github.com/fsnotify/fsnotify v1.8.0 // indirect 33 | github.com/fxamacker/cbor/v2 v2.7.0 // indirect 34 | github.com/go-errors/errors v1.5.1 // indirect 35 | github.com/go-logr/logr v1.4.2 // indirect 36 | github.com/go-openapi/jsonpointer v0.21.0 // indirect 37 | github.com/go-openapi/jsonreference v0.21.0 // indirect 38 | github.com/go-openapi/swag v0.23.0 // indirect 39 | github.com/gogo/protobuf v1.3.2 // indirect 40 | github.com/google/btree v1.1.3 // indirect 41 | github.com/google/gnostic-models v0.6.9 // indirect 42 | github.com/google/go-cmp v0.7.0 // indirect 43 | github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect 44 | github.com/google/uuid v1.6.0 // indirect 45 | github.com/gorilla/websocket v1.5.4-0.20250319132907-e064f32e3674 // indirect 46 | github.com/gregjones/httpcache v0.0.0-20190611155906-901d90724c79 // indirect 47 | github.com/inconshreveable/mousetrap v1.1.0 // indirect 48 | github.com/josharian/intern v1.0.0 // indirect 49 | github.com/json-iterator/go v1.1.12 // indirect 50 | github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de // indirect 51 | github.com/mailru/easyjson v0.9.0 // indirect 52 | github.com/mitchellh/go-wordwrap v1.0.1 // indirect 53 | github.com/moby/spdystream v0.5.0 // indirect 54 | github.com/moby/term v0.5.0 // indirect 55 | github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect 56 | github.com/modern-go/reflect2 v1.0.2 // indirect 57 | github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00 // indirect 58 | github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect 59 | github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f // indirect 60 | github.com/peterbourgon/diskv v2.0.1+incompatible // indirect 61 | github.com/pkg/errors v0.9.1 // indirect 62 | github.com/prometheus/client_golang v1.22.0 // indirect 63 | github.com/prometheus/client_model v0.6.1 // indirect 64 | github.com/prometheus/common v0.62.0 // indirect 65 | github.com/prometheus/procfs v0.15.1 // indirect 66 | github.com/russross/blackfriday/v2 v2.1.0 // indirect 67 | github.com/spf13/pflag v1.0.6 // indirect 68 | github.com/x448/float16 v0.8.4 // indirect 69 | github.com/xlab/treeprint v1.2.0 // indirect 70 | golang.org/x/net v0.38.0 // indirect 71 | golang.org/x/oauth2 v0.27.0 // indirect 72 | golang.org/x/sync v0.12.0 // indirect 73 | golang.org/x/sys v0.31.0 // indirect 74 | golang.org/x/term v0.30.0 // indirect 75 | golang.org/x/text v0.23.0 // indirect 76 | golang.org/x/time v0.9.0 // indirect 77 | golang.org/x/tools v0.28.0 // indirect 78 | gomodules.xyz/jsonpatch/v2 v2.4.0 // indirect 79 | google.golang.org/protobuf v1.36.5 // indirect 80 | gopkg.in/evanphx/json-patch.v4 v4.12.0 // indirect 81 | gopkg.in/inf.v0 v0.9.1 // indirect 82 | gopkg.in/yaml.v3 v3.0.1 // indirect 83 | k8s.io/apiextensions-apiserver v0.32.1 // indirect 84 | k8s.io/cli-runtime v0.32.0 // indirect 85 | k8s.io/component-base v0.33.1 // indirect 86 | k8s.io/klog/v2 v2.130.1 // indirect 87 | k8s.io/kube-openapi v0.0.0-20250318190949-c8a335a9a2ff // indirect 88 | k8s.io/utils v0.0.0-20241210054802-24370beab758 // indirect 89 | sigs.k8s.io/json v0.0.0-20241014173422-cfa47c3a1cc8 // indirect 90 | sigs.k8s.io/kustomize/api v0.18.0 // indirect 91 | sigs.k8s.io/kustomize/kyaml v0.18.1 // indirect 92 | sigs.k8s.io/randfill v1.0.0 // indirect 93 | sigs.k8s.io/structured-merge-diff/v4 v4.6.0 // indirect 94 | ) 95 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161 h1:L/gRVlceqvL25UVaW/CKtUDjefjrs0SPonmDGUVOYP0= 2 | github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= 3 | github.com/MakeNowJust/heredoc v1.0.0 h1:cXCdzVdstXyiTqTvfqk9SDHpKNjxuom+DOlyEeQ4pzQ= 4 | github.com/MakeNowJust/heredoc v1.0.0/go.mod h1:mG5amYoWBHf8vpLOuehzbGGw0EHxpZZ6lCpQ4fNJ8LE= 5 | github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio= 6 | github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= 7 | github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= 8 | github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= 9 | github.com/blang/semver/v4 v4.0.0 h1:1PFHFE6yCCTv8C1TeyNNarDzntLi7wMI5i/pzqYIsAM= 10 | github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ= 11 | github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= 12 | github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= 13 | github.com/chai2010/gettext-go v1.0.3 h1:9liNh8t+u26xl5ddmWLmsOsdNLwkdRTg5AG+JnTiM80= 14 | github.com/chai2010/gettext-go v1.0.3/go.mod h1:y+wnP2cHYaVj19NZhYKAwEMH2CI1gNHeQQ+5AjwawxA= 15 | github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= 16 | github.com/creack/pty v1.1.18 h1:n56/Zwd5o6whRC5PMGretI4IdRLlmBXYNjScPaBgsbY= 17 | github.com/creack/pty v1.1.18/go.mod h1:MOBLtS5ELjhRRrroQr9kyvTxUAFNvYEK993ew/Vr4O4= 18 | github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 19 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 20 | github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= 21 | github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 22 | github.com/emicklei/go-restful/v3 v3.12.1 h1:PJMDIM/ak7btuL8Ex0iYET9hxM3CI2sjZtzpL63nKAU= 23 | github.com/emicklei/go-restful/v3 v3.12.1/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= 24 | github.com/evanphx/json-patch v5.6.0+incompatible h1:jBYDEEiFBPxA0v50tFdvOzQQTCvpL6mnFh5mB2/l16U= 25 | github.com/evanphx/json-patch v5.6.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= 26 | github.com/evanphx/json-patch/v5 v5.9.11 h1:/8HVnzMq13/3x9TPvjG08wUGqBTmZBsCWzjTM0wiaDU= 27 | github.com/evanphx/json-patch/v5 v5.9.11/go.mod h1:3j+LviiESTElxA4p3EMKAB9HXj3/XEtnUf6OZxqIQTM= 28 | github.com/exponent-io/jsonpath v0.0.0-20210407135951-1de76d718b3f h1:Wl78ApPPB2Wvf/TIe2xdyJxTlb6obmF18d8QdkxNDu4= 29 | github.com/exponent-io/jsonpath v0.0.0-20210407135951-1de76d718b3f/go.mod h1:OSYXu++VVOHnXeitef/D8n/6y4QV8uLHSFXX4NeXMGc= 30 | github.com/fsnotify/fsnotify v1.8.0 h1:dAwr6QBTBZIkG8roQaJjGof0pp0EeF+tNV7YBP3F/8M= 31 | github.com/fsnotify/fsnotify v1.8.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0= 32 | github.com/fxamacker/cbor/v2 v2.7.0 h1:iM5WgngdRBanHcxugY4JySA0nk1wZorNOpTgCMedv5E= 33 | github.com/fxamacker/cbor/v2 v2.7.0/go.mod h1:pxXPTn3joSm21Gbwsv0w9OSA2y1HFR9qXEeXQVeNoDQ= 34 | github.com/go-errors/errors v1.5.1 h1:ZwEMSLRCapFLflTpT7NKaAc7ukJ8ZPEjzlxt8rPN8bk= 35 | github.com/go-errors/errors v1.5.1/go.mod h1:sIVyrIiJhuEF+Pj9Ebtd6P/rEYROXFi3BopGUQ5a5Og= 36 | github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= 37 | github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= 38 | github.com/go-logr/zapr v1.3.0 h1:XGdV8XW8zdwFiwOA2Dryh1gj2KRQyOOoNmBy4EplIcQ= 39 | github.com/go-logr/zapr v1.3.0/go.mod h1:YKepepNBd1u/oyhd/yQmtjVXmm9uML4IXUgMOwR8/Gg= 40 | github.com/go-openapi/jsonpointer v0.21.0 h1:YgdVicSA9vH5RiHs9TZW5oyafXZFc6+2Vc1rr/O9oNQ= 41 | github.com/go-openapi/jsonpointer v0.21.0/go.mod h1:IUyH9l/+uyhIYQ/PXVA41Rexl+kOkAPDdXEYns6fzUY= 42 | github.com/go-openapi/jsonreference v0.21.0 h1:Rs+Y7hSXT83Jacb7kFyjn4ijOuVGSvOdF2+tg1TRrwQ= 43 | github.com/go-openapi/jsonreference v0.21.0/go.mod h1:LmZmgsrTkVg9LG4EaHeY8cBDslNPMo06cago5JNLkm4= 44 | github.com/go-openapi/swag v0.23.0 h1:vsEVJDUo2hPJ2tu0/Xc+4noaxyEffXNIs3cOULZ+GrE= 45 | github.com/go-openapi/swag v0.23.0/go.mod h1:esZ8ITTYEsH1V2trKHjAN8Ai7xHb8RV+YSZ577vPjgQ= 46 | github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1vB6EwHI= 47 | github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8= 48 | github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= 49 | github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= 50 | github.com/google/btree v1.1.3 h1:CVpQJjYgC4VbzxeGVHfvZrv1ctoYCAI8vbl07Fcxlyg= 51 | github.com/google/btree v1.1.3/go.mod h1:qOPhT0dTNdNzV6Z/lhRX0YXUafgPLFUh+gZMl761Gm4= 52 | github.com/google/gnostic-models v0.6.9 h1:MU/8wDLif2qCXZmzncUQ/BOfxWfthHi63KqpoNbWqVw= 53 | github.com/google/gnostic-models v0.6.9/go.mod h1:CiWsm0s6BSQd1hRn8/QmxqB6BesYcbSZxsz9b0KuDBw= 54 | github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= 55 | github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= 56 | github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= 57 | github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= 58 | github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= 59 | github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= 60 | github.com/google/pprof v0.0.0-20241029153458-d1b30febd7db h1:097atOisP2aRj7vFgYQBbFN4U4JNXUNYpxael3UzMyo= 61 | github.com/google/pprof v0.0.0-20241029153458-d1b30febd7db/go.mod h1:vavhavw2zAxS5dIdcRluK6cSGGPlZynqzFM8NdvU144= 62 | github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 h1:El6M4kTTCOh6aBiKaUGG7oYTSPP8MxqL4YI3kZKwcP4= 63 | github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510/go.mod h1:pupxD2MaaD3pAXIBCelhxNneeOaAeabZDe5s4K6zSpQ= 64 | github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= 65 | github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= 66 | github.com/gorilla/websocket v1.5.4-0.20250319132907-e064f32e3674 h1:JeSE6pjso5THxAzdVpqr6/geYxZytqFMBCOtn/ujyeo= 67 | github.com/gorilla/websocket v1.5.4-0.20250319132907-e064f32e3674/go.mod h1:r4w70xmWCQKmi1ONH4KIaBptdivuRPyosB9RmPlGEwA= 68 | github.com/gregjones/httpcache v0.0.0-20190611155906-901d90724c79 h1:+ngKgrYPPJrOjhax5N+uePQ0Fh1Z7PheYoUI/0nzkPA= 69 | github.com/gregjones/httpcache v0.0.0-20190611155906-901d90724c79/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= 70 | github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= 71 | github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= 72 | github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= 73 | github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= 74 | github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= 75 | github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= 76 | github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= 77 | github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= 78 | github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo= 79 | github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ= 80 | github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= 81 | github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= 82 | github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= 83 | github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= 84 | github.com/kubernetes-csi/external-snapshotter/client/v7 v7.0.0 h1:j3YK74myEQRxR/srciTpOrm221SAvz6J5OVWbyfeXFo= 85 | github.com/kubernetes-csi/external-snapshotter/client/v7 v7.0.0/go.mod h1:FlyYFe32mPxKEPaRXKNxfX576d1AoCzstYDoOOnyMA4= 86 | github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= 87 | github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= 88 | github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de h1:9TO3cAIGXtEhnIaL+V+BEER86oLrvS+kWobKpbJuye0= 89 | github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de/go.mod h1:zAbeS9B/r2mtpb6U+EI2rYA5OAXxsYw6wTamcNW+zcE= 90 | github.com/mailru/easyjson v0.9.0 h1:PrnmzHw7262yW8sTBwxi1PdJA3Iw/EKBa8psRf7d9a4= 91 | github.com/mailru/easyjson v0.9.0/go.mod h1:1+xMtQp2MRNVL/V1bOzuP3aP8VNwRW55fQUto+XFtTU= 92 | github.com/mitchellh/go-wordwrap v1.0.1 h1:TLuKupo69TCn6TQSyGxwI1EblZZEsQ0vMlAFQflz0v0= 93 | github.com/mitchellh/go-wordwrap v1.0.1/go.mod h1:R62XHJLzvMFRBbcrT7m7WgmE1eOyTSsCt+hzestvNj0= 94 | github.com/moby/spdystream v0.5.0 h1:7r0J1Si3QO/kjRitvSLVVFUjxMEb/YLj6S9FF62JBCU= 95 | github.com/moby/spdystream v0.5.0/go.mod h1:xBAYlnt/ay+11ShkdFKNAG7LsyK/tmNBVvVOwrfMgdI= 96 | github.com/moby/term v0.5.0 h1:xt8Q1nalod/v7BqbG21f8mQPqH+xAaC9C3N3wfWbVP0= 97 | github.com/moby/term v0.5.0/go.mod h1:8FzsFHVUBGZdbDsJw/ot+X+d5HLUbvklYLJ9uGfcI3Y= 98 | github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= 99 | github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= 100 | github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= 101 | github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= 102 | github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= 103 | github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00 h1:n6/2gBQ3RWajuToeY6ZtZTIKv2v7ThUy5KKusIT0yc0= 104 | github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00/go.mod h1:Pm3mSP3c5uWn86xMLZ5Sa7JB9GsEZySvHYXCTK4E9q4= 105 | github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= 106 | github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= 107 | github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f h1:y5//uYreIhSUg3J1GEMiLbxo1LJaP8RfCpH6pymGZus= 108 | github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw= 109 | github.com/onsi/ginkgo/v2 v2.22.0 h1:Yed107/8DjTr0lKCNt7Dn8yQ6ybuDRQoMGrNFKzMfHg= 110 | github.com/onsi/ginkgo/v2 v2.22.0/go.mod h1:7Du3c42kxCUegi0IImZ1wUQzMBVecgIHjR1C+NkhLQo= 111 | github.com/onsi/gomega v1.36.1 h1:bJDPBO7ibjxcbHMgSCoo4Yj18UWbKDlLwX1x9sybDcw= 112 | github.com/onsi/gomega v1.36.1/go.mod h1:PvZbdDc8J6XJEpDK4HCuRBm8a6Fzp9/DmhC9C7yFlog= 113 | github.com/peterbourgon/diskv v2.0.1+incompatible h1:UBdAOUP5p4RWqPBg048CAvpKN+vxiaj6gdUUzhl4XmI= 114 | github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= 115 | github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= 116 | github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= 117 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 118 | github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= 119 | github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 120 | github.com/prometheus/client_golang v1.22.0 h1:rb93p9lokFEsctTys46VnV1kLCDpVZ0a/Y92Vm0Zc6Q= 121 | github.com/prometheus/client_golang v1.22.0/go.mod h1:R7ljNsLXhuQXYZYtw6GAE9AZg8Y7vEW5scdCXrWRXC0= 122 | github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p8ais2e9E= 123 | github.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQyHDk9E3jmP2AmGiY= 124 | github.com/prometheus/common v0.62.0 h1:xasJaQlnWAeyHdUBeGjXmutelfJHWMRr+Fg4QszZ2Io= 125 | github.com/prometheus/common v0.62.0/go.mod h1:vyBcEuLSvWos9B1+CyL7JZ2up+uFzXhkqml0W5zIY1I= 126 | github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc= 127 | github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk= 128 | github.com/rogpeppe/go-internal v1.13.1 h1:KvO1DLK/DRN07sQ1LQKScxyZJuNnedQ5/wKSR38lUII= 129 | github.com/rogpeppe/go-internal v1.13.1/go.mod h1:uMEvuHeurkdAXX61udpOXGD/AzZDWNMNyH2VO9fmH0o= 130 | github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= 131 | github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= 132 | github.com/sergi/go-diff v1.2.0 h1:XU+rvMAioB0UC3q1MFrIQy4Vo5/4VsRDQQXHsEya6xQ= 133 | github.com/sergi/go-diff v1.2.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= 134 | github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= 135 | github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= 136 | github.com/spf13/afero v1.8.2 h1:xehSyVa0YnHWsJ49JFljMpg1HX19V6NDZ1fkm1Xznbo= 137 | github.com/spf13/afero v1.8.2/go.mod h1:CtAatgMJh6bJEIs48Ay/FOnkljP3WeGUG0MC1RfAqwo= 138 | github.com/spf13/cobra v1.9.1 h1:CXSaggrXdbHK9CF+8ywj8Amf7PBRmPCOJugH954Nnlo= 139 | github.com/spf13/cobra v1.9.1/go.mod h1:nDyEzZ8ogv936Cinf6g1RU9MRY64Ir93oCnqb9wxYW0= 140 | github.com/spf13/pflag v1.0.6 h1:jFzHGLGAlb3ruxLB8MhbI6A8+AQX/2eW4qeyNZXNp2o= 141 | github.com/spf13/pflag v1.0.6/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= 142 | github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 143 | github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= 144 | github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= 145 | github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= 146 | github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= 147 | github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= 148 | github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= 149 | github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= 150 | github.com/vmware-tanzu/velero v1.15.2 h1:zB4nRgknByjFasZLb7XHU/OsQ+1fs9iaIL3yhzuIeMQ= 151 | github.com/vmware-tanzu/velero v1.15.2/go.mod h1:bZbnBC9OcwXfsovU0uCHwPlbm3ba8N9fwvBkwnU2vls= 152 | github.com/weaveworks/kured v0.0.0-20220810042013-9d4ebfc1f82a h1:iHQFuASiFNyPMF0MYU9sW15wxEwNwxwxe8tDSP935wg= 153 | github.com/weaveworks/kured v0.0.0-20220810042013-9d4ebfc1f82a/go.mod h1:7GhHjYd1Tj6Jg9BQ5uUiB90aHxjW1JG6JvPB/YPyzWA= 154 | github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= 155 | github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= 156 | github.com/xlab/treeprint v1.2.0 h1:HzHnuAF1plUN2zGlAFHbSQP2qJ0ZAD3XF5XD7OesXRQ= 157 | github.com/xlab/treeprint v1.2.0/go.mod h1:gj5Gd3gPdKtR1ikdDK6fnFLdmIS0X30kTTuNd/WEJu0= 158 | github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= 159 | github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= 160 | go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= 161 | go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= 162 | go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= 163 | go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= 164 | go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= 165 | go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= 166 | golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= 167 | golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= 168 | golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= 169 | golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= 170 | golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= 171 | golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= 172 | golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 173 | golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 174 | golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= 175 | golang.org/x/net v0.38.0 h1:vRMAPTMaeGqVhG5QyLJHqNDwecKTomGeqbnfZyKlBI8= 176 | golang.org/x/net v0.38.0/go.mod h1:ivrbrMbzFq5J41QOQh0siUuly180yBYtLp+CKbEaFx8= 177 | golang.org/x/oauth2 v0.27.0 h1:da9Vo7/tDv5RH/7nZDz1eMGS/q1Vv1N/7FCrBhI9I3M= 178 | golang.org/x/oauth2 v0.27.0/go.mod h1:onh5ek6nERTohokkhCD/y2cV4Do3fxFHFuAejCkRWT8= 179 | golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 180 | golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 181 | golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 182 | golang.org/x/sync v0.12.0 h1:MHc5BpPuC30uJk597Ri8TV3CNZcTLu6B6z4lJy+g6Jw= 183 | golang.org/x/sync v0.12.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= 184 | golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 185 | golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 186 | golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 187 | golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 188 | golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 189 | golang.org/x/sys v0.31.0 h1:ioabZlmFYtWhL+TRYpcnNlLwhyxaM9kWTDEmfnprqik= 190 | golang.org/x/sys v0.31.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= 191 | golang.org/x/term v0.30.0 h1:PQ39fJZ+mfadBm0y5WlL4vlM7Sx1Hgf13sMIY2+QS9Y= 192 | golang.org/x/term v0.30.0/go.mod h1:NYYFdzHoI5wRh/h5tDMdMqCqPJZEuNqVR5xJLd/n67g= 193 | golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= 194 | golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= 195 | golang.org/x/text v0.23.0 h1:D71I7dUrlY+VX0gQShAThNGHFxZ13dGLBHQLVl1mJlY= 196 | golang.org/x/text v0.23.0/go.mod h1:/BLNzu4aZCJ1+kcD0DNRotWKage4q2rGVAg4o22unh4= 197 | golang.org/x/time v0.9.0 h1:EsRrnYcQiGH+5FfbgvV4AP7qEZstoyrHB0DzarOQ4ZY= 198 | golang.org/x/time v0.9.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= 199 | golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= 200 | golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= 201 | golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= 202 | golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= 203 | golang.org/x/tools v0.28.0 h1:WuB6qZ4RPCQo5aP3WdKZS7i595EdWqWR8vqJTlwTVK8= 204 | golang.org/x/tools v0.28.0/go.mod h1:dcIOrVd3mfQKTgrDVQHqCPMWy6lnhfhtX3hLXYVLfRw= 205 | golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 206 | golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 207 | golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 208 | golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 209 | gomodules.xyz/jsonpatch/v2 v2.4.0 h1:Ci3iUJyx9UeRx7CeFN8ARgGbkESwJK+KB9lLcWxY/Zw= 210 | gomodules.xyz/jsonpatch/v2 v2.4.0/go.mod h1:AH3dM2RI6uoBZxn3LVrfvJ3E0/9dG4cSrbuBJT4moAY= 211 | google.golang.org/protobuf v1.36.5 h1:tPhr+woSbjfYvY6/GPufUoYizxw1cF/yFoxJ2fmpwlM= 212 | google.golang.org/protobuf v1.36.5/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= 213 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 214 | gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= 215 | gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= 216 | gopkg.in/evanphx/json-patch.v4 v4.12.0 h1:n6jtcsulIzXPJaxegRbvFNNrZDjbij7ny3gmSPG+6V4= 217 | gopkg.in/evanphx/json-patch.v4 v4.12.0/go.mod h1:p8EYWUEYMpynmqDbY58zCKCFZw8pRWMG4EsWvDvM72M= 218 | gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= 219 | gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= 220 | gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 221 | gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= 222 | gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 223 | k8s.io/api v0.33.1 h1:tA6Cf3bHnLIrUK4IqEgb2v++/GYUtqiu9sRVk3iBXyw= 224 | k8s.io/api v0.33.1/go.mod h1:87esjTn9DRSRTD4fWMXamiXxJhpOIREjWOSjsW1kEHw= 225 | k8s.io/apiextensions-apiserver v0.32.1 h1:hjkALhRUeCariC8DiVmb5jj0VjIc1N0DREP32+6UXZw= 226 | k8s.io/apiextensions-apiserver v0.32.1/go.mod h1:sxWIGuGiYov7Io1fAS2X06NjMIk5CbRHc2StSmbaQto= 227 | k8s.io/apimachinery v0.33.1 h1:mzqXWV8tW9Rw4VeW9rEkqvnxj59k1ezDUl20tFK/oM4= 228 | k8s.io/apimachinery v0.33.1/go.mod h1:BHW0YOu7n22fFv/JkYOEfkUYNRN0fj0BlvMFWA7b+SM= 229 | k8s.io/apiserver v0.33.1 h1:yLgLUPDVC6tHbNcw5uE9mo1T6ELhJj7B0geifra3Qdo= 230 | k8s.io/apiserver v0.33.1/go.mod h1:VMbE4ArWYLO01omz+k8hFjAdYfc3GVAYPrhP2tTKccs= 231 | k8s.io/cli-runtime v0.32.0 h1:dP+OZqs7zHPpGQMCGAhectbHU2SNCuZtIimRKTv2T1c= 232 | k8s.io/cli-runtime v0.32.0/go.mod h1:Mai8ht2+esoDRK5hr861KRy6z0zHsSTYttNVJXgP3YQ= 233 | k8s.io/client-go v0.33.1 h1:ZZV/Ks2g92cyxWkRRnfUDsnhNn28eFpt26aGc8KbXF4= 234 | k8s.io/client-go v0.33.1/go.mod h1:JAsUrl1ArO7uRVFWfcj6kOomSlCv+JpvIsp6usAGefA= 235 | k8s.io/component-base v0.33.1 h1:EoJ0xA+wr77T+G8p6T3l4efT2oNwbqBVKR71E0tBIaI= 236 | k8s.io/component-base v0.33.1/go.mod h1:guT/w/6piyPfTgq7gfvgetyXMIh10zuXA6cRRm3rDuY= 237 | k8s.io/klog/v2 v2.130.1 h1:n9Xl7H1Xvksem4KFG4PYbdQCQxqc/tTUyrgXaOhHSzk= 238 | k8s.io/klog/v2 v2.130.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE= 239 | k8s.io/kube-openapi v0.0.0-20250318190949-c8a335a9a2ff h1:/usPimJzUKKu+m+TE36gUyGcf03XZEP0ZIKgKj35LS4= 240 | k8s.io/kube-openapi v0.0.0-20250318190949-c8a335a9a2ff/go.mod h1:5jIi+8yX4RIb8wk3XwBo5Pq2ccx4FP10ohkbSKCZoK8= 241 | k8s.io/kubectl v0.32.0 h1:rpxl+ng9qeG79YA4Em9tLSfX0G8W0vfaiPVrc/WR7Xw= 242 | k8s.io/kubectl v0.32.0/go.mod h1:qIjSX+QgPQUgdy8ps6eKsYNF+YmFOAO3WygfucIqFiE= 243 | k8s.io/utils v0.0.0-20241210054802-24370beab758 h1:sdbE21q2nlQtFh65saZY+rRM6x6aJJI8IUa1AmH/qa0= 244 | k8s.io/utils v0.0.0-20241210054802-24370beab758/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= 245 | sigs.k8s.io/controller-runtime v0.20.4 h1:X3c+Odnxz+iPTRobG4tp092+CvBU9UK0t/bRf+n0DGU= 246 | sigs.k8s.io/controller-runtime v0.20.4/go.mod h1:xg2XB0K5ShQzAgsoujxuKN4LNXR2LfwwHsPj7Iaw+XY= 247 | sigs.k8s.io/json v0.0.0-20241014173422-cfa47c3a1cc8 h1:gBQPwqORJ8d8/YNZWEjoZs7npUVDpVXUUOFfW6CgAqE= 248 | sigs.k8s.io/json v0.0.0-20241014173422-cfa47c3a1cc8/go.mod h1:mdzfpAEoE6DHQEN0uh9ZbOCuHbLK5wOm7dK4ctXE9Tg= 249 | sigs.k8s.io/kustomize/api v0.18.0 h1:hTzp67k+3NEVInwz5BHyzc9rGxIauoXferXyjv5lWPo= 250 | sigs.k8s.io/kustomize/api v0.18.0/go.mod h1:f8isXnX+8b+SGLHQ6yO4JG1rdkZlvhaCf/uZbLVMb0U= 251 | sigs.k8s.io/kustomize/kyaml v0.18.1 h1:WvBo56Wzw3fjS+7vBjN6TeivvpbW9GmRaWZ9CIVmt4E= 252 | sigs.k8s.io/kustomize/kyaml v0.18.1/go.mod h1:C3L2BFVU1jgcddNBE1TxuVLgS46TjObMwW5FT9FcjYo= 253 | sigs.k8s.io/randfill v0.0.0-20250304075658-069ef1bbf016/go.mod h1:XeLlZ/jmk4i1HRopwe7/aU3H5n1zNUcX6TM94b3QxOY= 254 | sigs.k8s.io/randfill v1.0.0 h1:JfjMILfT8A6RbawdsK2JXGBR5AQVfd+9TbzrlneTyrU= 255 | sigs.k8s.io/randfill v1.0.0/go.mod h1:XeLlZ/jmk4i1HRopwe7/aU3H5n1zNUcX6TM94b3QxOY= 256 | sigs.k8s.io/structured-merge-diff/v4 v4.6.0 h1:IUA9nvMmnKWcj5jl84xn+T5MnlZKThmUW1TdblaLVAc= 257 | sigs.k8s.io/structured-merge-diff/v4 v4.6.0/go.mod h1:dDy58f92j70zLsuZVuUX5Wp9vtxXpaZnkPGWeqDfCps= 258 | sigs.k8s.io/yaml v1.4.0 h1:Mk1wCc2gy/F0THH0TAp1QYyJNzRm2KCLy3o5ASXVI5E= 259 | sigs.k8s.io/yaml v1.4.0/go.mod h1:Ejl7/uTz7PSA4eKMyQCUTnhZYNmLIl+5c2lQPGR2BPY= 260 | -------------------------------------------------------------------------------- /integration/test.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -o pipefail 4 | set -e 5 | 6 | IMG=$1 7 | 8 | clean_up() { 9 | [ -f manifest/kustomization.yaml ] && rm manifest/kustomization.yaml 10 | } 11 | 12 | trap clean_up EXIT 13 | 14 | if ! command -v docker; then 15 | echo "docker could not be found" 16 | exit 1 17 | fi 18 | 19 | if ! command -v kind; then 20 | echo "kind could not be found" 21 | exit 1 22 | fi 23 | 24 | if ! command -v kustomize; then 25 | echo "kustomize could not be found" 26 | exit 1 27 | fi 28 | 29 | if ! command -v kubectl; then 30 | echo "kubectl could not be found" 31 | exit 1 32 | fi 33 | 34 | # Create KIND cluster 35 | COUNT=`kind get clusters | wc -l` 36 | if [ $COUNT -eq 0 ]; then 37 | kind create cluster 38 | fi 39 | 40 | # Find KIND cluster 41 | TAG=`docker ps | grep "kindest/node" | awk '{ print $1 }'` 42 | 43 | # Load docker image into KIND cluster 44 | kind load docker-image ${IMG} 45 | 46 | # Generate kustomization.yaml 47 | cd manifest 48 | kustomize create --autodetect || true 49 | cd - 50 | 51 | # Apply kustomize patch 52 | cat << EOF >> manifest/kustomization.yaml 53 | apiVersion: kustomize.config.k8s.io/v1beta1 54 | kind: Kustomization 55 | patchesStrategicMerge: 56 | - |- 57 | apiVersion: apps/v1 58 | kind: DaemonSet 59 | metadata: 60 | name: kucero 61 | namespace: kube-system 62 | spec: 63 | template: 64 | spec: 65 | containers: 66 | - name: kucero 67 | image: ${IMG} 68 | args: 69 | - --polling-period=1m 70 | - --renew-before=8761h 71 | - --enable-kubelet-csr-controller=true 72 | - --enable-kubelet-server-cert-rotation=false 73 | EOF 74 | kustomize build manifest | kubectl apply -f - 75 | 76 | APISERVER_ETCD_CLIENT_WAS=`docker exec -t ${TAG} openssl x509 -in /etc/kubernetes/pki/apiserver-etcd-client.crt -nocert -enddate | awk -F'=' '{print $2}'` 77 | APISERVER_KUBELET_CLIENT_WAS=`docker exec -t ${TAG} openssl x509 -in /etc/kubernetes/pki/apiserver-kubelet-client.crt -nocert -enddate | awk -F'=' '{print $2}'` 78 | APISERVER_WAS=`docker exec -t ${TAG} openssl x509 -in /etc/kubernetes/pki/apiserver.crt -nocert -enddate | awk -F'=' '{print $2}'` 79 | FRONT_PROXY_CLIENT_WAS=`docker exec -t ${TAG} openssl x509 -in /etc/kubernetes/pki/front-proxy-client.crt -nocert -enddate | awk -F'=' '{print $2}'` 80 | ETCD_HEALTHCHECK_CLIENT_WAS=`docker exec -t ${TAG} openssl x509 -in /etc/kubernetes/pki/etcd/healthcheck-client.crt -nocert -enddate | awk -F'=' '{print $2}'` 81 | ETCD_PEER_WAS=`docker exec -t ${TAG} openssl x509 -in /etc/kubernetes/pki/etcd/peer.crt -nocert -enddate | awk -F'=' '{print $2}'` 82 | ETCD_SERVER_WAS=`docker exec -t ${TAG} openssl x509 -in /etc/kubernetes/pki/etcd/server.crt -nocert -enddate | awk -F'=' '{print $2}'` 83 | 84 | kubectl wait pods --for=condition=ready -n kube-system --all --timeout=3m 85 | sleep 3m 86 | 87 | APISERVER_ETCD_CLIENT_IS=`docker exec -t ${TAG} openssl x509 -in /etc/kubernetes/pki/apiserver-etcd-client.crt -nocert -enddate | awk -F'=' '{print $2}'` 88 | APISERVER_KUBELET_CLIENT_IS=`docker exec -t ${TAG} openssl x509 -in /etc/kubernetes/pki/apiserver-kubelet-client.crt -nocert -enddate | awk -F'=' '{print $2}'` 89 | APISERVER_IS=`docker exec -t ${TAG} openssl x509 -in /etc/kubernetes/pki/apiserver.crt -nocert -enddate | awk -F'=' '{print $2}'` 90 | FRONT_PROXY_CLIENT_IS=`docker exec -t ${TAG} openssl x509 -in /etc/kubernetes/pki/front-proxy-client.crt -nocert -enddate | awk -F'=' '{print $2}'` 91 | ETCD_HEALTHCHECK_CLIENT_IS=`docker exec -t ${TAG} openssl x509 -in /etc/kubernetes/pki/etcd/healthcheck-client.crt -nocert -enddate | awk -F'=' '{print $2}'` 92 | ETCD_PEER_IS=`docker exec -t ${TAG} openssl x509 -in /etc/kubernetes/pki/etcd/peer.crt -nocert -enddate | awk -F'=' '{print $2}'` 93 | ETCD_SERVER_IS=`docker exec -t ${TAG} openssl x509 -in /etc/kubernetes/pki/etcd/server.crt -nocert -enddate | awk -F'=' '{print $2}'` 94 | 95 | if [ "${APISERVER_ETCD_CLIENT_WAS}" = "${APISERVER_ETCD_CLIENT_IS}" ]; then 96 | echo "ERROR: apiserver-etcd-client.crt not renewed" 97 | exit 1 98 | else 99 | echo "PASS: apiserver-etcd-client.crt renewed" 100 | fi 101 | if [ "${APISERVER_KUBELET_CLIENT_WAS}" = "${APISERVER_KUBELET_CLIENT_IS}" ]; then 102 | echo "ERROR: apiserver-kubelet-client.crt not renewed" 103 | exit 1 104 | else 105 | echo "PASS: apiserver-kubelet-client.crt renewed" 106 | fi 107 | if [ "${APISERVER_WAS}" = "${APISERVER_IS}" ]; then 108 | echo "ERROR: apiserver.crt not renewed" 109 | exit 1 110 | else 111 | echo "PASS: apiserver.crt renewed" 112 | fi 113 | if [ "${FRONT_PROXY_CLIENT_WAS}" = "${FRONT_PROXY_CLIENT_IS}" ]; then 114 | echo "ERROR: front-proxy-client.crt not renewed" 115 | exit 1 116 | else 117 | echo "PASS: front-proxy-client.crt renewed" 118 | fi 119 | if [ "${ETCD_HEALTHCHECK_CLIENT_WAS}" = "${ETCD_HEALTHCHECK_CLIENT_IS}" ]; then 120 | echo "ERROR: etcd/healthcheck-client.crt not renewed" 121 | exit 1 122 | else 123 | echo "PASS: etcd/healthcheck-client.crt renewed" 124 | fi 125 | if [ "${ETCD_PEER_WAS}" = "${ETCD_PEER_IS}" ]; then 126 | echo "ERROR: etcd/peer.crt not renewed" 127 | exit 1 128 | else 129 | echo "PASS: etcd/peer.crt renewed" 130 | fi 131 | if [ "${ETCD_SERVER_WAS}" = "${ETCD_SERVER_IS}" ]; then 132 | echo "ERROR: etcd/server.crt not renewed" 133 | exit 1 134 | else 135 | echo "PASS: etcd/server.crt renewed" 136 | fi 137 | -------------------------------------------------------------------------------- /logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SUSE/kucero/b2dee0107875eecf256606955794d6e12fec42d7/logo.png -------------------------------------------------------------------------------- /manifest/daemonset.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: apps/v1 2 | kind: DaemonSet 3 | metadata: 4 | name: kucero # Must match "--ds-name" 5 | namespace: kube-system # Must match "--ds-namespace" 6 | spec: 7 | selector: 8 | matchLabels: 9 | name: kucero 10 | revisionHistoryLimit: 3 11 | updateStrategy: 12 | type: RollingUpdate 13 | template: 14 | metadata: 15 | labels: 16 | name: kucero 17 | spec: 18 | serviceAccountName: kucero 19 | tolerations: 20 | - key: node-role.kubernetes.io/master 21 | operator: Exists 22 | effect: NoSchedule 23 | hostPID: true # Facilitate entering the host mount namespace via init 24 | restartPolicy: Always 25 | volumes: 26 | - name: kubelet-conf 27 | hostPath: 28 | path: /etc/kubernetes/kubelet.conf 29 | type: File 30 | - name: ca-crt 31 | hostPath: 32 | path: /etc/kubernetes/pki/ca.crt 33 | type: File 34 | - name: ca-key 35 | hostPath: 36 | path: /etc/kubernetes/pki/ca.key 37 | type: FileOrCreate 38 | - name: kubelet-config-yaml 39 | hostPath: 40 | path: /var/lib/kubelet/config.yaml 41 | type: File 42 | containers: 43 | - name: kucero 44 | image: jenting/kucero:v1.6.6 45 | imagePullPolicy: IfNotPresent 46 | securityContext: 47 | privileged: true # Give permission to nsenter /proc/1/ns/mnt 48 | env: 49 | # Pass in the name of the node on which this pod is scheduled 50 | # for use with drain/uncordon operations and lock acquisition 51 | - name: KUCERO_NODE_NAME 52 | valueFrom: 53 | fieldRef: 54 | fieldPath: spec.nodeName 55 | command: 56 | - /usr/bin/kucero 57 | volumeMounts: 58 | - mountPath: /etc/kubernetes/kubelet.conf 59 | name: kubelet-conf 60 | - mountPath: /etc/kubernetes/pki/ca.crt 61 | name: ca-crt 62 | readOnly: true 63 | - mountPath: /etc/kubernetes/pki/ca.key 64 | name: ca-key 65 | readOnly: true 66 | - mountPath: /var/lib/kubelet/config.yaml 67 | name: kubelet-config-yaml 68 | -------------------------------------------------------------------------------- /manifest/privileged.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: rbac.authorization.k8s.io/v1 2 | kind: ClusterRole 3 | metadata: 4 | name: kucero 5 | rules: 6 | # Allow kucero to read spec.unschedulable 7 | # Allow kubectl to drain/uncordon 8 | # 9 | # NB: These permissions are tightly coupled to the bundled version of kubectl; the ones below 10 | # match https://github.com/kubernetes/kubernetes/blob/v1.12.1/pkg/kubectl/cmd/drain.go 11 | # 12 | - apiGroups: [""] 13 | resources: ["nodes"] 14 | verbs: ["get", "patch"] 15 | - apiGroups: [""] 16 | resources: ["pods"] 17 | verbs: ["list", "delete", "get"] 18 | - apiGroups: ["apps"] 19 | resources: ["daemonsets"] 20 | verbs: ["get"] 21 | - apiGroups: [""] 22 | resources: ["pods/eviction"] 23 | verbs: ["create"] 24 | - apiGroups: ["certificates.k8s.io"] 25 | resources: ["signers"] 26 | resourceNames: ["kubernetes.io/kubelet-serving"] 27 | verbs: ["approve", "sign"] 28 | - apiGroups: ["certificates.k8s.io"] 29 | resources: ["certificatesigningrequests"] 30 | verbs: ["get", "list", "watch"] 31 | - apiGroups: ["certificates.k8s.io"] 32 | resources: ["certificatesigningrequests/approval"] 33 | verbs: ["create", "update"] 34 | - apiGroups: ["certificates.k8s.io"] 35 | resources: ["certificatesigningrequests/status"] 36 | verbs: ["patch"] 37 | - apiGroups: ["authorization.k8s.io"] 38 | resources: ["subjectaccessreviews"] 39 | verbs: ["create"] 40 | - apiGroups: ["coordination.k8s.io"] 41 | resources: ["leases"] 42 | verbs: ["create", "get", "update"] 43 | - apiGroups: [""] 44 | resources: ["events"] 45 | verbs: ["create", "patch"] 46 | --- 47 | apiVersion: rbac.authorization.k8s.io/v1 48 | kind: ClusterRoleBinding 49 | metadata: 50 | name: kucero 51 | roleRef: 52 | apiGroup: rbac.authorization.k8s.io 53 | kind: ClusterRole 54 | name: kucero 55 | subjects: 56 | - kind: ServiceAccount 57 | name: kucero 58 | namespace: kube-system 59 | --- 60 | apiVersion: rbac.authorization.k8s.io/v1 61 | kind: Role 62 | metadata: 63 | name: kucero 64 | namespace: kube-system 65 | rules: 66 | # Allow kucero to lock/unlock itself 67 | - apiGroups: ["apps"] 68 | resources: ["daemonsets"] 69 | resourceNames: ["kucero"] 70 | verbs: ["update"] 71 | # Allow kucero to access configmap 72 | - apiGroups: [""] 73 | resources: ["configmaps"] 74 | verbs: ["get", "list", "watch", "create", "update", "patch", "delete"] 75 | - apiGroups: [""] 76 | resources: ["configmaps/status"] 77 | verbs: ["get", "update", "patch"] 78 | - apiGroups: [""] 79 | resources: ["events"] 80 | verbs: ["create"] 81 | --- 82 | apiVersion: rbac.authorization.k8s.io/v1 83 | kind: RoleBinding 84 | metadata: 85 | name: kucero 86 | namespace: kube-system 87 | subjects: 88 | - kind: ServiceAccount 89 | namespace: kube-system 90 | name: kucero 91 | roleRef: 92 | apiGroup: rbac.authorization.k8s.io 93 | kind: Role 94 | name: kucero 95 | --- 96 | apiVersion: v1 97 | kind: ServiceAccount 98 | metadata: 99 | name: kucero 100 | namespace: kube-system 101 | -------------------------------------------------------------------------------- /pkg/host/command.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright (c) 2020 SUSE LLC. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package host 18 | 19 | import ( 20 | "os/exec" 21 | 22 | "github.com/sirupsen/logrus" 23 | ) 24 | 25 | // NewCommand creates a new Command with stdout/stderr wired to our standard logger 26 | func NewCommand(name string, arg ...string) *exec.Cmd { 27 | cmd := exec.Command(name, arg...) 28 | 29 | cmd.Stdout = logrus.NewEntry(logrus.StandardLogger()). 30 | WithField("cmd", cmd.Args[0]). 31 | WithField("std", "out"). 32 | WriterLevel(logrus.InfoLevel) 33 | 34 | cmd.Stderr = logrus.NewEntry(logrus.StandardLogger()). 35 | WithField("cmd", cmd.Args[0]). 36 | WithField("std", "err"). 37 | WriterLevel(logrus.WarnLevel) 38 | 39 | return cmd 40 | } 41 | 42 | // NewCommand creates a new Command with stderr wired to our standard logger 43 | func NewCommandWithStdout(name string, arg ...string) *exec.Cmd { 44 | cmd := exec.Command(name, arg...) 45 | 46 | cmd.Stderr = logrus.NewEntry(logrus.StandardLogger()). 47 | WithField("cmd", cmd.Args[0]). 48 | WithField("std", "err"). 49 | WriterLevel(logrus.WarnLevel) 50 | 51 | return cmd 52 | } 53 | -------------------------------------------------------------------------------- /pkg/host/kubectl.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright (c) 2020 SUSE LLC. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package host 18 | 19 | import ( 20 | "context" 21 | "os" 22 | 23 | corev1 "k8s.io/api/core/v1" 24 | "k8s.io/client-go/kubernetes" 25 | kubectldrain "k8s.io/kubectl/pkg/drain" 26 | 27 | "github.com/sirupsen/logrus" 28 | ) 29 | 30 | // Uncordon executes `kubectl uncordon ` 31 | // on the host system 32 | func Uncordon(client *kubernetes.Clientset, corev1Node *corev1.Node) error { 33 | nodeName := corev1Node.GetName() 34 | logrus.Infof("Uncordoning %s node", nodeName) 35 | 36 | drainer := &kubectldrain.Helper{ 37 | Ctx: context.TODO(), 38 | Client: client, 39 | Out: os.Stdout, 40 | ErrOut: os.Stderr, 41 | } 42 | // RunCordonOrUncordon runs either Cordon or Uncordon. 43 | // The desired value "false" is passed to "Unschedulable" to indicate that the node is schedulable. 44 | if err := kubectldrain.RunCordonOrUncordon(drainer, corev1Node, false); err != nil { 45 | logrus.Errorf("Error uncordonning %s: %v", nodeName, err) 46 | } 47 | return nil 48 | } 49 | 50 | // Cordon executes `kubectl cordon ` 51 | // on the host system 52 | func Cordon(client *kubernetes.Clientset, corev1Node *corev1.Node) error { 53 | nodeName := corev1Node.GetName() 54 | logrus.Infof("Cordoning %s node", nodeName) 55 | 56 | drainer := &kubectldrain.Helper{ 57 | Ctx: context.TODO(), 58 | Client: client, 59 | Out: os.Stdout, 60 | ErrOut: os.Stderr, 61 | } 62 | // RunCordonOrUncordon runs either Cordon or Uncordon. 63 | // The desired value "true" is passed to "Unschedulable" to indicate that the node is unschedulable. 64 | if err := kubectldrain.RunCordonOrUncordon(drainer, corev1Node, true); err != nil { 65 | logrus.Errorf("Error cordonning %s: %v", nodeName, err) 66 | } 67 | return nil 68 | } 69 | 70 | // Drain executes `kubectl drain --ignore-daemonsets --delete-local-data --force ` 71 | // on the host system 72 | func Drain(client *kubernetes.Clientset, corev1Node *corev1.Node) error { 73 | nodeName := corev1Node.GetName() 74 | logrus.Infof("Draining %s node", nodeName) 75 | 76 | drainer := &kubectldrain.Helper{ 77 | Ctx: context.TODO(), 78 | Client: client, 79 | Force: true, 80 | DeleteEmptyDirData: true, 81 | IgnoreAllDaemonSets: true, 82 | Out: os.Stdout, 83 | ErrOut: os.Stderr, 84 | } 85 | if err := kubectldrain.RunNodeDrain(drainer, nodeName); err != nil { 86 | logrus.Errorf("Error draining %s: %v", nodeName, err) 87 | } 88 | return nil 89 | } 90 | -------------------------------------------------------------------------------- /pkg/host/kubelet.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright (c) 2020 SUSE LLC. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package host 18 | 19 | import "github.com/sirupsen/logrus" 20 | 21 | // RestartKubelet executes `systemctl restart kubelet` 22 | // on the host system 23 | func RestartKubelet(nodeName string) error { 24 | logrus.Infof("Commanding restart kubelet on %s node", nodeName) 25 | 26 | // Relies on hostPID:true and privileged:true to enter host mount space 27 | cmd := NewCommand("/usr/bin/nsenter", "-m/proc/1/ns/mnt", "/usr/bin/systemctl", "restart", "kubelet") 28 | err := cmd.Run() 29 | if err != nil { 30 | logrus.Errorf("Error invoking %s: %v", cmd.Args, err) 31 | } 32 | 33 | return err 34 | } 35 | -------------------------------------------------------------------------------- /pkg/pki/authority/authority.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2019 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package authority 18 | 19 | import ( 20 | "crypto" 21 | "crypto/rand" 22 | "crypto/x509" 23 | "fmt" 24 | "math/big" 25 | "time" 26 | ) 27 | 28 | var serialNumberLimit = new(big.Int).Lsh(big.NewInt(1), 128) 29 | 30 | // CertificateAuthority implements a certificate authority that supports policy 31 | // based signing. It's used by the signing controller. 32 | type CertificateAuthority struct { 33 | // RawCert is an optional field to determine if signing cert/key pairs have changed 34 | RawCert []byte 35 | // RawKey is an optional field to determine if signing cert/key pairs have changed 36 | RawKey []byte 37 | 38 | Certificate *x509.Certificate 39 | PrivateKey crypto.Signer 40 | Backdate time.Duration 41 | Now func() time.Time 42 | } 43 | 44 | // Sign signs a certificate request, applying a SigningPolicy and returns a DER 45 | // encoded x509 certificate. 46 | func (ca *CertificateAuthority) Sign(crDER []byte, policy SigningPolicy) ([]byte, error) { 47 | now := time.Now() 48 | if ca.Now != nil { 49 | now = ca.Now() 50 | } 51 | 52 | nbf := now.Add(-ca.Backdate) 53 | if !nbf.Before(ca.Certificate.NotAfter) { 54 | return nil, fmt.Errorf("the signer has expired: NotAfter=%v", ca.Certificate.NotAfter) 55 | } 56 | 57 | cr, err := x509.ParseCertificateRequest(crDER) 58 | if err != nil { 59 | return nil, fmt.Errorf("unable to parse certificate request: %v", err) 60 | } 61 | if err := cr.CheckSignature(); err != nil { 62 | return nil, fmt.Errorf("unable to verify certificate request signature: %v", err) 63 | } 64 | 65 | serialNumber, err := rand.Int(rand.Reader, serialNumberLimit) 66 | if err != nil { 67 | return nil, fmt.Errorf("unable to generate a serial number for %s: %v", cr.Subject.CommonName, err) 68 | } 69 | 70 | tmpl := &x509.Certificate{ 71 | SerialNumber: serialNumber, 72 | Subject: cr.Subject, 73 | DNSNames: cr.DNSNames, 74 | IPAddresses: cr.IPAddresses, 75 | EmailAddresses: cr.EmailAddresses, 76 | URIs: cr.URIs, 77 | PublicKeyAlgorithm: cr.PublicKeyAlgorithm, 78 | PublicKey: cr.PublicKey, 79 | Extensions: cr.Extensions, 80 | ExtraExtensions: cr.ExtraExtensions, 81 | NotBefore: nbf, 82 | } 83 | if err := policy.apply(tmpl); err != nil { 84 | return nil, err 85 | } 86 | 87 | if !tmpl.NotAfter.Before(ca.Certificate.NotAfter) { 88 | tmpl.NotAfter = ca.Certificate.NotAfter 89 | } 90 | if !now.Before(ca.Certificate.NotAfter) { 91 | return nil, fmt.Errorf("refusing to sign a certificate that expired in the past") 92 | } 93 | 94 | der, err := x509.CreateCertificate(rand.Reader, tmpl, ca.Certificate, cr.PublicKey, ca.PrivateKey) 95 | if err != nil { 96 | return nil, fmt.Errorf("failed to sign certificate: %v", err) 97 | } 98 | return der, nil 99 | } 100 | -------------------------------------------------------------------------------- /pkg/pki/authority/policies.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2019 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package authority 18 | 19 | import ( 20 | "crypto/x509" 21 | "fmt" 22 | "sort" 23 | "time" 24 | 25 | capi "k8s.io/api/certificates/v1" 26 | ) 27 | 28 | // SigningPolicy validates a CertificateRequest before it's signed by the 29 | // CertificateAuthority. It may default or otherwise mutate a certificate 30 | // template. 31 | type SigningPolicy interface { 32 | // not-exporting apply forces signing policy implementations to be internal 33 | // to this package. 34 | apply(template *x509.Certificate) error 35 | } 36 | 37 | // PermissiveSigningPolicy is the signing policy historically used by the local 38 | // signer. 39 | // 40 | // * It forwards all SANs from the original signing request. 41 | // * It sets allowed usages as configured in the policy. 42 | // * It sets NotAfter based on the TTL configured in the policy. 43 | // * It zeros all extensions. 44 | // * It sets BasicConstraints to true. 45 | // * It sets IsCA to false. 46 | type PermissiveSigningPolicy struct { 47 | // TTL is the certificate TTL. It's used to calculate the NotAfter value of 48 | // the certificate. 49 | TTL time.Duration 50 | // Usages are the allowed usages of a certificate. 51 | Usages []capi.KeyUsage 52 | } 53 | 54 | func (p PermissiveSigningPolicy) apply(tmpl *x509.Certificate) error { 55 | usage, extUsages, err := keyUsagesFromStrings(p.Usages) 56 | if err != nil { 57 | return err 58 | } 59 | tmpl.KeyUsage = usage 60 | tmpl.ExtKeyUsage = extUsages 61 | tmpl.NotAfter = tmpl.NotBefore.Add(p.TTL) 62 | 63 | tmpl.ExtraExtensions = nil 64 | tmpl.Extensions = nil 65 | tmpl.BasicConstraintsValid = true 66 | tmpl.IsCA = false 67 | 68 | return nil 69 | } 70 | 71 | var keyUsageDict = map[capi.KeyUsage]x509.KeyUsage{ 72 | capi.UsageSigning: x509.KeyUsageDigitalSignature, 73 | capi.UsageDigitalSignature: x509.KeyUsageDigitalSignature, 74 | capi.UsageContentCommitment: x509.KeyUsageContentCommitment, 75 | capi.UsageKeyEncipherment: x509.KeyUsageKeyEncipherment, 76 | capi.UsageKeyAgreement: x509.KeyUsageKeyAgreement, 77 | capi.UsageDataEncipherment: x509.KeyUsageDataEncipherment, 78 | capi.UsageCertSign: x509.KeyUsageCertSign, 79 | capi.UsageCRLSign: x509.KeyUsageCRLSign, 80 | capi.UsageEncipherOnly: x509.KeyUsageEncipherOnly, 81 | capi.UsageDecipherOnly: x509.KeyUsageDecipherOnly, 82 | } 83 | 84 | var extKeyUsageDict = map[capi.KeyUsage]x509.ExtKeyUsage{ 85 | capi.UsageAny: x509.ExtKeyUsageAny, 86 | capi.UsageServerAuth: x509.ExtKeyUsageServerAuth, 87 | capi.UsageClientAuth: x509.ExtKeyUsageClientAuth, 88 | capi.UsageCodeSigning: x509.ExtKeyUsageCodeSigning, 89 | capi.UsageEmailProtection: x509.ExtKeyUsageEmailProtection, 90 | capi.UsageSMIME: x509.ExtKeyUsageEmailProtection, 91 | capi.UsageIPsecEndSystem: x509.ExtKeyUsageIPSECEndSystem, 92 | capi.UsageIPsecTunnel: x509.ExtKeyUsageIPSECTunnel, 93 | capi.UsageIPsecUser: x509.ExtKeyUsageIPSECUser, 94 | capi.UsageTimestamping: x509.ExtKeyUsageTimeStamping, 95 | capi.UsageOCSPSigning: x509.ExtKeyUsageOCSPSigning, 96 | capi.UsageMicrosoftSGC: x509.ExtKeyUsageMicrosoftServerGatedCrypto, 97 | capi.UsageNetscapeSGC: x509.ExtKeyUsageNetscapeServerGatedCrypto, 98 | } 99 | 100 | // keyUsagesFromStrings will translate a slice of usage strings from the 101 | // certificates API ("pkg/apis/certificates".KeyUsage) to x509.KeyUsage and 102 | // x509.ExtKeyUsage types. 103 | func keyUsagesFromStrings(usages []capi.KeyUsage) (x509.KeyUsage, []x509.ExtKeyUsage, error) { 104 | var keyUsage x509.KeyUsage 105 | var unrecognized []capi.KeyUsage 106 | extKeyUsages := make(map[x509.ExtKeyUsage]struct{}) 107 | for _, usage := range usages { 108 | if val, ok := keyUsageDict[usage]; ok { 109 | keyUsage |= val 110 | } else if val, ok := extKeyUsageDict[usage]; ok { 111 | extKeyUsages[val] = struct{}{} 112 | } else { 113 | unrecognized = append(unrecognized, usage) 114 | } 115 | } 116 | 117 | var sorted sortedExtKeyUsage 118 | for eku := range extKeyUsages { 119 | sorted = append(sorted, eku) 120 | } 121 | sort.Sort(sorted) 122 | 123 | if len(unrecognized) > 0 { 124 | return 0, nil, fmt.Errorf("unrecognized usage values: %q", unrecognized) 125 | } 126 | 127 | return keyUsage, []x509.ExtKeyUsage(sorted), nil 128 | } 129 | 130 | type sortedExtKeyUsage []x509.ExtKeyUsage 131 | 132 | func (s sortedExtKeyUsage) Len() int { 133 | return len(s) 134 | } 135 | 136 | func (s sortedExtKeyUsage) Swap(i, j int) { 137 | s[i], s[j] = s[j], s[i] 138 | } 139 | 140 | func (s sortedExtKeyUsage) Less(i, j int) bool { 141 | return s[i] < s[j] 142 | } 143 | -------------------------------------------------------------------------------- /pkg/pki/cert/certificate.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright (c) 2020 SUSE LLC. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package cert 18 | 19 | type Certificate interface { 20 | // CheckExpiration checks node certificate 21 | // returns the certificates which are going to expires 22 | CheckExpiration() ([]string, error) 23 | 24 | // Rotate rotates the node certificates 25 | // which are going to expires 26 | Rotate(expiryCertificates []string) error 27 | } 28 | -------------------------------------------------------------------------------- /pkg/pki/cert/kubeadm/exec.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright (c) 2020 SUSE LLC. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package kubeadm 18 | 19 | import ( 20 | "fmt" 21 | "regexp" 22 | "strings" 23 | "time" 24 | 25 | "k8s.io/apimachinery/pkg/util/version" 26 | 27 | "github.com/sirupsen/logrus" 28 | 29 | "github.com/jenting/kucero/pkg/host" 30 | "github.com/jenting/kucero/pkg/pki/clock" 31 | ) 32 | 33 | // kubeadmAlphaCertsCheckExpiration executes `kubeadm alpha certs check-expiration` 34 | // returns the certificates which are going to expires 35 | func kubeadmAlphaCertsCheckExpiration(expiryTimeToRotate time.Duration, clock clock.Clock) ([]string, error) { 36 | expiryCertificates := []string{} 37 | 38 | // Relies on hostPID:true and privileged:true to enter host mount space 39 | cmd := host.NewCommandWithStdout("/usr/bin/nsenter", "-m/proc/1/ns/mnt", "/usr/bin/kubeadm", "version", "-oshort") 40 | out, err := cmd.Output() 41 | if err != nil { 42 | logrus.Errorf("Error invoking %s: %v", cmd.Args, err) 43 | return expiryCertificates, err 44 | } 45 | 46 | // kubeadm >= 1.20.0: kubeadm certs check-expiration 47 | // otherwise: kubeadm alpha certs check-expiration 48 | ver := strings.TrimSuffix(string(out), "\n") 49 | if version.MustParseSemantic(ver).AtLeast(version.MustParseSemantic("v1.20.0")) { 50 | cmd = host.NewCommandWithStdout("/usr/bin/nsenter", "-m/proc/1/ns/mnt", "/usr/bin/kubeadm", "certs", "check-expiration") 51 | } else { 52 | cmd = host.NewCommandWithStdout("/usr/bin/nsenter", "-m/proc/1/ns/mnt", "/usr/bin/kubeadm", "alpha", "certs", "check-expiration") 53 | } 54 | stdout, err := cmd.Output() 55 | if err != nil { 56 | logrus.Errorf("Error invoking %s: %v", cmd.Args, err) 57 | return expiryCertificates, err 58 | } 59 | 60 | stdoutS := string(stdout) 61 | kv := parsekubeadmAlphaCertsCheckExpiration(stdoutS) 62 | for cert, t := range kv { 63 | expiry := checkCertificateExpiry(cert, t, expiryTimeToRotate, clock) 64 | if expiry { 65 | expiryCertificates = append(expiryCertificates, cert) 66 | } 67 | } 68 | 69 | return expiryCertificates, nil 70 | } 71 | 72 | func kubeadmAlphaCertsRenew(certificateName, certificatePath string) error { 73 | // Relies on hostPID:true and privileged:true to enter host mount space 74 | cmd := host.NewCommandWithStdout("/usr/bin/nsenter", "-m/proc/1/ns/mnt", "/usr/bin/kubeadm", "version", "-oshort") 75 | out, err := cmd.Output() 76 | if err != nil { 77 | logrus.Errorf("Error invoking %s: %v", cmd.Args, err) 78 | return err 79 | } 80 | 81 | // kubeadm >= 1.20.0: kubeadm certs renew 82 | // otherwise: kubeadm alpha certs renew 83 | ver := strings.TrimSuffix(string(out), "\n") 84 | if version.MustParseSemantic(ver).AtLeast(version.MustParseSemantic("v1.20.0")) { 85 | cmd = host.NewCommandWithStdout("/usr/bin/nsenter", "-m/proc/1/ns/mnt", "/usr/bin/kubeadm", "certs", "renew", certificateName) 86 | } else { 87 | cmd = host.NewCommandWithStdout("/usr/bin/nsenter", "-m/proc/1/ns/mnt", "/usr/bin/kubeadm", "alpha", "certs", "renew", certificateName) 88 | } 89 | return cmd.Run() 90 | } 91 | 92 | // parsekubeadmAlphaCertsCheckExpiration processes the `kubeadm alpha certs check-expiration` 93 | // output and returns the certificate and expires information 94 | func parsekubeadmAlphaCertsCheckExpiration(input string) map[string]time.Time { 95 | certExpires := make(map[string]time.Time) 96 | 97 | r := regexp.MustCompile("(.*) ([a-zA-Z]+ [0-9]{1,2}, [0-9]{4} [0-9]{1,2}:[0-9]{2} [a-zA-Z]+) (.*)") 98 | lines := strings.Split(input, "\n") 99 | parse := false 100 | for _, line := range lines { 101 | if parse { 102 | ss := r.FindStringSubmatch(line) 103 | if len(ss) < 3 { 104 | continue 105 | } 106 | 107 | cert := strings.TrimSpace(ss[1]) 108 | t, err := time.Parse("Jan 02, 2006 15:04 MST", ss[2]) 109 | if err != nil { 110 | fmt.Printf("err: %v\n", err) 111 | continue 112 | } 113 | 114 | certExpires[cert] = t 115 | } 116 | 117 | if strings.Contains(line, "CERTIFICATE") && strings.Contains(line, "EXPIRES") { 118 | parse = true 119 | } 120 | } 121 | 122 | return certExpires 123 | } 124 | 125 | // checkCertificateExpiry checks if the time `t` is less than the time duration `expiryTimeToRotate` 126 | func checkCertificateExpiry(name string, t time.Time, expiryTimeToRotate time.Duration, clock clock.Clock) bool { 127 | tn := clock.Now() 128 | if t.Before(tn) { 129 | logrus.Infof("The certificate %s is expiry already", name) 130 | return true 131 | } else if t.Sub(tn) <= expiryTimeToRotate { 132 | logrus.Infof("The certificate %s notAfter is less than user specified expiry time %s", name, expiryTimeToRotate) 133 | return true 134 | } 135 | 136 | logrus.Infof("The certificate %s is still valid for %s", name, t.Sub(tn)) 137 | return false 138 | } 139 | -------------------------------------------------------------------------------- /pkg/pki/cert/kubeadm/exec_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright (c) 2020 SUSE LLC. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package kubeadm 18 | 19 | import ( 20 | "reflect" 21 | "testing" 22 | "time" 23 | 24 | "github.com/jenting/kucero/pkg/pki/clock" 25 | ) 26 | 27 | func Test_parseKubeadmCertsCheckExpiration(t *testing.T) { 28 | tests := []struct { 29 | name string 30 | input string 31 | expect map[string]time.Time 32 | }{ 33 | { 34 | name: "kubeadm 1.15.2 and 1.16.2", 35 | input: ` 36 | CERTIFICATE EXPIRES RESIDUAL TIME EXTERNALLY MANAGED 37 | admin.conf May 12, 2021 02:29 UTC 364d no 38 | apiserver May 12, 2021 02:29 UTC 364d no 39 | apiserver-etcd-client May 12, 2021 02:29 UTC 364d no 40 | apiserver-kubelet-client May 12, 2021 02:29 UTC 364d no 41 | controller-manager.conf May 12, 2021 02:29 UTC 364d no 42 | etcd-healthcheck-client May 12, 2021 02:29 UTC 364d no 43 | etcd-peer May 12, 2021 02:29 UTC 364d no 44 | etcd-server May 12, 2021 02:29 UTC 364d no 45 | front-proxy-client May 12, 2021 02:29 UTC 364d no 46 | scheduler.conf May 12, 2021 02:29 UTC 364d no 47 | `, 48 | expect: map[string]time.Time{ 49 | "admin.conf": time.Date(2021, time.May, 12, 02, 29, 00, 00, time.UTC), 50 | "apiserver": time.Date(2021, time.May, 12, 02, 29, 00, 00, time.UTC), 51 | "apiserver-etcd-client": time.Date(2021, time.May, 12, 02, 29, 00, 00, time.UTC), 52 | "apiserver-kubelet-client": time.Date(2021, time.May, 12, 02, 29, 00, 00, time.UTC), 53 | "controller-manager.conf": time.Date(2021, time.May, 12, 02, 29, 00, 00, time.UTC), 54 | "etcd-healthcheck-client": time.Date(2021, time.May, 12, 02, 29, 00, 00, time.UTC), 55 | "etcd-peer": time.Date(2021, time.May, 12, 02, 29, 00, 00, time.UTC), 56 | "etcd-server": time.Date(2021, time.May, 12, 02, 29, 00, 00, time.UTC), 57 | "front-proxy-client": time.Date(2021, time.May, 12, 02, 29, 00, 00, time.UTC), 58 | "scheduler.conf": time.Date(2021, time.May, 12, 02, 29, 00, 00, time.UTC), 59 | }, 60 | }, 61 | { 62 | name: "kubeadm 1.17.4", 63 | input: ` 64 | CERTIFICATE EXPIRES RESIDUAL TIME CERTIFICATE AUTHORITY EXTERNALLY MANAGED 65 | admin.conf May 12, 2021 02:29 UTC 364d no 66 | apiserver May 12, 2021 02:29 UTC 364d ca no 67 | apiserver-etcd-client May 12, 2021 02:29 UTC 364d etcd-ca no 68 | apiserver-kubelet-client May 12, 2021 02:29 UTC 364d ca no 69 | controller-manager.conf May 12, 2021 02:29 UTC 364d no 70 | etcd-healthcheck-client May 12, 2021 02:29 UTC 364d etcd-ca no 71 | etcd-peer May 12, 2021 02:29 UTC 364d etcd-ca no 72 | etcd-server May 12, 2021 02:29 UTC 364d etcd-ca no 73 | front-proxy-client May 12, 2021 02:29 UTC 364d front-proxy-ca no 74 | scheduler.conf May 12, 2021 02:29 UTC 364d no 75 | 76 | CERTIFICATE AUTHORITY EXPIRES RESIDUAL TIME EXTERNALLY MANAGED 77 | ca Mar 11, 2030 01:51 UTC 9y no 78 | etcd-ca Mar 11, 2030 01:51 UTC 9y no 79 | front-proxy-ca Mar 11, 2030 01:51 UTC 9y no 80 | `, 81 | expect: map[string]time.Time{ 82 | "admin.conf": time.Date(2021, time.May, 12, 02, 29, 00, 00, time.UTC), 83 | "apiserver": time.Date(2021, time.May, 12, 02, 29, 00, 00, time.UTC), 84 | "apiserver-etcd-client": time.Date(2021, time.May, 12, 02, 29, 00, 00, time.UTC), 85 | "apiserver-kubelet-client": time.Date(2021, time.May, 12, 02, 29, 00, 00, time.UTC), 86 | "ca": time.Date(2030, time.March, 11, 01, 51, 00, 00, time.UTC), 87 | "controller-manager.conf": time.Date(2021, time.May, 12, 02, 29, 00, 00, time.UTC), 88 | "etcd-ca": time.Date(2030, time.March, 11, 01, 51, 00, 00, time.UTC), 89 | "etcd-healthcheck-client": time.Date(2021, time.May, 12, 02, 29, 00, 00, time.UTC), 90 | "etcd-peer": time.Date(2021, time.May, 12, 02, 29, 00, 00, time.UTC), 91 | "etcd-server": time.Date(2021, time.May, 12, 02, 29, 00, 00, time.UTC), 92 | "front-proxy-ca": time.Date(2030, time.March, 11, 01, 51, 00, 00, time.UTC), 93 | "front-proxy-client": time.Date(2021, time.May, 12, 02, 29, 00, 00, time.UTC), 94 | "scheduler.conf": time.Date(2021, time.May, 12, 02, 29, 00, 00, time.UTC), 95 | }, 96 | }, 97 | } 98 | 99 | for _, tt := range tests { 100 | tt := tt 101 | t.Run(tt.name, func(t *testing.T) { 102 | got := parsekubeadmAlphaCertsCheckExpiration(tt.input) 103 | if !reflect.DeepEqual(got, tt.expect) { 104 | t.Errorf("got %v is not equals to expected", got) 105 | } 106 | }) 107 | } 108 | } 109 | 110 | type stubClock struct { 111 | } 112 | 113 | func newStubClock() clock.Clock { 114 | return &stubClock{} 115 | } 116 | 117 | func (s *stubClock) Now() time.Time { 118 | return time.Date(2000, time.January, 02, 03, 04, 05, 06, time.UTC) 119 | } 120 | 121 | func Test_checkExpiry(t *testing.T) { 122 | stubClock := newStubClock() 123 | 124 | tests := []struct { 125 | name string 126 | inputT time.Time 127 | inputExpiryTimeToRotate time.Duration 128 | expect bool 129 | }{ 130 | { 131 | name: "expired certificate", 132 | inputT: time.Date(1960, time.May, 12, 02, 29, 00, 00, time.UTC), 133 | inputExpiryTimeToRotate: time.Minute, 134 | expect: true, 135 | }, 136 | { 137 | name: "going to expire certificate", 138 | inputT: time.Date(2000, time.January, 02, 03, 05, 05, 06, time.UTC), 139 | inputExpiryTimeToRotate: time.Minute, 140 | expect: true, 141 | }, 142 | { 143 | name: "still valid certificate", 144 | inputT: time.Date(2100, time.May, 12, 02, 29, 00, 00, time.UTC), 145 | inputExpiryTimeToRotate: time.Minute, 146 | expect: false, 147 | }, 148 | } 149 | 150 | for _, tt := range tests { 151 | tt := tt 152 | t.Run(tt.name, func(t *testing.T) { 153 | got := checkCertificateExpiry(tt.name, tt.inputT, tt.inputExpiryTimeToRotate, stubClock) 154 | if !reflect.DeepEqual(got, tt.expect) { 155 | t.Errorf("got %t is not equals to expected %t", got, tt.expect) 156 | } 157 | }) 158 | } 159 | } 160 | -------------------------------------------------------------------------------- /pkg/pki/cert/kubeadm/kubeadm.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright (c) 2020 SUSE LLC. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package kubeadm 18 | 19 | import ( 20 | "fmt" 21 | "path/filepath" 22 | "strings" 23 | "time" 24 | 25 | "github.com/sirupsen/logrus" 26 | 27 | "github.com/jenting/kucero/pkg/host" 28 | "github.com/jenting/kucero/pkg/pki/cert" 29 | "github.com/jenting/kucero/pkg/pki/clock" 30 | ) 31 | 32 | var certificates map[string]string = map[string]string{ 33 | "admin.conf": "/etc/kubernetes/admin.conf", 34 | "controller-manager.conf": "/etc/kubernetes/controller-manager.conf", 35 | "scheduler.conf": "/etc/kubernetes/scheduler.conf", 36 | "apiserver": "/etc/kubernetes/pki/apiserver.crt", 37 | "apiserver-etcd-client": "/etc/kubernetes/pki/apiserver-etcd-client.crt", 38 | "apiserver-kubelet-client": "/etc/kubernetes/pki/apiserver-kubelet-client.crt", 39 | "front-proxy-client": "/etc/kubernetes/pki/front-proxy-client.crt", 40 | "etcd-healthcheck-client": "/etc/kubernetes/pki/etcd/healthcheck-client.crt", 41 | "etcd-peer": "/etc/kubernetes/pki/etcd/peer.crt", 42 | "etcd-server": "/etc/kubernetes/pki/etcd/server.crt", 43 | } 44 | 45 | type Kubeadm struct { 46 | nodeName string 47 | expiryTimeToRotate time.Duration 48 | clock clock.Clock 49 | } 50 | 51 | // New returns the kubeadm instance 52 | func New(nodeName string, expiryTimeToRotate time.Duration) cert.Certificate { 53 | return &Kubeadm{ 54 | nodeName: nodeName, 55 | expiryTimeToRotate: expiryTimeToRotate, 56 | clock: clock.NewRealClock(), 57 | } 58 | } 59 | 60 | // CheckExpiration checks control plane node certificate 61 | // returns the certificates which are going to expires 62 | func (k *Kubeadm) CheckExpiration() ([]string, error) { 63 | logrus.Infof("Commanding check %s node certificate expiration", k.nodeName) 64 | 65 | return kubeadmAlphaCertsCheckExpiration(k.expiryTimeToRotate, k.clock) 66 | } 67 | 68 | // Rotate executes the steps to rotates the certificate 69 | // including backing up certificate, rotates certificate, and restart kubelet 70 | func (k *Kubeadm) Rotate(expiryCertificates []string) error { 71 | var errs error 72 | for _, certificateName := range expiryCertificates { 73 | certificatePath, ok := certificates[certificateName] 74 | if !ok { 75 | continue 76 | } 77 | 78 | if err := backupCertificate(k.nodeName, certificateName, certificatePath); err != nil { 79 | errs = fmt.Errorf("%w; ", err) 80 | continue 81 | } 82 | 83 | if err := rotateCertificate(k.nodeName, certificateName, certificatePath); err != nil { 84 | errs = fmt.Errorf("%w; ", err) 85 | continue 86 | } 87 | } 88 | if errs != nil { 89 | return errs 90 | } 91 | 92 | if err := host.RestartKubelet(k.nodeName); err != nil { 93 | errs = fmt.Errorf("%w; ", err) 94 | } 95 | 96 | return errs 97 | } 98 | 99 | // backupCertificate backups the certificate/kubeconfig 100 | // under folder /etc/kubernetes issued by kubeadm 101 | func backupCertificate(nodeName string, certificateName, certificatePath string) error { 102 | logrus.Infof("Commanding backup %s node certificate %s path %s", nodeName, certificateName, certificatePath) 103 | 104 | dir := filepath.Dir(certificatePath) 105 | base := filepath.Base(certificatePath) 106 | ext := filepath.Ext(certificatePath) 107 | certificateBackupPath := filepath.Join(dir, strings.TrimSuffix(base, ext)+"-"+time.Now().Format("20060102030405")+ext+".bak") 108 | 109 | // Relies on hostPID:true and privileged:true to enter host mount space 110 | var err error 111 | cmd := host.NewCommand("/usr/bin/nsenter", "-m/proc/1/ns/mnt", "/usr/bin/cp", certificatePath, certificateBackupPath) 112 | err = cmd.Run() 113 | if err != nil { 114 | logrus.Errorf("Error invoking %s: %v", cmd.Args, err) 115 | } 116 | 117 | return err 118 | } 119 | 120 | // rotateCertificate calls `kubeadm alpha certs renew ` 121 | // on the host system to rotates kubeadm issued certificates 122 | func rotateCertificate(nodeName string, certificateName, certificatePath string) error { 123 | logrus.Infof("Commanding rotate %s node certificate %s path %s", nodeName, certificateName, certificatePath) 124 | 125 | err := kubeadmAlphaCertsRenew(certificateName, certificatePath) 126 | if err != nil { 127 | logrus.Errorf("Error invoking command: %v", err) 128 | } 129 | 130 | return err 131 | } 132 | -------------------------------------------------------------------------------- /pkg/pki/cert/null/null.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright (c) 2020 SUSE LLC. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package null 18 | 19 | import ( 20 | "time" 21 | 22 | "github.com/jenting/kucero/pkg/pki/cert" 23 | ) 24 | 25 | type Null struct { 26 | } 27 | 28 | // New returns the kubeadm instance 29 | func New(nodeName string, expiryTimeToRotate time.Duration) cert.Certificate { 30 | return &Null{} 31 | } 32 | 33 | // CheckExpiration returns empty slice string array and nil 34 | func (n *Null) CheckExpiration() ([]string, error) { 35 | return []string{}, nil 36 | } 37 | 38 | // Rotate returns nil 39 | func (n *Null) Rotate(expiryCertificates []string) error { 40 | return nil 41 | } 42 | -------------------------------------------------------------------------------- /pkg/pki/cert/parser.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright (c) 2020 SUSE LLC. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package cert 18 | 19 | import ( 20 | "crypto/x509" 21 | "encoding/pem" 22 | "errors" 23 | ) 24 | 25 | // ParseCSR decodes a PEM encoded CSR 26 | // Copied from https://github.com/kubernetes/kubernetes/blob/v1.18.0-beta.2/pkg/apis/certificates/v1beta1/helpers.go 27 | func ParseCSR(pemBytes []byte) (*x509.CertificateRequest, error) { 28 | // extract PEM from request object 29 | block, _ := pem.Decode(pemBytes) 30 | if block == nil || block.Type != "CERTIFICATE REQUEST" { 31 | return nil, errors.New("PEM block type must be CERTIFICATE REQUEST") 32 | } 33 | csr, err := x509.ParseCertificateRequest(block.Bytes) 34 | if err != nil { 35 | return nil, err 36 | } 37 | return csr, nil 38 | } 39 | -------------------------------------------------------------------------------- /pkg/pki/clock/clock.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright (c) 2020 SUSE LLC. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package clock 18 | 19 | import "time" 20 | 21 | type Clock interface { 22 | Now() time.Time 23 | } 24 | 25 | type RealClock struct { 26 | } 27 | 28 | func NewRealClock() Clock { 29 | return &RealClock{} 30 | } 31 | 32 | func (r *RealClock) Now() time.Time { 33 | return time.Now() 34 | } 35 | -------------------------------------------------------------------------------- /pkg/pki/conf/conf.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright (c) 2020 SUSE LLC. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package conf 18 | 19 | type Config interface { 20 | // CheckConfig checks whether the configuration need to be update 21 | // returns the configuration and it's update config callback function 22 | CheckConfig() ([]string, error) 23 | 24 | // UpdateConfig updates the configuration by 25 | // passing the configuration and it's update config callback function 26 | UpdateConfig([]string) error 27 | } 28 | -------------------------------------------------------------------------------- /pkg/pki/conf/kubelet/exec.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright (c) 2020 SUSE LLC. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package kubelet 18 | 19 | import ( 20 | "bytes" 21 | "fmt" 22 | "io/ioutil" 23 | "os" 24 | 25 | "k8s.io/client-go/tools/clientcmd" 26 | "sigs.k8s.io/yaml" 27 | ) 28 | 29 | type action struct { 30 | check func(*Kubelet, string) (bool, error) 31 | update func(*Kubelet, string, string) error 32 | } 33 | 34 | var configs map[string]action = map[string]action{ 35 | "/etc/kubernetes/kubelet.conf": { 36 | check: func(k *Kubelet, filepath string) (bool, error) { 37 | return k.checkEtcKubernetesKubeletConf(filepath) 38 | }, 39 | update: func(k *Kubelet, oldFilepath, newFilepath string) error { 40 | return k.updateEtcKubernetesKubeletConf(oldFilepath, newFilepath) 41 | }, 42 | }, 43 | "/var/lib/kubelet/config.yaml": { 44 | check: func(k *Kubelet, filepath string) (bool, error) { 45 | return k.checkVarLibKubeletConfigYaml(filepath) 46 | }, 47 | update: func(k *Kubelet, oldFilepath, newFilepath string) error { 48 | return k.updateVarLibKubeletConfigYaml(oldFilepath, newFilepath) 49 | }, 50 | }, 51 | } 52 | 53 | // checkEtcKubernetesKubeletConf checks /etc/kubernetes/kubelet.conf need to be update 54 | // if client-certificate-data or client-key-data exist 55 | func (k *Kubelet) checkEtcKubernetesKubeletConf(filepath string) (bool, error) { 56 | kubeletConfig, err := clientcmd.LoadFromFile(filepath) 57 | if err != nil { 58 | return false, err 59 | } 60 | 61 | for _, authInfo := range kubeletConfig.AuthInfos { 62 | if len(authInfo.ClientKeyData) > 0 || len(authInfo.ClientCertificateData) > 0 { 63 | return true, nil 64 | } 65 | } 66 | return false, nil 67 | } 68 | 69 | // updateEtcKubernetesKubeletConf updates the /etc/kubernetes/kubelet.conf 70 | // from 71 | // client-certificate-data: 72 | // client-key-data: 73 | // to 74 | // client-certificate: /var/lib/kubelet/pki/kubelet-client-current.pem 75 | // client-key: /var/lib/kubelet/pki/kubelet-client-current.pem 76 | func (k *Kubelet) updateEtcKubernetesKubeletConf(oldFilepath, newFilepath string) error { 77 | kubeletConfig, err := clientcmd.LoadFromFile(oldFilepath) 78 | if err != nil { 79 | return err 80 | } 81 | 82 | for _, authInfo := range kubeletConfig.AuthInfos { 83 | if len(authInfo.ClientKeyData) > 0 || len(authInfo.ClientCertificateData) > 0 { 84 | authInfo.ClientKeyData = []byte{} 85 | authInfo.ClientCertificateData = []byte{} 86 | authInfo.ClientKey = "/var/lib/kubelet/pki/kubelet-client-current.pem" 87 | authInfo.ClientCertificate = "/var/lib/kubelet/pki/kubelet-client-current.pem" 88 | } 89 | } 90 | 91 | if err = clientcmd.WriteToFile(*kubeletConfig, newFilepath); err != nil { 92 | return fmt.Errorf("failed to serialize %q", newFilepath) 93 | } 94 | return nil 95 | } 96 | 97 | // kubeletConfiguration contains the configuration for the /var/lib/kubelet/config.yaml 98 | type kubeletConfiguration struct { 99 | RotateCertificates bool `json:"rotateCertificates,omitempty"` 100 | ServerTLSBootstrap bool `json:"serverTLSBootstrap,omitempty"` 101 | } 102 | 103 | // checkVarLibKubeletConfigYaml checks /var/lib/kubelet/config.yaml need to be update 104 | // if rotateCertificates and serverTLSBootstrap does not match the configuration 105 | func (k *Kubelet) checkVarLibKubeletConfigYaml(filepath string) (bool, error) { 106 | kubeletConfig, err := ioutil.ReadFile(filepath) 107 | if err != nil { 108 | return false, err 109 | } 110 | 111 | kc := kubeletConfiguration{} 112 | if err := yaml.Unmarshal(kubeletConfig, &kc); err != nil { 113 | return false, err 114 | } 115 | 116 | if kc.RotateCertificates != k.enableKubeletClientCertRotation { 117 | return true, nil 118 | } 119 | if kc.ServerTLSBootstrap != k.enableKubeletServerCertRotation { 120 | return true, nil 121 | } 122 | return false, nil 123 | } 124 | 125 | // updateVarLibKubeletConfigYaml updates /var/lib/kubelet/config.yaml of 126 | // the key `rotateCertificates` and `serverTLSBootstrap` 127 | func (k *Kubelet) updateVarLibKubeletConfigYaml(oldFilepath, newFilepath string) error { 128 | kubeletConfig, err := ioutil.ReadFile(oldFilepath) 129 | if err != nil { 130 | return err 131 | } 132 | 133 | kc := kubeletConfiguration{} 134 | if err := yaml.Unmarshal(kubeletConfig, &kc); err != nil { 135 | return err 136 | } 137 | 138 | if kc.RotateCertificates != k.enableKubeletClientCertRotation { 139 | // set "rotateCertificates: true" or "rotateCertificates: false" in /var/lib/kubelet/config.yaml to enable/disable kubelet client cert rotation 140 | if bytes.Contains(kubeletConfig, []byte("rotateCertificates:")) { 141 | kubeletConfig = bytes.Replace(kubeletConfig, 142 | []byte(fmt.Sprintf("rotateCertificates: %t", kc.RotateCertificates)), 143 | []byte(fmt.Sprintf("rotateCertificates: %t", k.enableKubeletClientCertRotation)), 144 | 1) 145 | } else { 146 | kubeletConfig = append(kubeletConfig, []byte(fmt.Sprintf("rotateCertificates: %t\n", k.enableKubeletClientCertRotation))...) 147 | } 148 | } 149 | if kc.ServerTLSBootstrap != k.enableKubeletServerCertRotation { 150 | // set "serverTLSBootstrap: true" or "serverTLSBootstrap: false" in /var/lib/kubelet/config.yaml to enable/disable kubelet server cert rotation 151 | if bytes.Contains(kubeletConfig, []byte("serverTLSBootstrap:")) { 152 | kubeletConfig = bytes.Replace(kubeletConfig, 153 | []byte(fmt.Sprintf("serverTLSBootstrap: %t", kc.ServerTLSBootstrap)), 154 | []byte(fmt.Sprintf("serverTLSBootstrap: %t", k.enableKubeletServerCertRotation)), 155 | 1) 156 | } else { 157 | kubeletConfig = append(kubeletConfig, []byte(fmt.Sprintf("serverTLSBootstrap: %t\n", k.enableKubeletServerCertRotation))...) 158 | } 159 | } 160 | 161 | f, err := os.Stat(oldFilepath) 162 | if err != nil { 163 | return err 164 | } 165 | return ioutil.WriteFile(newFilepath, kubeletConfig, f.Mode()) 166 | } 167 | -------------------------------------------------------------------------------- /pkg/pki/conf/kubelet/exec_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright (c) 2020 SUSE LLC. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package kubelet 18 | 19 | import ( 20 | "io/ioutil" 21 | "log" 22 | "os" 23 | "reflect" 24 | "testing" 25 | ) 26 | 27 | func TestEtcKubernetesKubeletConf(t *testing.T) { 28 | tests := []struct { 29 | name string 30 | filepath string 31 | expect bool 32 | }{ 33 | { 34 | name: "kubelet.conf before Kubernetes version 1.17", 35 | filepath: "tests/1.16-kubelet.conf", 36 | expect: true, 37 | }, 38 | { 39 | name: "kubelet.conf after Kubernetes version 1.17", 40 | filepath: "tests/1.17-kubelet.conf", 41 | expect: false, 42 | }, 43 | } 44 | 45 | for _, tt := range tests { 46 | tt := tt 47 | t.Run(tt.name, func(t *testing.T) { 48 | k := &Kubelet{nodeName: tt.name} 49 | got, err := k.checkEtcKubernetesKubeletConf(tt.filepath) 50 | if err != nil { 51 | t.Errorf("expected no error but error reported: %v\n", err) 52 | } 53 | 54 | if !reflect.DeepEqual(got, tt.expect) { 55 | t.Errorf("got %v is not equals to expected", got) 56 | } 57 | 58 | // create a temporary file to save the updated kubelet.conf 59 | file, err := ioutil.TempFile("tests", "*-kubelet.yaml") 60 | if err != nil { 61 | log.Fatal(err) 62 | } 63 | defer os.Remove(file.Name()) 64 | 65 | err = k.updateEtcKubernetesKubeletConf(tt.filepath, file.Name()) 66 | if err != nil { 67 | t.Errorf("expected no error but error reported: %v\n", err) 68 | } 69 | 70 | got, err = k.checkEtcKubernetesKubeletConf(file.Name()) 71 | if err != nil { 72 | t.Errorf("expected no error but error reported: %v\n", err) 73 | } 74 | if got { 75 | t.Errorf("expected no update but a update reported: %t\n", got) 76 | } 77 | }) 78 | } 79 | } 80 | 81 | func TestVarLibKubeletConfigYaml(t *testing.T) { 82 | tests := []struct { 83 | name string 84 | filepath string 85 | enableKubeletClientCertRotation bool 86 | enableKubeletServerCertRotation bool 87 | expect bool 88 | }{ 89 | { 90 | name: "client: client enabled, server enabled", 91 | filepath: "tests/client-kubelet.yaml", 92 | enableKubeletClientCertRotation: true, 93 | enableKubeletServerCertRotation: true, 94 | expect: true, 95 | }, 96 | { 97 | name: "client: client enabled, server disabled", 98 | filepath: "tests/client-kubelet.yaml", 99 | enableKubeletClientCertRotation: true, 100 | enableKubeletServerCertRotation: false, 101 | expect: false, 102 | }, 103 | { 104 | name: "client: client disabled, server enabled", 105 | filepath: "tests/client-kubelet.yaml", 106 | enableKubeletClientCertRotation: false, 107 | enableKubeletServerCertRotation: true, 108 | expect: true, 109 | }, 110 | { 111 | name: "client: client disabled, server disabled", 112 | filepath: "tests/client-kubelet.yaml", 113 | enableKubeletClientCertRotation: false, 114 | enableKubeletServerCertRotation: false, 115 | expect: true, 116 | }, 117 | { 118 | name: "server: client enabled, server enabled", 119 | filepath: "tests/server-kubelet.yaml", 120 | enableKubeletClientCertRotation: true, 121 | enableKubeletServerCertRotation: true, 122 | expect: true, 123 | }, 124 | { 125 | name: "server: client enabled, server disabled", 126 | filepath: "tests/server-kubelet.yaml", 127 | enableKubeletClientCertRotation: true, 128 | enableKubeletServerCertRotation: false, 129 | expect: true, 130 | }, 131 | { 132 | name: "server: client disabled, server enabled", 133 | filepath: "tests/server-kubelet.yaml", 134 | enableKubeletClientCertRotation: false, 135 | enableKubeletServerCertRotation: true, 136 | expect: false, 137 | }, 138 | { 139 | name: "server: client disabled, server disabled", 140 | filepath: "tests/server-kubelet.yaml", 141 | enableKubeletClientCertRotation: false, 142 | enableKubeletServerCertRotation: false, 143 | expect: true, 144 | }, 145 | } 146 | 147 | for _, tt := range tests { 148 | tt := tt 149 | t.Run(tt.name, func(t *testing.T) { 150 | k := &Kubelet{ 151 | nodeName: tt.name, 152 | enableKubeletClientCertRotation: tt.enableKubeletClientCertRotation, 153 | enableKubeletServerCertRotation: tt.enableKubeletServerCertRotation, 154 | } 155 | got, err := k.checkVarLibKubeletConfigYaml(tt.filepath) 156 | if err != nil { 157 | t.Errorf("expected no error but error reported: %v\n", err) 158 | } 159 | 160 | if !reflect.DeepEqual(got, tt.expect) { 161 | t.Errorf("got %v is not equals to expected", got) 162 | } 163 | 164 | // create a temporary file to save the updated kubelet.yaml 165 | file, err := ioutil.TempFile("tests", "*-kubelet.yaml") 166 | if err != nil { 167 | log.Fatal(err) 168 | } 169 | defer os.Remove(file.Name()) 170 | 171 | err = k.updateVarLibKubeletConfigYaml(tt.filepath, file.Name()) 172 | if err != nil { 173 | t.Errorf("expected no error but error reported: %v\n", err) 174 | } 175 | 176 | got, err = k.checkVarLibKubeletConfigYaml(file.Name()) 177 | if err != nil { 178 | t.Errorf("expected no error but error reported: %v\n", err) 179 | } 180 | if got { 181 | t.Errorf("expected no update but a update reported: %t\n", got) 182 | } 183 | }) 184 | } 185 | } 186 | -------------------------------------------------------------------------------- /pkg/pki/conf/kubelet/kubelet.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright (c) 2020 SUSE LLC. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package kubelet 18 | 19 | import ( 20 | "fmt" 21 | 22 | "github.com/sirupsen/logrus" 23 | 24 | "github.com/jenting/kucero/pkg/host" 25 | "github.com/jenting/kucero/pkg/pki/conf" 26 | ) 27 | 28 | type Kubelet struct { 29 | nodeName string 30 | enableKubeletClientCertRotation bool 31 | enableKubeletServerCertRotation bool 32 | } 33 | 34 | // New returns the kubelet instance 35 | func New(nodeName string, enableKubeletClientCertRotation, enableKubeletServerCertRotation bool) conf.Config { 36 | return &Kubelet{ 37 | nodeName: nodeName, 38 | enableKubeletClientCertRotation: enableKubeletClientCertRotation, 39 | enableKubeletServerCertRotation: enableKubeletServerCertRotation, 40 | } 41 | } 42 | 43 | func (k *Kubelet) CheckConfig() ([]string, error) { 44 | logrus.Infof("Commanding check %s node kubelet configuration", k.nodeName) 45 | 46 | var errs error 47 | var configsToBeUpdate []string 48 | for filepath, action := range configs { 49 | toUpdate, err := action.check(k, filepath) 50 | if err != nil { 51 | errs = fmt.Errorf("%w; ", err) 52 | continue 53 | } 54 | if toUpdate { 55 | configsToBeUpdate = append(configsToBeUpdate, filepath) 56 | } 57 | } 58 | 59 | return configsToBeUpdate, errs 60 | } 61 | 62 | func (k *Kubelet) UpdateConfig(configsToBeUpdate []string) error { 63 | var errs error 64 | for _, configToBeUpdate := range configsToBeUpdate { 65 | logrus.Infof("Commanding update %s node kubelet config path %s", k.nodeName, configToBeUpdate) 66 | 67 | action, ok := configs[configToBeUpdate] 68 | if !ok { 69 | return fmt.Errorf("map key %s does not exist", configToBeUpdate) 70 | } 71 | err := action.update(k, configToBeUpdate, configToBeUpdate) 72 | if err != nil { 73 | errs = fmt.Errorf("%w; ", err) 74 | continue 75 | } 76 | } 77 | if errs != nil { 78 | return errs 79 | } 80 | 81 | if err := host.RestartKubelet(k.nodeName); err != nil { 82 | errs = fmt.Errorf("%w; ", err) 83 | } 84 | 85 | return errs 86 | } 87 | -------------------------------------------------------------------------------- /pkg/pki/conf/kubelet/tests/1.16-kubelet.conf: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | clusters: 3 | - cluster: 4 | certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUN5RENDQWJDZ0F3SUJBZ0lCQURBTkJna3Foa2lHOXcwQkFRc0ZBREFWTVJNd0VRWURWUVFERXdwcmRXSmwKY201bGRHVnpNQjRYRFRJd01Ea3lPVEF4TkRneU1sb1hEVE13TURreU56QXhORGd5TWxvd0ZURVRNQkVHQTFVRQpBeE1LYTNWaVpYSnVaWFJsY3pDQ0FTSXdEUVlKS29aSWh2Y05BUUVCQlFBRGdnRVBBRENDQVFvQ2dnRUJBTGZtCmlVeDNXeUl2WWJVU3c0R2l4SEFPNzY3Sk9ybTZVaWd6VkQzUS9kM1JjSzlEdnQ1MWtwVVMyR0pIZjh0ZW1UaG4KQ0N3YWl5SkxWbXRZWWVWcW53YmF0RUtQbHlsSFE0MTVEUmpNY00wM1BmVWJkOU5ZRHFCVHVKSmNSdk42QVBOQQpBSERuQWRtNjBMNzFvYkFGbkRrNDRDNjVjL2hqYzFzT29sVWtxdDdPejEvUFZWblg5R0U2TWQzck5RN0RGdndaCjlVOFZpeTJyUytSY1ltTVE0VmdCQTdacnJ5YmtLZG5QdXhHMXE4S25SSmNydW9ZVVYrQWxBbUlxSENQRzZGS20KcVBiYS9lY0VmZXJFc1JsdFZFVFFLcEdjMk4wWlc2L3Jkc1EyOE80Z1JnK0U2V0k3dlN2OWV4K3I2K3ExM0Fzcwp0bE8yVlJ3RWlCVDJvNm5kUThFQ0F3RUFBYU1qTUNFd0RnWURWUjBQQVFIL0JBUURBZ0trTUE4R0ExVWRFd0VCCi93UUZNQU1CQWY4d0RRWUpLb1pJaHZjTkFRRUxCUUFEZ2dFQkFDQUlXUG5mVWR1bUhiak9qTWhYSW1rZEJrbzIKTVE0SnVOUlN6MW5aYlpRaklyNmJJdmpSM1dMV0xyWVFldGNUTllJTGwwb1VuWS8wb3pSUlA5d0QyRWRaV05sUwppUitoTTM4dm02REp3d0JKSUZvMDdtejFURURWK1hRVGZwM0ljRGtrbjk4d25QMU1LanovUWcwTGJqZmk2aDlUCmZ1dkZDMDFDZWJGN0FLL3dEV3pTa1FVSzNxTy9rL3ZQTVp2OHR3WjBoZENjUmIvSVgzTnJOVEViN3BnOVR6cU0KeDk2bGxMZmFVWUhZRUM1ZFpQOEI0anJPcWRibEg4Tnc3TlVsWXFWY3NJbUZOVEVvNmNMQ29XN3FTMFJKVmFXdworUFdlaUZ6V3dZb0lFSzUrMDhtYSsvd3BzeU1hd1pxcE4xckpTRkJBcVNiN3cyRjFLY3hQamladjV6ND0KLS0tLS1FTkQgQ0VSVElGSUNBVEUtLS0tLQo= 5 | server: https://1.16-control-plane:6443 6 | name: "1.16" 7 | contexts: 8 | - context: 9 | cluster: "1.16" 10 | user: system:node:1.16-control-plane 11 | name: system:node:1.16-control-plane@1.16 12 | current-context: system:node:1.16-control-plane@1.16 13 | kind: Config 14 | preferences: {} 15 | users: 16 | - name: system:node:1.16-control-plane 17 | user: 18 | client-certificate-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUMvakNDQWVhZ0F3SUJBZ0lJZk1MYXF3cE56YmN3RFFZSktvWklodmNOQVFFTEJRQXdGVEVUTUJFR0ExVUUKQXhNS2EzVmlaWEp1WlhSbGN6QWVGdzB5TURBNU1qa3dNVFE0TWpKYUZ3MHlNVEE1TWprd01UUTRNalJhTUVBeApGVEFUQmdOVkJBb1RESE41YzNSbGJUcHViMlJsY3pFbk1DVUdBMVVFQXhNZWMzbHpkR1Z0T201dlpHVTZNUzR4Ck5pMWpiMjUwY205c0xYQnNZVzVsTUlJQklqQU5CZ2txaGtpRzl3MEJBUUVGQUFPQ0FROEFNSUlCQ2dLQ0FRRUEKd0kva293bCtKTENyQWVIN2JpQkRhVlRKbEFBQ3ZBUzQwZ1lrZERSVnpQUzJzWU05MG5aUW5wUGtrR1EyWGFJMwpLVVNubHp1STJma2xPVEZldk9jekFnbzQ3RUlaSnQ3U3h6cFBETU54TTBKa2w4cXp0bWdwSjhSQzVjd1Nad3dkClVKR05VK3drQWpDTGVWRDY4blZmZTh5Ukk5Z3AvY2pNVnVBQ0pHSUtsSS9DVWZKbjA1US9URWc2WlBvamFSVWsKQkhjaEtyTmFneFVIeHpWRGZWQytJcmpwL0VsSDV1dXdrQWJRcVNydWdnMHZUaWFYRDRjT0hFZzJXNWdMQzNGYgplWmV1RmlZMlM3Vmh0MG54N2lmYkRSeHRvNW0wSXhmTFlSaXhyOHVOVkpoeFN2YnNxRFlTWFU3NVlzT1V3SE5CCnhOYkNPQmRwQzBMYkNTN0VRZG1wOFFJREFRQUJveWN3SlRBT0JnTlZIUThCQWY4RUJBTUNCYUF3RXdZRFZSMGwKQkF3d0NnWUlLd1lCQlFVSEF3SXdEUVlKS29aSWh2Y05BUUVMQlFBRGdnRUJBSEZXeG1GTFY0UXVvMzgvWWxpMwpGN2kyMithWXloMG5RYVZIQzBRSWN6cEltcnBQT0dSUkpoaEtJaS9ZMEx5QjE4TlNHN0xEUU9CQnhmQmdHS3VtClNua2cyYTE0NTY4REM2QSt1TTdvUWhJaVprcmVlRWlla2c1b3E4dm5IZlJjbGl1MUdqaDdVTWgrcXBuUEpiUS8Kc0FrNXlnOVFQR0ZUZG85UWg0aFpwV2I2WHNkcGU2dTFxUERSL0ROVjgwVlk3aWVkOG1zdStmV1g4K2pUak1nLwpHVGdkbmVUOUNWR092dkJzZksrWDRmSVd4L1psRjM4LzlLWXZzTmhUMDdKUnhVTDBQZ2Zkbjk1ZDZCZGVSdDNSCkF4cG9QM3ZyeUl3OXJieTB0TXR5cGZXMGZGbk1jemdzUFNMQzhtZjBBS2ROYzhtWnBZcFZQcnhqYUp4OUx0VlUKbDY4PQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0tCg== 19 | client-key-data: LS0tLS1CRUdJTiBSU0EgUFJJVkFURSBLRVktLS0tLQpNSUlFcFFJQkFBS0NBUUVBd0kva293bCtKTENyQWVIN2JpQkRhVlRKbEFBQ3ZBUzQwZ1lrZERSVnpQUzJzWU05CjBuWlFucFBra0dRMlhhSTNLVVNubHp1STJma2xPVEZldk9jekFnbzQ3RUlaSnQ3U3h6cFBETU54TTBKa2w4cXoKdG1ncEo4UkM1Y3dTWnd3ZFVKR05VK3drQWpDTGVWRDY4blZmZTh5Ukk5Z3AvY2pNVnVBQ0pHSUtsSS9DVWZKbgowNVEvVEVnNlpQb2phUlVrQkhjaEtyTmFneFVIeHpWRGZWQytJcmpwL0VsSDV1dXdrQWJRcVNydWdnMHZUaWFYCkQ0Y09IRWcyVzVnTEMzRmJlWmV1RmlZMlM3Vmh0MG54N2lmYkRSeHRvNW0wSXhmTFlSaXhyOHVOVkpoeFN2YnMKcURZU1hVNzVZc09Vd0hOQnhOYkNPQmRwQzBMYkNTN0VRZG1wOFFJREFRQUJBb0lCQVFDQWJyakJJVEM0U2ZrVQozV08wVk9LQlcyTERTcXFJT3VJR2VzQlBuU1JvK2dMc0xLdk1Oc3VZcU8xMmZOQmg3VEs3WEtZbTBUZDZvcGo0CkozeVhvK0M2aUpUa3gwVTlUcnJvMUhRckFDUDlXMWY2Z2VFUlZURzE2MG52T1FoQklLeGdzMk8reUs1UG80OHoKRUJjQUpkYlVuR0ZsdGVSejl6eUs2UWVrdXB0eDc3UFNwa2g3QnlhbW5ZZlVEK1hWdnFuczdPMHYxbjE1WW92NQp0VkVTWHlGT0RQR1Qwb0grV2IvNE13Umc0aXpvNHczNHM4RERuZ0dmUVlmRmVTd0d4azluMm5vME1pMEpSTUJYCnI1SFlYeWhuY2ZXMXh6bzM1clNMR3B5ckxRR2U5VjcvdDVabFIvbW5TVzl3Ri9CWjNQcm1DdnN3UjBwMTlBSlIKQnA0bXd1RVJBb0dCQU5ORFpPRlBiQkdiT1NPMTAyMXFnTzlUdmNBZ3ZWL2lKZXlnaExvMzd3SENaNHpOcktySwpvNHZJM0ZTcVpRN0pxL1h6SWhmeEtwTXpNMklnM21LekdqQUpnYnNyR1VlalhuQXo5aFk4YUs2aTc2WVpyeGNPCjNXcElkY3piaWtPb0swRUxlQnVoQ3JlbFFiSDI1dnRLdEtxWUlUZThJdkc2TXZNcFUvV3JOWkpkQW9HQkFPbFcKdE1VbE9vcDhXcGlGeXd5UnpPcnRvTmhMR3pPNTBad2Y4T2J1c01MeVE1cGhLL1pPM2dCeGtqUEZodDBKbXhtcwpLRDRwQzQweGduaWdJaWtsRTBYQnRHWjBSb055OGgrYnJoYWx6NWRMaVdKRjM5QmU3Smd5VzVpcUpjckJHYnpCCkgzNDNPNUppaHRYYUdXYU1tMTY2WmxjU3NNbkZYMWdCY1dkaFhtU2xBb0dCQUxOZldWdmo3MEhUcFdRb2tXY3AKTW1nQmxlNTIzZE4zVS9QaEpsQm1CVXhkSDBaeHF0VW9VRXZ6TXYrLzNTWDlIVUFxT3h4UTYzRW4yOHpKVTRoQgp3VTVnQkFKQ3lhZ0ZrYjE2b01xb1o4ay9GbkxWQTlCMEVwS1hDMDN3YVNpZ3RIajVuL1QxZXdBS0ZBbGlOU1BQCmZpNEtGTW9adHlHK2tsdmJEeXBiUDVVUkFvR0JBSWpRNFQrelc0YjE4VHo3UGhxSnB1aWRHckdZSUVRTkVWMkUKMFFEbk5kZW9xNERxdHhjK0dTZVRjQkZSSVltYlowai9TNzFlM2JvVkxKWkQ5cU8vYkhSN1pxYW9TT3dzU3RIRAp1NmpsNGptby9jNFVnRzR5S25IM0hRUC95QmNCY0hEZm4vS211aFJVRnhGdlIzTjh3R0VqMUh2N3BwTTRXblpFCnFsVGVuN1ZOQW9HQVB3aGU1WUIrZlhldzZUWGlQWGFtTjNvZjJ1L1Z6MjlKKzgxTE9UenRBSUEyRVhaa3k0N2UKcG94anVxaEFFYWFNbGtFL1hvKzZHenBnTHI2dWxGQjBLVXRhbmtEdjdmM2xXMy91RmgvN2dBbEJxazlZSmovTQpxVC9xdG82WCsrMmdkQkkwVWxZKzQ2SGpKcXVUNEVUVFZTSTVsWFNuSHFBdXVkcm1HWThNT3MwPQotLS0tLUVORCBSU0EgUFJJVkFURSBLRVktLS0tLQo= 20 | -------------------------------------------------------------------------------- /pkg/pki/conf/kubelet/tests/1.17-kubelet.conf: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | clusters: 3 | - cluster: 4 | certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUN5RENDQWJDZ0F3SUJBZ0lCQURBTkJna3Foa2lHOXcwQkFRc0ZBREFWTVJNd0VRWURWUVFERXdwcmRXSmwKY201bGRHVnpNQjRYRFRJd01Ea3lPVEF4TlRVeE4xb1hEVE13TURreU56QXhOVFV4TjFvd0ZURVRNQkVHQTFVRQpBeE1LYTNWaVpYSnVaWFJsY3pDQ0FTSXdEUVlKS29aSWh2Y05BUUVCQlFBRGdnRVBBRENDQVFvQ2dnRUJBTFlWCkFlOTk3bDAzTUx2SnZlUjdjZHUrWkdxNFRiSEMzcFA1VHdHWjhCTFRVY0lwakRYaFRiQkxjaFRlRVB1YmpoblAKenh1ZlVLTTlHU1lSK3dvdEhuVm1yNkppTHU5bGY0S1puUDI5N3lMRG5Wbm1YQ05jS2JtOEhSM0Q3d3l5UGcrMApHQzhQbUhLTElKb3dpUVNMQzlYWGprZU9YbXFqTDVFZ3FURHZuVGVKNk5OUytmSkJQNDh0bFJXVTAvdmpEL1ZzCm5SRFhaS3BQbXEzNFRYSHFIcDM3WGpsaWpPcVFJSmY0NDJPZnVCeWFxNUREdTZHbzhRMVZVaGh3ZmdYN3dNMmgKMk9RcXVlaWJCV1RxeWpOUHdwaFRmb0dIVFEvNVlmbFF0Z3ZhWmUxUm1PeStPbnJoQkErRExMb21qVWF4QW9saQpSbk9Kc1FkOHVOazc5TWVtUTlrQ0F3RUFBYU1qTUNFd0RnWURWUjBQQVFIL0JBUURBZ0trTUE4R0ExVWRFd0VCCi93UUZNQU1CQWY4d0RRWUpLb1pJaHZjTkFRRUxCUUFEZ2dFQkFERVdPTktIRExkd0VsRkdheXVHT0ZlWVNxNWMKV2o5U2R6eDlWUWo1TXVKYUhLbmR0cnBibjhWd051Zjd2b0xQVjRxazJSRmNyQUFLYzlBcjRpVnUzbXRrVUhuWQo2YjlvdUNvOVYyWmNQbk5XOWpyWUMwYm1UckdycFZTWFhPWG1VdjlnUk51d0p0SWFvYUtOYW5ieHhLQzNYeFZRCndhMm5JQ0JhSWJ6M0V1TWNwbEVXR3ZHbmYzNFdjODRKcXZDdGxNZ2FtOEpIY0g2K04wcS9hMTZLK1BtN0JGUk0KNDM0MzI2dk01aUhWM2lGWTA2SFlNeXQvL2VVOERiRVlYZ3pSQ1VudGhOUHltSWNWQW9LQXJ1MlVSUUxGbnFocwp5aEwxT0xtZW0vall6d0F1S1ZUWXZURXdWcG53UlBaNXhTUkwvSjlRS1BoSS9ZYTFUS3VYTkwwNng0VT0KLS0tLS1FTkQgQ0VSVElGSUNBVEUtLS0tLQo= 5 | server: https://1.17-control-plane:6443 6 | name: "1.17" 7 | contexts: 8 | - context: 9 | cluster: "1.17" 10 | user: system:node:1.17-control-plane 11 | name: system:node:1.17-control-plane@1.17 12 | current-context: system:node:1.17-control-plane@1.17 13 | kind: Config 14 | preferences: {} 15 | users: 16 | - name: system:node:1.17-control-plane 17 | user: 18 | client-certificate: /var/lib/kubelet/pki/kubelet-client-current.pem 19 | client-key: /var/lib/kubelet/pki/kubelet-client-current.pem 20 | -------------------------------------------------------------------------------- /pkg/pki/conf/kubelet/tests/client-kubelet.yaml: -------------------------------------------------------------------------------- 1 | address: 0.0.0.0 2 | apiVersion: kubelet.config.k8s.io/v1beta1 3 | authentication: 4 | anonymous: 5 | enabled: false 6 | webhook: 7 | cacheTTL: 2m0s 8 | enabled: true 9 | x509: 10 | clientCAFile: /etc/kubernetes/pki/ca.crt 11 | authorization: 12 | mode: Webhook 13 | webhook: 14 | cacheAuthorizedTTL: 5m0s 15 | cacheUnauthorizedTTL: 30s 16 | cgroupDriver: cgroupfs 17 | cgroupsPerQOS: true 18 | clusterDNS: 19 | - 10.96.0.10 20 | clusterDomain: cluster.local 21 | configMapAndSecretChangeDetectionStrategy: Watch 22 | containerLogMaxFiles: 5 23 | containerLogMaxSize: 10Mi 24 | contentType: application/vnd.kubernetes.protobuf 25 | cpuCFSQuota: true 26 | cpuCFSQuotaPeriod: 100ms 27 | cpuManagerPolicy: none 28 | cpuManagerReconcilePeriod: 10s 29 | enableControllerAttachDetach: true 30 | enableDebuggingHandlers: true 31 | enforceNodeAllocatable: 32 | - pods 33 | eventBurst: 10 34 | eventRecordQPS: 5 35 | evictionHard: 36 | imagefs.available: 0% 37 | nodefs.available: 0% 38 | nodefs.inodesFree: 0% 39 | evictionPressureTransitionPeriod: 5m0s 40 | failSwapOn: true 41 | fileCheckFrequency: 20s 42 | hairpinMode: promiscuous-bridge 43 | healthzBindAddress: 127.0.0.1 44 | healthzPort: 10248 45 | httpCheckFrequency: 20s 46 | imageGCHighThresholdPercent: 100 47 | imageGCLowThresholdPercent: 80 48 | imageMinimumGCAge: 2m0s 49 | iptablesDropBit: 15 50 | iptablesMasqueradeBit: 14 51 | kind: KubeletConfiguration 52 | kubeAPIBurst: 10 53 | kubeAPIQPS: 5 54 | makeIPTablesUtilChains: true 55 | maxOpenFiles: 1000000 56 | maxPods: 110 57 | nodeLeaseDurationSeconds: 40 58 | nodeStatusReportFrequency: 1m0s 59 | nodeStatusUpdateFrequency: 10s 60 | oomScoreAdj: -999 61 | podPidsLimit: -1 62 | port: 10250 63 | registryBurst: 10 64 | registryPullQPS: 5 65 | resolvConf: /etc/resolv.conf 66 | rotateCertificates: true 67 | runtimeRequestTimeout: 2m0s 68 | serializeImagePulls: true 69 | staticPodPath: /etc/kubernetes/manifests 70 | streamingConnectionIdleTimeout: 4h0m0s 71 | syncFrequency: 1m0s 72 | topologyManagerPolicy: none 73 | volumeStatsAggPeriod: 1m0s 74 | -------------------------------------------------------------------------------- /pkg/pki/conf/kubelet/tests/server-kubelet.yaml: -------------------------------------------------------------------------------- 1 | address: 0.0.0.0 2 | apiVersion: kubelet.config.k8s.io/v1beta1 3 | authentication: 4 | anonymous: 5 | enabled: false 6 | webhook: 7 | cacheTTL: 2m0s 8 | enabled: true 9 | x509: 10 | clientCAFile: /etc/kubernetes/pki/ca.crt 11 | authorization: 12 | mode: Webhook 13 | webhook: 14 | cacheAuthorizedTTL: 5m0s 15 | cacheUnauthorizedTTL: 30s 16 | cgroupDriver: cgroupfs 17 | cgroupsPerQOS: true 18 | clusterDNS: 19 | - 10.96.0.10 20 | clusterDomain: cluster.local 21 | configMapAndSecretChangeDetectionStrategy: Watch 22 | containerLogMaxFiles: 5 23 | containerLogMaxSize: 10Mi 24 | contentType: application/vnd.kubernetes.protobuf 25 | cpuCFSQuota: true 26 | cpuCFSQuotaPeriod: 100ms 27 | cpuManagerPolicy: none 28 | cpuManagerReconcilePeriod: 10s 29 | enableControllerAttachDetach: true 30 | enableDebuggingHandlers: true 31 | enforceNodeAllocatable: 32 | - pods 33 | eventBurst: 10 34 | eventRecordQPS: 5 35 | evictionHard: 36 | imagefs.available: 0% 37 | nodefs.available: 0% 38 | nodefs.inodesFree: 0% 39 | evictionPressureTransitionPeriod: 5m0s 40 | failSwapOn: true 41 | fileCheckFrequency: 20s 42 | hairpinMode: promiscuous-bridge 43 | healthzBindAddress: 127.0.0.1 44 | healthzPort: 10248 45 | httpCheckFrequency: 20s 46 | imageGCHighThresholdPercent: 100 47 | imageGCLowThresholdPercent: 80 48 | imageMinimumGCAge: 2m0s 49 | iptablesDropBit: 15 50 | iptablesMasqueradeBit: 14 51 | kind: KubeletConfiguration 52 | kubeAPIBurst: 10 53 | kubeAPIQPS: 5 54 | makeIPTablesUtilChains: true 55 | maxOpenFiles: 1000000 56 | maxPods: 110 57 | nodeLeaseDurationSeconds: 40 58 | nodeStatusReportFrequency: 1m0s 59 | nodeStatusUpdateFrequency: 10s 60 | oomScoreAdj: -999 61 | podPidsLimit: -1 62 | port: 10250 63 | registryBurst: 10 64 | registryPullQPS: 5 65 | resolvConf: /etc/resolv.conf 66 | serverTLSBootstrap: true 67 | runtimeRequestTimeout: 2m0s 68 | serializeImagePulls: true 69 | staticPodPath: /etc/kubernetes/manifests 70 | streamingConnectionIdleTimeout: 4h0m0s 71 | syncFrequency: 1m0s 72 | topologyManagerPolicy: none 73 | volumeStatsAggPeriod: 1m0s 74 | -------------------------------------------------------------------------------- /pkg/pki/node/node.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright (c) 2020 SUSE LLC. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package node 18 | 19 | import ( 20 | "time" 21 | 22 | "github.com/jenting/kucero/pkg/pki/cert" 23 | "github.com/jenting/kucero/pkg/pki/cert/kubeadm" 24 | "github.com/jenting/kucero/pkg/pki/cert/null" 25 | "github.com/jenting/kucero/pkg/pki/conf" 26 | "github.com/jenting/kucero/pkg/pki/conf/kubelet" 27 | ) 28 | 29 | type Node struct { 30 | conf.Config // configureation interface 31 | cert.Certificate // certificate interface 32 | } 33 | 34 | // New checks if it's a control plane node or worker node 35 | // then returns the corresponding node interface 36 | func New(isControlPlane bool, name string, expiryTimeToRotate time.Duration, enableKubeletClientCertRotation, enableKubeletServerCertRotation bool) *Node { 37 | if isControlPlane { 38 | return &Node{ 39 | Config: kubelet.New(name, enableKubeletClientCertRotation, enableKubeletServerCertRotation), 40 | Certificate: kubeadm.New(name, expiryTimeToRotate), 41 | } 42 | } 43 | return &Node{ 44 | Config: kubelet.New(name, enableKubeletClientCertRotation, enableKubeletServerCertRotation), 45 | Certificate: null.New(name, expiryTimeToRotate), 46 | } 47 | } 48 | -------------------------------------------------------------------------------- /pkg/pki/signer/ca_provider.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2020 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package signer 18 | 19 | import ( 20 | "bytes" 21 | "crypto" 22 | "fmt" 23 | "sync/atomic" 24 | "time" 25 | 26 | "k8s.io/apiserver/pkg/server/dynamiccertificates" 27 | "k8s.io/client-go/util/cert" 28 | "k8s.io/client-go/util/keyutil" 29 | 30 | "github.com/jenting/kucero/pkg/pki/authority" 31 | ) 32 | 33 | func newCAProvider(caFile, caKeyFile string) (*caProvider, error) { 34 | caLoader, err := dynamiccertificates.NewDynamicServingContentFromFiles("csr-controller", caFile, caKeyFile) 35 | if err != nil { 36 | return nil, fmt.Errorf("error reading CA cert file %q: %v", caFile, err) 37 | } 38 | 39 | ret := &caProvider{ 40 | caLoader: caLoader, 41 | } 42 | if err := ret.setCA(); err != nil { 43 | return nil, err 44 | } 45 | 46 | return ret, nil 47 | } 48 | 49 | type caProvider struct { 50 | caValue atomic.Value 51 | caLoader *dynamiccertificates.DynamicCertKeyPairContent 52 | } 53 | 54 | // setCA unconditionally stores the current cert/key content 55 | func (p *caProvider) setCA() error { 56 | certPEM, keyPEM := p.caLoader.CurrentCertKeyContent() 57 | 58 | certs, err := cert.ParseCertsPEM(certPEM) 59 | if err != nil { 60 | return fmt.Errorf("error reading CA cert file %q: %v", p.caLoader.Name(), err) 61 | } 62 | if len(certs) != 1 { 63 | return fmt.Errorf("error reading CA cert file %q: expected 1 certificate, found %d", p.caLoader.Name(), len(certs)) 64 | } 65 | 66 | key, err := keyutil.ParsePrivateKeyPEM(keyPEM) 67 | if err != nil { 68 | return fmt.Errorf("error reading CA key file %q: %v", p.caLoader.Name(), err) 69 | } 70 | priv, ok := key.(crypto.Signer) 71 | if !ok { 72 | return fmt.Errorf("error reading CA key file %q: key did not implement crypto.Signer", p.caLoader.Name()) 73 | } 74 | 75 | ca := &authority.CertificateAuthority{ 76 | RawCert: certPEM, 77 | RawKey: keyPEM, 78 | 79 | Certificate: certs[0], 80 | PrivateKey: priv, 81 | Backdate: 5 * time.Minute, 82 | } 83 | p.caValue.Store(ca) 84 | 85 | return nil 86 | } 87 | 88 | // currentCA provides the curent value of the CA. 89 | // It always check for a stale value. This is cheap because it's all an in memory cache of small slices. 90 | func (p *caProvider) currentCA() (*authority.CertificateAuthority, error) { 91 | certPEM, keyPEM := p.caLoader.CurrentCertKeyContent() 92 | currCA := p.caValue.Load().(*authority.CertificateAuthority) 93 | if bytes.Equal(currCA.RawCert, certPEM) && bytes.Equal(currCA.RawKey, keyPEM) { 94 | return currCA, nil 95 | } 96 | 97 | // the bytes weren't equal, so we have to set and then load 98 | if err := p.setCA(); err != nil { 99 | return currCA, err 100 | } 101 | return p.caValue.Load().(*authority.CertificateAuthority), nil 102 | } 103 | -------------------------------------------------------------------------------- /pkg/pki/signer/signer.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2019 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | // Package signer implements a CA signer that uses keys stored on local disk. 18 | package signer 19 | 20 | import ( 21 | "crypto/x509" 22 | "encoding/pem" 23 | "time" 24 | 25 | capi "k8s.io/api/certificates/v1" 26 | "k8s.io/client-go/util/certificate/csr" 27 | 28 | "github.com/jenting/kucero/pkg/pki/authority" 29 | ) 30 | 31 | type Signer struct { 32 | caProvider *caProvider 33 | certTTL time.Duration 34 | } 35 | 36 | func NewSigner(caFile, caKeyFile string, duration time.Duration) (*Signer, error) { 37 | caProvider, err := newCAProvider(caFile, caKeyFile) 38 | if err != nil { 39 | return nil, err 40 | } 41 | 42 | ret := &Signer{ 43 | caProvider: caProvider, 44 | certTTL: duration, 45 | } 46 | return ret, nil 47 | } 48 | 49 | func (s *Signer) Sign(x509cr *x509.CertificateRequest, spec capi.CertificateSigningRequestSpec) ([]byte, error) { 50 | currCA, err := s.caProvider.currentCA() 51 | if err != nil { 52 | return nil, err 53 | } 54 | der, err := currCA.Sign(x509cr.Raw, authority.PermissiveSigningPolicy{ 55 | TTL: s.duration(spec.ExpirationSeconds), 56 | Usages: spec.Usages, 57 | }) 58 | if err != nil { 59 | return nil, err 60 | } 61 | return pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der}), nil 62 | } 63 | 64 | func (s *Signer) duration(expirationSeconds *int32) time.Duration { 65 | if expirationSeconds == nil { 66 | return s.certTTL 67 | } 68 | 69 | // honor requested duration is if it is less than the default TTL 70 | // use 10 min (2x hard coded backdate above) as a sanity check lower bound 71 | const min = 10 * time.Minute 72 | switch requestedDuration := csr.ExpirationSecondsToDuration(*expirationSeconds); { 73 | case requestedDuration > s.certTTL: 74 | return s.certTTL 75 | case requestedDuration < min: 76 | return min 77 | default: 78 | return requestedDuration 79 | } 80 | } 81 | --------------------------------------------------------------------------------