├── LICENSE ├── README.md ├── build └── tools │ └── Dockerfile ├── deploy └── helm │ └── kubernetes │ ├── Chart.yaml │ ├── README.md │ ├── manifests │ ├── coredns.yaml │ ├── konnectivity-agent-deployment.yaml │ ├── konnectivity-agent-rbac.yaml │ └── konnectivity-server-rbac.yaml │ ├── scripts │ └── configure-cluster.sh │ ├── templates │ ├── NOTES.txt │ ├── _helpers.tpl │ ├── admin-configmap.yaml │ ├── admin-deployment.yaml │ ├── apiserver-config.yaml │ ├── apiserver-deployment.yaml │ ├── apiserver-service.yaml │ ├── controller-manager-configmap.yaml │ ├── controller-manager-deployment.yaml │ ├── controller-manager-service.yaml │ ├── etcd-backup-cronjob.yaml │ ├── etcd-backup-persistentvolumeclaim.yaml │ ├── etcd-certs.yaml │ ├── etcd-service.yaml │ ├── etcd-statefulset.yaml │ ├── extra-manifests.yaml │ ├── konnectivity-certs.yaml │ ├── konnectivity-manifests.yaml │ ├── konnectivity-server-configmap.yaml │ ├── konnectivity-server-deployment.yaml │ ├── konnectivity-server-service.yaml │ ├── kubeadm-config.yaml │ ├── kubeadm-cronjob.yaml │ ├── kubeadm-job.yaml │ ├── kubeadm-scripts.yaml │ ├── kubedns-manifests.yaml │ ├── kubernetes-certs.yaml │ ├── kubernetes-front-proxy-certs.yaml │ ├── scheduler-configmap.yaml │ ├── scheduler-deployment.yaml │ ├── scheduler-service.yaml │ └── selfsigning-issuer.yaml │ └── values.yaml ├── docs └── README.md └── hack └── version-bump.sh /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright [yyyy] [name of copyright owner] 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Kubernetes-in-Kubernetes 2 | 3 | Deploy Kubernetes in Kubernetes using Helm 4 | 5 | ![demo](https://gist.githubusercontent.com/kvaps/3cc5d772d750f8f2d36a76d00c3342b1/raw/8596ae812e83d186d0d57d448159b05f84cb0d53/kubernetes-in-kubernetes.gif) 6 | 7 | ## Requirements 8 | 9 | * Kubernetes v1.21+ 10 | * Helm v3 11 | * cert-manager v1.0.0+ 12 | 13 | ## Quick Start 14 | 15 | ### Preparation 16 | 17 | * Install [cert-manager]. 18 | 19 | * If you running over [minikube] you might also need to install a provisioner, you can use [local-path-provisioner] for example. 20 | 21 | [cert-manager]: https://cert-manager.io/docs/installation 22 | [minikube]: https://github.com/kubernetes/minikube 23 | [local-path-provisioner]: https://github.com/rancher/local-path-provisioner#installation 24 | 25 | ### Installation 26 | 27 | ```bash 28 | helm repo add kvaps https://kvaps.github.io/charts 29 | helm install foo kvaps/kubernetes --version 0.13.5 \ 30 | --namespace foo \ 31 | --create-namespace \ 32 | --set persistence.storageClassName=local-path 33 | ``` 34 | 35 | ### Cleanup 36 | 37 | ```bash 38 | kubectl delete namespace foo 39 | ``` 40 | 41 | ## Usage 42 | 43 | Kubernetes-in-Kubernetes is just a control plane, in most cases it's useless without workers. 44 | If you're looking for a real use case, check out the following projects that implement worker nodes management: 45 | 46 | * **[Kubefarm]** - Automated Kubernetes deployment and the PXE-bootable servers farm 47 | 48 | [kubefarm]: https://github.com/kubefarm/kubefarm 49 | -------------------------------------------------------------------------------- /build/tools/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM docker.io/library/alpine:3.14 2 | 3 | RUN apk add --no-cache openssl 4 | ARG VERSION=v1.22.4 5 | RUN wget "https://storage.googleapis.com/kubernetes-release/release/${VERSION}/bin/linux/amd64/kubectl" \ 6 | -O /usr/local/bin/kubectl \ 7 | && chmod +x /usr/local/bin/kubectl 8 | RUN wget "https://storage.googleapis.com/kubernetes-release/release/${VERSION}/bin/linux/amd64/kubeadm" \ 9 | -O /usr/local/bin/kubeadm \ 10 | && chmod +x /usr/local/bin/kubeadm 11 | -------------------------------------------------------------------------------- /deploy/helm/kubernetes/Chart.yaml: -------------------------------------------------------------------------------- 1 | name: kubernetes 2 | description: Production-Grade Container Scheduling and Management 3 | version: 0.13.5 4 | appVersion: 1.22.4 5 | icon: https://upload.wikimedia.org/wikipedia/commons/thumb/3/39/Kubernetes_logo_without_workmark.svg/723px-Kubernetes_logo_without_workmark.svg.png 6 | keywords: 7 | - kubernetes 8 | - go 9 | - cncf 10 | - containers 11 | home: https://github.com/kubefarm/kubernetes-in-kubernetes 12 | sources: 13 | - https://github.com/kubefarm/kubernetes-in-kubernetes 14 | - https://github.com/kubernetes/kubernetes 15 | maintainers: 16 | - name: kvaps 17 | email: kvapss@gmail.com 18 | - name: krakazyabra 19 | email: pronin.egor@gmail.com 20 | - name: mrakopes 21 | email: dave@mtfbwy.cz 22 | -------------------------------------------------------------------------------- /deploy/helm/kubernetes/README.md: -------------------------------------------------------------------------------- 1 | ../../../README.md -------------------------------------------------------------------------------- /deploy/helm/kubernetes/manifests/coredns.yaml: -------------------------------------------------------------------------------- 1 | # Source: https://github.com/kubernetes/kubernetes/blob/master/cluster/addons/dns/coredns/coredns.yaml.base 2 | --- 3 | apiVersion: v1 4 | kind: ServiceAccount 5 | metadata: 6 | name: coredns 7 | namespace: kube-system 8 | labels: 9 | kubernetes.io/cluster-service: "true" 10 | addonmanager.kubernetes.io/mode: Reconcile 11 | --- 12 | apiVersion: rbac.authorization.k8s.io/v1 13 | kind: ClusterRole 14 | metadata: 15 | labels: 16 | kubernetes.io/bootstrapping: rbac-defaults 17 | addonmanager.kubernetes.io/mode: Reconcile 18 | name: system:coredns 19 | rules: 20 | - apiGroups: 21 | - "" 22 | resources: 23 | - endpoints 24 | - services 25 | - pods 26 | - namespaces 27 | verbs: 28 | - list 29 | - watch 30 | - apiGroups: 31 | - "" 32 | resources: 33 | - nodes 34 | verbs: 35 | - get 36 | - apiGroups: 37 | - discovery.k8s.io 38 | resources: 39 | - endpointslices 40 | verbs: 41 | - list 42 | - watch 43 | --- 44 | apiVersion: rbac.authorization.k8s.io/v1 45 | kind: ClusterRoleBinding 46 | metadata: 47 | annotations: 48 | rbac.authorization.kubernetes.io/autoupdate: "true" 49 | labels: 50 | kubernetes.io/bootstrapping: rbac-defaults 51 | addonmanager.kubernetes.io/mode: EnsureExists 52 | name: system:coredns 53 | roleRef: 54 | apiGroup: rbac.authorization.k8s.io 55 | kind: ClusterRole 56 | name: system:coredns 57 | subjects: 58 | - kind: ServiceAccount 59 | name: coredns 60 | namespace: kube-system 61 | --- 62 | apiVersion: v1 63 | kind: ConfigMap 64 | metadata: 65 | name: coredns 66 | namespace: kube-system 67 | data: 68 | Corefile: | 69 | .:53 { 70 | errors 71 | health { 72 | lameduck 5s 73 | } 74 | ready 75 | kubernetes {{ .Values.networking.dnsDomain }} in-addr.arpa ip6.arpa { 76 | pods insecure 77 | fallthrough in-addr.arpa ip6.arpa 78 | ttl 30 79 | } 80 | prometheus :9153 81 | forward . /etc/resolv.conf { 82 | max_concurrent 1000 83 | } 84 | cache 30 85 | loop 86 | reload 87 | loadbalance 88 | } 89 | --- 90 | apiVersion: apps/v1 91 | kind: Deployment 92 | metadata: 93 | name: coredns 94 | namespace: kube-system 95 | labels: 96 | k8s-app: kube-dns 97 | kubernetes.io/cluster-service: "true" 98 | addonmanager.kubernetes.io/mode: Reconcile 99 | kubernetes.io/name: "CoreDNS" 100 | spec: 101 | replicas: {{ .Values.coredns.replicaCount }} 102 | strategy: 103 | type: RollingUpdate 104 | rollingUpdate: 105 | maxUnavailable: 1 106 | selector: 107 | matchLabels: 108 | k8s-app: kube-dns 109 | template: 110 | metadata: 111 | labels: 112 | k8s-app: kube-dns 113 | spec: 114 | securityContext: 115 | seccompProfile: 116 | type: RuntimeDefault 117 | priorityClassName: system-cluster-critical 118 | serviceAccountName: coredns 119 | affinity: 120 | podAntiAffinity: 121 | preferredDuringSchedulingIgnoredDuringExecution: 122 | - weight: 100 123 | podAffinityTerm: 124 | labelSelector: 125 | matchExpressions: 126 | - key: k8s-app 127 | operator: In 128 | values: ["kube-dns"] 129 | topologyKey: kubernetes.io/hostname 130 | tolerations: 131 | - key: "CriticalAddonsOnly" 132 | operator: "Exists" 133 | nodeSelector: 134 | kubernetes.io/os: linux 135 | {{- with .Values.coredns.image.pullSecrets }} 136 | imagePullSecrets: 137 | {{- toYaml . | nindent 10 }} 138 | {{- end }} 139 | containers: 140 | - name: coredns 141 | {{- with .Values.coredns.image }} 142 | image: "{{ .repository }}{{ if .digest }}@{{ .digest }}{{ else }}:{{ .tag }}{{ end }}" 143 | imagePullPolicy: {{ .pullPolicy }} 144 | {{- end }} 145 | resources: 146 | {{- toYaml .Values.coredns.resources | nindent 10 }} 147 | args: [ "-conf", "/etc/coredns/Corefile" ] 148 | volumeMounts: 149 | - name: config-volume 150 | mountPath: /etc/coredns 151 | readOnly: true 152 | ports: 153 | - containerPort: 53 154 | name: dns 155 | protocol: UDP 156 | - containerPort: 53 157 | name: dns-tcp 158 | protocol: TCP 159 | - containerPort: 9153 160 | name: metrics 161 | protocol: TCP 162 | livenessProbe: 163 | httpGet: 164 | path: /health 165 | port: 8080 166 | scheme: HTTP 167 | initialDelaySeconds: 60 168 | timeoutSeconds: 5 169 | successThreshold: 1 170 | failureThreshold: 5 171 | readinessProbe: 172 | httpGet: 173 | path: /ready 174 | port: 8181 175 | scheme: HTTP 176 | securityContext: 177 | allowPrivilegeEscalation: false 178 | capabilities: 179 | add: 180 | - NET_BIND_SERVICE 181 | drop: 182 | - all 183 | readOnlyRootFilesystem: true 184 | dnsPolicy: Default 185 | volumes: 186 | - name: config-volume 187 | configMap: 188 | name: coredns 189 | items: 190 | - key: Corefile 191 | path: Corefile 192 | --- 193 | apiVersion: v1 194 | kind: Service 195 | metadata: 196 | name: kube-dns 197 | namespace: kube-system 198 | annotations: 199 | prometheus.io/port: "9153" 200 | prometheus.io/scrape: "true" 201 | labels: 202 | k8s-app: kube-dns 203 | kubernetes.io/cluster-service: "true" 204 | addonmanager.kubernetes.io/mode: Reconcile 205 | kubernetes.io/name: "CoreDNS" 206 | spec: 207 | selector: 208 | k8s-app: kube-dns 209 | clusterIP: {{ template "kubernetes.getCoreDNS" . }} 210 | ports: 211 | - name: dns 212 | port: 53 213 | protocol: UDP 214 | - name: dns-tcp 215 | port: 53 216 | protocol: TCP 217 | - name: metrics 218 | port: 9153 219 | protocol: TCP 220 | -------------------------------------------------------------------------------- /deploy/helm/kubernetes/manifests/konnectivity-agent-deployment.yaml: -------------------------------------------------------------------------------- 1 | {{- $fullName := include "kubernetes.fullname" . -}} 2 | 3 | {{- if .Values.konnectivityAgent.daemonSet -}} 4 | apiVersion: apps/v1 5 | kind: DaemonSet 6 | {{- else -}} 7 | apiVersion: apps/v1 8 | kind: Deployment 9 | {{- end }} 10 | metadata: 11 | labels: 12 | addonmanager.kubernetes.io/mode: Reconcile 13 | k8s-app: konnectivity-agent 14 | {{- with .Values.konnectivityAgent.labels }} 15 | {{- toYaml . | nindent 4 }} 16 | {{- end }} 17 | {{- with .Values.konnectivityAgent.annotations }} 18 | annotations: 19 | {{- toYaml . | nindent 4 }} 20 | {{- end }} 21 | namespace: kube-system 22 | name: konnectivity-agent 23 | spec: 24 | {{- if not .Values.konnectivityAgent.daemonSet }} 25 | replicas: {{ .Values.konnectivityAgent.replicaCount }} 26 | {{- end }} 27 | selector: 28 | matchLabels: 29 | k8s-app: konnectivity-agent 30 | template: 31 | metadata: 32 | labels: 33 | k8s-app: konnectivity-agent 34 | {{- with .Values.konnectivityAgent.podLabels }} 35 | {{- toYaml . | nindent 8 }} 36 | {{- end }} 37 | {{- with .Values.konnectivityAgent.podAnnotations }} 38 | annotations: 39 | {{- toYaml . | nindent 8 }} 40 | {{- end }} 41 | spec: 42 | {{- with .Values.konnectivityAgent.nodeSelector }} 43 | nodeSelector: 44 | {{- toYaml . | nindent 8 }} 45 | {{- end }} 46 | hostNetwork: {{ .Values.konnectivityAgent.hostNetwork }} 47 | securityContext: 48 | seccompProfile: 49 | type: RuntimeDefault 50 | priorityClassName: system-cluster-critical 51 | tolerations: 52 | - key: "CriticalAddonsOnly" 53 | operator: "Exists" 54 | {{- with .Values.konnectivityAgent.tolerations }} 55 | {{- toYaml . | nindent 6 }} 56 | {{- end }} 57 | {{- if or .Values.konnectivityServer.affinity .Values.konnectivityServer.podAntiAffinity }} 58 | affinity: 59 | {{- with .Values.konnectivityServer.affinity }} 60 | {{- toYaml . | nindent 8 }} 61 | {{- end }} 62 | {{- if eq .Values.konnectivityServer.podAntiAffinity "hard" }} 63 | podAntiAffinity: 64 | requiredDuringSchedulingIgnoredDuringExecution: 65 | - topologyKey: "{{ .Values.konnectivityServer.podAntiAffinityTopologyKey }}" 66 | labelSelector: 67 | matchLabels: 68 | app: {{ $fullName }}-konnectivity-server 69 | {{- else if eq .Values.konnectivityServer.podAntiAffinity "soft" }} 70 | podAntiAffinity: 71 | preferredDuringSchedulingIgnoredDuringExecution: 72 | - weight: 1 73 | podAffinityTerm: 74 | topologyKey: "{{ .Values.konnectivityServer.podAntiAffinityTopologyKey }}" 75 | labelSelector: 76 | matchLabels: 77 | app: {{ $fullName }}-konnectivity-server 78 | {{- end }} 79 | {{- end }} 80 | {{- with .Values.konnectivityServer.image.pullSecrets }} 81 | imagePullSecrets: 82 | {{- toYaml . | nindent 10 }} 83 | {{- end }} 84 | containers: 85 | - name: konnectivity-agent 86 | {{- with .Values.konnectivityAgent.image }} 87 | image: "{{ .repository }}{{ if .digest }}@{{ .digest }}{{ else }}:{{ .tag }}{{ end }}" 88 | imagePullPolicy: {{ .pullPolicy }} 89 | {{- end }} 90 | command: 91 | - /proxy-agent 92 | - --logtostderr=true 93 | - --ca-cert=/var/run/secrets/kubernetes.io/serviceaccount/ca.crt 94 | - --service-account-token-path=/var/run/secrets/tokens/konnectivity-agent-token 95 | 96 | {{- if not (hasKey .Values.konnectivityAgent.extraArgs "proxy-server-host") }} 97 | {{- if and (eq .Values.konnectivityServer.mode "HTTPConnect") .Values.konnectivityServer.service.loadBalancerIP }} 98 | - --proxy-server-host={{ .Values.konnectivityServer.service.loadBalancerIP }} 99 | {{- else if and (eq .Values.konnectivityServer.mode "GRPC") .Values.apiServer.service.loadBalancerIP }} 100 | - --proxy-server-host={{ .Values.apiServer.service.loadBalancerIP }} 101 | {{- else }} 102 | {{- fail ".konnectivityAgent.extraArgs.proxy-server-host must be specified!" }} 103 | {{- end }} 104 | {{- end }} 105 | 106 | {{- if not (hasKey .Values.konnectivityAgent.extraArgs "proxy-server-port") }} 107 | {{- if eq .Values.konnectivityServer.service.type "LoadBalancer" }} 108 | - --proxy-server-port={{ .Values.konnectivityServer.service.ports.agent }} 109 | {{- else if .Values.konnectivityServer.service.NodePort }} 110 | - --proxy-server-port={{ .Values.konnectivityServer.service.nodePorts.agent }} 111 | {{- else }} 112 | {{- fail ".konnectivityAgent.extraArgs.proxy-server-port must be specified!" }} 113 | {{- end }} 114 | {{- end }} 115 | 116 | - --admin-server-port={{ .Values.konnectivityAgent.ports.admin }} 117 | - --health-server-port={{ .Values.konnectivityAgent.ports.health }} 118 | 119 | {{- range $key, $value := .Values.konnectivityAgent.extraArgs }} 120 | - --{{ $key }}={{ $value }} 121 | {{- end }} 122 | ports: 123 | - containerPort: {{ .Values.konnectivityAgent.ports.admin }} 124 | name: admin 125 | - containerPort: {{ .Values.konnectivityAgent.ports.health }} 126 | name: health 127 | volumeMounts: 128 | - mountPath: /var/run/secrets/tokens 129 | name: konnectivity-agent-token 130 | {{- with .Values.konnectivityAgent.extraVolumeMounts }} 131 | {{- toYaml . | nindent 8 }} 132 | {{- end }} 133 | livenessProbe: 134 | httpGet: 135 | path: /healthz 136 | port: {{ .Values.konnectivityAgent.ports.health }} 137 | scheme: HTTP 138 | initialDelaySeconds: 15 139 | timeoutSeconds: 15 140 | {{- with .Values.konnectivityAgent.sidecars }} 141 | {{- toYaml . | nindent 6 }} 142 | {{- end }} 143 | serviceAccountName: konnectivity-agent 144 | volumes: 145 | - name: konnectivity-agent-token 146 | projected: 147 | sources: 148 | - serviceAccountToken: 149 | path: konnectivity-agent-token 150 | audience: system:konnectivity-server 151 | {{- with .Values.konnectivityAgent.extraVolumes }} 152 | {{- toYaml . | nindent 6 }} 153 | {{- end }} 154 | -------------------------------------------------------------------------------- /deploy/helm/kubernetes/manifests/konnectivity-agent-rbac.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: ServiceAccount 3 | metadata: 4 | name: konnectivity-agent 5 | namespace: kube-system 6 | labels: 7 | kubernetes.io/cluster-service: "true" 8 | addonmanager.kubernetes.io/mode: Reconcile 9 | -------------------------------------------------------------------------------- /deploy/helm/kubernetes/manifests/konnectivity-server-rbac.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: rbac.authorization.k8s.io/v1 2 | kind: ClusterRoleBinding 3 | metadata: 4 | name: system:konnectivity-server 5 | labels: 6 | kubernetes.io/cluster-service: "true" 7 | addonmanager.kubernetes.io/mode: Reconcile 8 | roleRef: 9 | apiGroup: rbac.authorization.k8s.io 10 | kind: ClusterRole 11 | name: system:auth-delegator 12 | subjects: 13 | - apiGroup: rbac.authorization.k8s.io 14 | kind: User 15 | name: system:konnectivity-server 16 | -------------------------------------------------------------------------------- /deploy/helm/kubernetes/scripts/configure-cluster.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | set -e 3 | set -x 4 | ENDPOINT=$(awk -F'[ "]+' '$1 == "controlPlaneEndpoint:" {print $2}' /config/kubeadmcfg.yaml) 5 | 6 | # ------------------------------------------------------------------------------ 7 | # Update secrets and component configs 8 | # ------------------------------------------------------------------------------ 9 | 10 | # wait for cluster 11 | echo "Waiting for api-server endpoint ${ENDPOINT}..." 12 | until kubectl cluster-info >/dev/null 2>/dev/null; do 13 | sleep 1 14 | done 15 | 16 | # ------------------------------------------------------------------------------ 17 | # Cluster configuration 18 | # ------------------------------------------------------------------------------ 19 | export KUBECONFIG=/etc/kubernetes/admin.conf 20 | 21 | # upload configuration 22 | # TODO: https://github.com/kubefarm/kubernetes-in-kubernetes/issues/6 23 | kubeadm init phase upload-config kubeadm --config /config/kubeadmcfg.yaml 24 | 25 | # upload configuration 26 | # TODO: https://github.com/kubefarm/kubernetes-in-kubernetes/issues/5 27 | kubeadm init phase upload-config kubelet --config /config/kubeadmcfg.yaml -v1 2>&1 | 28 | while read line; do echo "$line" | grep 'Preserving the CRISocket information for the control-plane node' && killall kubeadm || echo "$line"; done 29 | 30 | # setup bootstrap-tokens 31 | # TODO: https://github.com/kubefarm/kubernetes-in-kubernetes/issues/7 32 | # TODO: https://github.com/kubernetes/kubernetes/issues/98881 33 | flatconfig=$(mktemp) 34 | kubectl config view --flatten > "$flatconfig" 35 | kubeadm init phase bootstrap-token --config /config/kubeadmcfg.yaml --skip-token-print --kubeconfig="$flatconfig" 36 | rm -f "$flatconfig" 37 | 38 | # correct apiserver address for the external clients 39 | kubectl apply -n kube-public -f - </dev/null || true 64 | {{- end }} 65 | 66 | {{- if .Values.konnectivityAgent.enabled }}{{"\n"}} 67 | # install konnectivity agent 68 | kubectl apply -f /manifests/konnectivity-agent-deployment.yaml -f /manifests/konnectivity-agent-rbac.yaml 69 | {{- else }}{{"\n"}} 70 | # uninstall konnectivity agent 71 | kubectl delete -f /manifests/konnectivity-agent-deployment.yaml -f /manifests/konnectivity-agent-rbac.yaml 2>/dev/null || true 72 | {{- end }} 73 | 74 | {{- if .Values.coredns.enabled }}{{"\n"}} 75 | # install coredns addon 76 | kubectl apply -f /manifests/coredns.yaml 77 | {{- else }}{{"\n"}} 78 | # uninstall coredns addon 79 | kubectl delete -f /manifests/coredns.yaml 2>/dev/null || true 80 | {{- end }} 81 | 82 | {{- if .Values.kubeProxy.enabled }}{{"\n"}} 83 | # install kube-proxy addon 84 | # TODO: https://github.com/kubefarm/kubernetes-in-kubernetes/issues/4 85 | kubeadm init phase addon kube-proxy --config /config/kubeadmcfg.yaml 86 | {{- else }}{{"\n"}} 87 | # uninstall kube-proxy addon 88 | kubectl -n kube-system delete configmap/kube-proxy daemonset/kube-proxy 2>/dev/null || true 89 | {{- end }} 90 | 91 | {{- with .Values.extraManifests }}{{"\n"}} 92 | kubectl apply{{- range $key, $value := . }} -f /manifests/{{ $key }}{{- end }} 93 | {{- end }} 94 | -------------------------------------------------------------------------------- /deploy/helm/kubernetes/templates/NOTES.txt: -------------------------------------------------------------------------------- 1 | {{- $fullName := include "kubernetes.fullname" . -}} 2 | {{- $cmd := printf "kubectl exec -n %s -ti deploy/%s-admin -- sh" .Release.Namespace $fullName -}} 3 | 1. {{ $fullName }} cluster deployed. 4 | 5 | Get shell in admin container: 6 | ┌─{{ "─" | repeat (len $cmd) }}─┐ 7 | │ {{ $cmd }} │ 8 | └─{{ "─" | repeat (len $cmd) }}─┘ 9 | -------------------------------------------------------------------------------- /deploy/helm/kubernetes/templates/_helpers.tpl: -------------------------------------------------------------------------------- 1 | {{/* vim: set filetype=gohtmltmpl: */}} 2 | {{/* 3 | Expand the name of the chart. 4 | */}} 5 | {{- define "kubernetes.name" -}} 6 | {{- default "kubernetes" .Values.nameOverride | trunc 63 | trimSuffix "-" -}} 7 | {{- end -}} 8 | 9 | {{/* 10 | Create a default fully qualified app name. 11 | We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). 12 | */}} 13 | {{- define "kubernetes.fullname" -}} 14 | {{- if .Values.fullnameOverride -}} 15 | {{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}} 16 | {{- else -}} 17 | {{- $name := default "kubernetes" .Values.nameOverride -}} 18 | {{- if or (eq $name .Release.Name) (eq (.Release.Name | upper) "RELEASE-NAME") -}} 19 | {{- $name | trunc 63 | trimSuffix "-" -}} 20 | {{- else -}} 21 | {{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} 22 | {{- end -}} 23 | {{- end -}} 24 | {{- end -}} 25 | 26 | {{/* 27 | Create a default certificate name. 28 | */}} 29 | {{- define "kubernetes.certname" -}} 30 | {{- if .Values.certnameOverride -}} 31 | {{- .Values.certnameOverride | trunc 63 | trimSuffix "-" -}} 32 | {{- else -}} 33 | {{- template "kubernetes.fullname" . -}} 34 | {{- end -}} 35 | {{- end -}} 36 | 37 | {{/* 38 | Generate etcd servers list. 39 | */}} 40 | {{- define "kubernetes.etcdEndpoints" -}} 41 | {{- $fullName := include "kubernetes.fullname" . -}} 42 | {{- range $etcdcount, $e := until (int .Values.etcd.replicaCount) -}} 43 | {{- printf "https://" -}} 44 | {{- printf "%s-etcd-%d." $fullName $etcdcount -}} 45 | {{- printf "%s-etcd:%d" $fullName (int $.Values.etcd.ports.client) -}} 46 | {{- if lt $etcdcount (sub (int $.Values.etcd.replicaCount) 1 ) -}} 47 | {{- printf "," -}} 48 | {{- end -}} 49 | {{- end -}} 50 | {{- end -}} 51 | 52 | {{- define "kubernetes.etcdInitialCluster" -}} 53 | {{- $fullName := include "kubernetes.fullname" . -}} 54 | {{- range $etcdcount, $e := until (int .Values.etcd.replicaCount) -}} 55 | {{- printf "%s-etcd-%d=" $fullName $etcdcount -}} 56 | {{- printf "https://" -}} 57 | {{- printf "%s-etcd-%d." $fullName $etcdcount -}} 58 | {{- printf "%s-etcd:%d" $fullName (int $.Values.etcd.ports.peer) -}} 59 | {{- if lt $etcdcount (sub (int $.Values.etcd.replicaCount) 1 ) -}} 60 | {{- printf "," -}} 61 | {{- end -}} 62 | {{- end -}} 63 | {{- end -}} 64 | 65 | {{/* 66 | Take the first IP address from the serviceSubnet for the kube-dns service. 67 | */}} 68 | {{- define "kubernetes.getCoreDNS" -}} 69 | {{- $octetsList := splitList "." .Values.networking.serviceSubnet -}} 70 | {{- printf "%d.%d.%d.%d" (index $octetsList 0 | int) (index $octetsList 1 | int) (index $octetsList 2 | int) 10 -}} 71 | {{- end -}} 72 | 73 | {{- define "kubernetes.getAPIAddress" -}} 74 | {{- $octetsList := splitList "." .Values.networking.serviceSubnet -}} 75 | {{- printf "%d.%d.%d.%d" (index $octetsList 0 | int) (index $octetsList 1 | int) (index $octetsList 2 | int) 1 -}} 76 | {{- end -}} 77 | 78 | {{/* 79 | Template for konnectivityServer containers 80 | */}} 81 | {{- define "kubernetes.konnectivityServer.containers" -}} 82 | - command: 83 | - /proxy-server 84 | - --logtostderr=true 85 | - --server-count={{ .Values.konnectivityServer.replicaCount }} 86 | - --server-id=$(POD_NAME) 87 | - --cluster-cert=/pki/apiserver/tls.crt 88 | - --cluster-key=/pki/apiserver/tls.key 89 | {{- if eq .Values.konnectivityServer.mode "HTTPConnect" }} 90 | - --mode=http-connect 91 | - --server-port={{ .Values.konnectivityServer.ports.server }} 92 | - --server-ca-cert=/pki/konnectivity-server/ca.crt 93 | - --server-cert=/pki/konnectivity-server/tls.crt 94 | - --server-key=/pki/konnectivity-server/tls.key 95 | {{- else }} 96 | - --mode=grpc 97 | - --uds-name=/run/konnectivity-server/konnectivity-server.socket 98 | - --server-port=0 99 | {{- end }} 100 | - --agent-port={{ .Values.konnectivityServer.ports.agent }} 101 | - --admin-port={{ .Values.konnectivityServer.ports.admin }} 102 | - --health-port={{ .Values.konnectivityServer.ports.health }} 103 | - --agent-namespace=kube-system 104 | - --agent-service-account=konnectivity-agent 105 | - --kubeconfig=/etc/kubernetes/konnectivity-server.conf 106 | - --authentication-audience=system:konnectivity-server 107 | {{- range $key, $value := .Values.konnectivityServer.extraArgs }} 108 | - --{{ $key }}={{ $value }} 109 | {{- end }} 110 | ports: 111 | {{- if eq .Values.konnectivityServer.mode "HTTPConnect" }} 112 | - containerPort: {{ .Values.konnectivityServer.ports.server }} 113 | name: server 114 | {{- end }} 115 | - containerPort: {{ .Values.konnectivityServer.ports.agent }} 116 | name: agent 117 | - containerPort: {{ .Values.konnectivityServer.ports.admin }} 118 | name: admin 119 | - containerPort: {{ .Values.konnectivityServer.ports.health }} 120 | name: health 121 | {{- with .Values.konnectivityServer.image }} 122 | image: "{{ .repository }}{{ if .digest }}@{{ .digest }}{{ else }}:{{ .tag }}{{ end }}" 123 | imagePullPolicy: {{ .pullPolicy }} 124 | {{- end }} 125 | livenessProbe: 126 | failureThreshold: 8 127 | httpGet: 128 | path: /healthz 129 | port: {{ .Values.konnectivityServer.ports.health }} 130 | scheme: HTTP 131 | initialDelaySeconds: 30 132 | timeoutSeconds: 60 133 | name: konnectivity-server 134 | resources: 135 | {{- toYaml .Values.konnectivityServer.resources | nindent 10 }} 136 | env: 137 | - name: POD_NAME 138 | valueFrom: 139 | fieldRef: 140 | fieldPath: metadata.name 141 | {{- with .Values.konnectivityServer.extraEnv }} 142 | {{- toYaml . | nindent 8 }} 143 | {{- end }} 144 | volumeMounts: 145 | - mountPath: /pki/apiserver 146 | name: pki-apiserver 147 | {{- if eq .Values.konnectivityServer.mode "HTTPConnect" }} 148 | - mountPath: /pki/konnectivity-server 149 | name: pki-konnectivity-server 150 | {{- else }} 151 | - mountPath: /run/konnectivity-server 152 | name: konnectivity-uds 153 | {{- end }} 154 | - mountPath: /pki/konnectivity-server-client 155 | name: pki-konnectivity-server-client 156 | - mountPath: /etc/kubernetes/ 157 | name: kubeconfig 158 | readOnly: true 159 | {{- with .Values.konnectivityServer.extraVolumeMounts }} 160 | {{- toYaml . | nindent 8 }} 161 | {{- end }} 162 | {{- end -}} 163 | 164 | {{/* 165 | Template for konnectivityServer volumes 166 | */}} 167 | {{- define "kubernetes.konnectivityServer.volumes" -}} 168 | - secret: 169 | secretName: "{{ template "kubernetes.fullname" . }}-pki-apiserver-server" 170 | name: pki-apiserver 171 | {{- if eq .Values.konnectivityServer.mode "HTTPConnect" }} 172 | - secret: 173 | secretName: "{{ template "kubernetes.fullname" . }}-pki-konnectivity-server" 174 | name: pki-konnectivity-server 175 | {{- else }} 176 | - secret: 177 | secretName: "{{ template "kubernetes.fullname" . }}-pki-konnectivity-server-client" 178 | name: pki-konnectivity-server-client 179 | - emptyDir: {} 180 | name: konnectivity-uds 181 | {{- end }} 182 | - configMap: 183 | name: "{{ template "kubernetes.fullname" . }}-konnectivity-server-conf" 184 | name: kubeconfig 185 | {{- with .Values.konnectivityServer.extraVolumes }} 186 | {{- toYaml . | nindent 6 }} 187 | {{- end }} 188 | {{- end -}} 189 | -------------------------------------------------------------------------------- /deploy/helm/kubernetes/templates/admin-configmap.yaml: -------------------------------------------------------------------------------- 1 | {{- if .Values.admin.enabled }} 2 | {{- $fullName := include "kubernetes.fullname" . -}} 3 | --- 4 | apiVersion: v1 5 | kind: ConfigMap 6 | metadata: 7 | name: {{ $fullName }}-admin-conf 8 | data: 9 | admin.conf: | 10 | apiVersion: v1 11 | clusters: 12 | - cluster: 13 | certificate-authority: /pki/admin-client/ca.crt 14 | server: https://{{ $fullName }}-apiserver:{{ .Values.apiServer.service.port }} 15 | name: default-cluster 16 | contexts: 17 | - context: 18 | cluster: default-cluster 19 | namespace: default 20 | user: default-auth 21 | name: default-context 22 | current-context: default-context 23 | kind: Config 24 | preferences: {} 25 | users: 26 | - name: default-auth 27 | user: 28 | client-certificate: /pki/admin-client/tls.crt 29 | client-key: /pki/admin-client/tls.key 30 | {{- end }} 31 | -------------------------------------------------------------------------------- /deploy/helm/kubernetes/templates/admin-deployment.yaml: -------------------------------------------------------------------------------- 1 | {{- if .Values.admin.enabled }} 2 | {{- $fullName := include "kubernetes.fullname" . -}} 3 | --- 4 | apiVersion: apps/v1 5 | kind: Deployment 6 | metadata: 7 | name: "{{ $fullName }}-admin" 8 | labels: 9 | app: "{{ $fullName }}-admin" 10 | {{- with .Values.admin.labels }} 11 | {{- toYaml . | nindent 4 }} 12 | {{- end }} 13 | {{- with .Values.admin.annotations }} 14 | annotations: 15 | {{- toYaml . | nindent 4 }} 16 | {{- end }} 17 | spec: 18 | replicas: {{ .Values.admin.replicaCount }} 19 | selector: 20 | matchLabels: 21 | app: "{{ $fullName }}-admin" 22 | template: 23 | metadata: 24 | labels: 25 | app: "{{ $fullName }}-admin" 26 | {{- with .Values.admin.podLabels }} 27 | {{- toYaml . | nindent 8 }} 28 | {{- end }} 29 | {{- with .Values.admin.podAnnotations }} 30 | annotations: 31 | {{- toYaml . | nindent 8 }} 32 | {{- end }} 33 | spec: 34 | {{- with .Values.admin.nodeSelector }} 35 | nodeSelector: 36 | {{- toYaml . | nindent 8 }} 37 | {{- end }} 38 | {{- with .Values.admin.tolerations }} 39 | tolerations: 40 | {{- toYaml . | nindent 6 }} 41 | {{- end }} 42 | {{- if or .Values.admin.affinity .Values.admin.podAntiAffinity }} 43 | affinity: 44 | {{- with .Values.admin.affinity }} 45 | {{- toYaml . | nindent 8 }} 46 | {{- end }} 47 | {{- if eq .Values.admin.podAntiAffinity "hard" }} 48 | podAntiAffinity: 49 | requiredDuringSchedulingIgnoredDuringExecution: 50 | - topologyKey: "{{ .Values.admin.podAntiAffinityTopologyKey }}" 51 | labelSelector: 52 | matchLabels: 53 | app: {{ $fullName }}-admin 54 | {{- else if eq .Values.admin.podAntiAffinity "soft" }} 55 | podAntiAffinity: 56 | preferredDuringSchedulingIgnoredDuringExecution: 57 | - weight: 1 58 | podAffinityTerm: 59 | topologyKey: "{{ .Values.admin.podAntiAffinityTopologyKey }}" 60 | labelSelector: 61 | matchLabels: 62 | app: {{ $fullName }}-admin 63 | {{- end }} 64 | {{- end }} 65 | {{- with .Values.admin.image.pullSecrets }} 66 | imagePullSecrets: 67 | {{- toYaml . | nindent 10 }} 68 | {{- end }} 69 | automountServiceAccountToken: false 70 | terminationGracePeriodSeconds: 5 71 | containers: 72 | - command: [ 'sleep', 'infinity' ] 73 | {{- with .Values.admin.image }} 74 | image: "{{ .repository }}{{ if .digest }}@{{ .digest }}{{ else }}:{{ .tag }}{{ end }}" 75 | imagePullPolicy: {{ .pullPolicy }} 76 | {{- end }} 77 | name: admin 78 | readinessProbe: 79 | exec: 80 | command: 81 | - kubectl 82 | - auth 83 | - can-i 84 | - '*' 85 | - '*' 86 | initialDelaySeconds: 15 87 | periodSeconds: 5 88 | resources: 89 | {{- toYaml .Values.admin.resources | nindent 10 }} 90 | env: 91 | - name: KUBECONFIG 92 | value: "/etc/kubernetes/admin.conf" 93 | {{- with .Values.admin.extraEnv }} 94 | {{- toYaml . | nindent 8 }} 95 | {{- end }} 96 | volumeMounts: 97 | - mountPath: /etc/kubernetes/ 98 | name: kubeconfig 99 | readOnly: true 100 | - mountPath: /pki/admin-client 101 | name: pki-admin-client 102 | - mountPath: /scripts 103 | name: scripts 104 | {{- if or .Values.extraManifests .Values.konnectivityServer.enabled .Values.konnectivityAgent.enabled }} 105 | - mountPath: /manifests 106 | name: manifests 107 | {{- end }} 108 | - mountPath: /config 109 | name: config 110 | {{- with .Values.admin.extraVolumeMounts }} 111 | {{- toYaml . | nindent 8 }} 112 | {{- end }} 113 | {{- with .Values.admin.sidecars }} 114 | {{- toYaml . | nindent 6 }} 115 | {{- end }} 116 | securityContext: 117 | seccompProfile: 118 | type: RuntimeDefault 119 | volumes: 120 | - configMap: 121 | name: "{{ $fullName }}-admin-conf" 122 | name: kubeconfig 123 | - secret: 124 | secretName: "{{ $fullName }}-pki-admin-client" 125 | name: pki-admin-client 126 | - name: scripts 127 | configMap: 128 | name: "{{ $fullName }}-kubeadm-scripts" 129 | defaultMode: 0777 130 | {{- if or .Values.extraManifests .Values.konnectivityServer.enabled .Values.konnectivityAgent.enabled }} 131 | - name: manifests 132 | projected: 133 | sources: 134 | {{- if or .Values.extraManifests }} 135 | - secret: 136 | name: "{{ $fullName }}-extra-manifests" 137 | {{- end }} 138 | {{- if or .Values.konnectivityServer.enabled .Values.konnectivityAgent.enabled }} 139 | - configMap: 140 | name: "{{ $fullName }}-konnectivity-manifests" 141 | {{- end }} 142 | {{- end }} 143 | - name: config 144 | configMap: 145 | name: "{{ $fullName }}-kubeadm-config" 146 | {{- with .Values.admin.extraVolumes }} 147 | {{- toYaml . | nindent 6 }} 148 | {{- end }} 149 | {{- end }} 150 | -------------------------------------------------------------------------------- /deploy/helm/kubernetes/templates/apiserver-config.yaml: -------------------------------------------------------------------------------- 1 | {{- $fullName := include "kubernetes.fullname" . -}} 2 | --- 3 | apiVersion: v1 4 | kind: ConfigMap 5 | metadata: 6 | name: {{ $fullName }}-apiserver-config 7 | data: 8 | egress-selector-configuration.yaml: | 9 | apiVersion: apiserver.k8s.io/v1beta1 10 | kind: EgressSelectorConfiguration 11 | egressSelections: 12 | - name: cluster 13 | connection: 14 | {{- if and .Values.konnectivityServer.enabled }} 15 | {{- if has .Values.konnectivityServer.mode (list "HTTPConnect" "GRPC") }} 16 | proxyProtocol: {{ .Values.konnectivityServer.mode }} 17 | {{- else }} 18 | {{- fail ".Values.konnectivityServer.mode supports only \"HTTPConnect\" and \"GRPC\" values" }} 19 | {{- end }} 20 | transport: 21 | {{- if eq .Values.konnectivityServer.mode "GRPC" }} 22 | uds: 23 | udsName: /run/konnectivity-server/konnectivity-server.socket 24 | {{- else }} 25 | tcp: 26 | url: "https://{{ $fullName }}-konnectivity-server:8131" 27 | TLSConfig: 28 | caBundle: /pki/konnectivity-client/ca.crt 29 | clientKey: /pki/konnectivity-client/tls.key 30 | clientCert: /pki/konnectivity-client/tls.crt 31 | {{- end }} 32 | {{- else }} 33 | proxyProtocol: Direct 34 | {{- end }} 35 | - name: master 36 | connection: 37 | proxyProtocol: Direct 38 | - name: etcd 39 | connection: 40 | proxyProtocol: Direct 41 | -------------------------------------------------------------------------------- /deploy/helm/kubernetes/templates/apiserver-deployment.yaml: -------------------------------------------------------------------------------- 1 | {{- if .Values.apiServer.enabled }} 2 | {{- $fullName := include "kubernetes.fullname" . -}} 3 | {{- $certName := include "kubernetes.certname" . -}} 4 | --- 5 | apiVersion: apps/v1 6 | kind: Deployment 7 | metadata: 8 | name: "{{ $fullName }}-apiserver" 9 | labels: 10 | app: "{{ $fullName }}-apiserver" 11 | {{- with .Values.apiServer.labels }} 12 | {{- toYaml . | nindent 4 }} 13 | {{- end }} 14 | {{- with .Values.apiServer.annotations }} 15 | annotations: 16 | {{- toYaml . | nindent 4 }} 17 | {{- end }} 18 | spec: 19 | replicas: {{ .Values.apiServer.replicaCount }} 20 | selector: 21 | matchLabels: 22 | app: "{{ $fullName }}-apiserver" 23 | template: 24 | metadata: 25 | labels: 26 | app: "{{ $fullName }}-apiserver" 27 | {{- with .Values.apiServer.podLabels }} 28 | {{- toYaml . | nindent 8 }} 29 | {{- end }} 30 | annotations: 31 | checksum/config: {{ include (print $.Template.BasePath "/apiserver-config.yaml") . | sha256sum }} 32 | {{- with .Values.apiServer.podAnnotations }} 33 | {{- toYaml . | nindent 8 }} 34 | {{- end }} 35 | spec: 36 | {{- with .Values.apiServer.nodeSelector }} 37 | nodeSelector: 38 | {{- toYaml . | nindent 8 }} 39 | {{- end }} 40 | {{- with .Values.apiServer.tolerations }} 41 | tolerations: 42 | {{- toYaml . | nindent 6 }} 43 | {{- end }} 44 | {{- if or .Values.apiServer.affinity .Values.apiServer.podAntiAffinity }} 45 | affinity: 46 | {{- with .Values.apiServer.affinity }} 47 | {{- toYaml . | nindent 8 }} 48 | {{- end }} 49 | {{- if eq .Values.apiServer.podAntiAffinity "hard" }} 50 | podAntiAffinity: 51 | requiredDuringSchedulingIgnoredDuringExecution: 52 | - topologyKey: "{{ .Values.apiServer.podAntiAffinityTopologyKey }}" 53 | labelSelector: 54 | matchLabels: 55 | app: {{ $fullName }}-apiserver 56 | {{- else if eq .Values.apiServer.podAntiAffinity "soft" }} 57 | podAntiAffinity: 58 | preferredDuringSchedulingIgnoredDuringExecution: 59 | - weight: 1 60 | podAffinityTerm: 61 | topologyKey: "{{ .Values.apiServer.podAntiAffinityTopologyKey }}" 62 | labelSelector: 63 | matchLabels: 64 | app: {{ $fullName }}-apiserver 65 | {{- end }} 66 | {{- end }} 67 | {{- with .Values.apiServer.image.pullSecrets }} 68 | imagePullSecrets: 69 | {{- toYaml . | nindent 10 }} 70 | {{- end }} 71 | automountServiceAccountToken: false 72 | containers: 73 | - command: 74 | - kube-apiserver 75 | - --allow-privileged=true 76 | - --authorization-mode=Node,RBAC 77 | - --bind-address=0.0.0.0 78 | - --client-ca-file=/pki/apiserver-server/ca.crt 79 | - --enable-admission-plugins=NodeRestriction 80 | - --enable-bootstrap-token-auth=true 81 | - --etcd-cafile=/pki/apiserver-etcd-client/ca.crt 82 | - --etcd-certfile=/pki/apiserver-etcd-client/tls.crt 83 | - --etcd-keyfile=/pki/apiserver-etcd-client/tls.key 84 | - --etcd-servers={{ template "kubernetes.etcdEndpoints" . }} 85 | - --insecure-port=0 86 | - --kubelet-client-certificate=/pki/apiserver-kubelet-client/tls.crt 87 | - --kubelet-client-key=/pki/apiserver-kubelet-client/tls.key 88 | - --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname 89 | - --proxy-client-cert-file=/pki/front-proxy-client/tls.crt 90 | - --proxy-client-key-file=/pki/front-proxy-client/tls.key 91 | - --requestheader-allowed-names={{ $certName }}-front-proxy-client 92 | - --requestheader-client-ca-file=/pki/front-proxy-client/ca.crt 93 | - --requestheader-extra-headers-prefix=X-Remote-Extra- 94 | - --requestheader-group-headers=X-Remote-Group 95 | - --requestheader-username-headers=X-Remote-User 96 | - --secure-port={{ .Values.apiServer.port }} 97 | - --service-account-key-file=/pki/sa/tls.crt 98 | - --service-cluster-ip-range={{ .Values.networking.serviceSubnet }} 99 | - --tls-cert-file=/pki/apiserver-server/tls.crt 100 | - --tls-private-key-file=/pki/apiserver-server/tls.key 101 | - --egress-selector-config-file=/etc/kubernetes/egress-selector-configuration.yaml 102 | - --service-account-issuer=https://kubernetes.default.svc.{{ .Values.networking.dnsDomain }} 103 | - --service-account-signing-key-file=/pki/sa/tls.key 104 | {{- if not (hasKey .Values.apiServer.extraArgs "advertise-address") }} 105 | {{- with .Values.apiServer.service.loadBalancerIP }} 106 | - --advertise-address={{ . }} 107 | {{- end }} 108 | {{- end }} 109 | {{- range $key, $value := .Values.apiServer.extraArgs }} 110 | - --{{ $key }}={{ $value }} 111 | {{- end }} 112 | ports: 113 | - containerPort: {{ .Values.apiServer.port }} 114 | name: client 115 | {{- with .Values.apiServer.image }} 116 | image: "{{ .repository }}{{ if .digest }}@{{ .digest }}{{ else }}:{{ .tag }}{{ end }}" 117 | imagePullPolicy: {{ .pullPolicy }} 118 | {{- end }} 119 | livenessProbe: 120 | failureThreshold: 8 121 | httpGet: 122 | path: /livez 123 | port: {{ .Values.apiServer.port }} 124 | scheme: HTTPS 125 | initialDelaySeconds: 15 126 | timeoutSeconds: 15 127 | name: kube-apiserver 128 | resources: 129 | {{- toYaml .Values.apiServer.resources | nindent 10 }} 130 | {{- with .Values.apiServer.extraEnv }} 131 | env: 132 | {{- toYaml . | nindent 8 }} 133 | {{- end }} 134 | volumeMounts: 135 | - mountPath: /etc/kubernetes 136 | name: apiserver-config 137 | - mountPath: /pki/front-proxy-client 138 | name: pki-front-proxy-client 139 | - mountPath: /pki/apiserver-server 140 | name: pki-apiserver-server 141 | - mountPath: /pki/apiserver-etcd-client 142 | name: pki-apiserver-etcd-client 143 | - mountPath: /pki/apiserver-kubelet-client 144 | name: pki-apiserver-kubelet-client 145 | - mountPath: /pki/sa 146 | name: pki-sa 147 | {{- if and .Values.konnectivityServer.enabled (eq .Values.konnectivityServer.mode "HTTPConnect") }} 148 | - mountPath: /pki/konnectivity-client 149 | name: pki-konnectivity-client 150 | {{- end }} 151 | {{- if and .Values.konnectivityServer.enabled (eq .Values.konnectivityServer.mode "GRPC") }} 152 | - mountPath: /run/konnectivity-server 153 | name: konnectivity-uds 154 | {{- end }} 155 | {{- with .Values.apiServer.extraVolumeMounts }} 156 | {{- toYaml . | nindent 8 }} 157 | {{- end }} 158 | {{- if and .Values.konnectivityServer.enabled (eq .Values.konnectivityServer.mode "GRPC") }} 159 | {{ template "kubernetes.konnectivityServer.containers" . }} 160 | {{- end }} 161 | {{- with .Values.apiServer.sidecars }} 162 | {{- toYaml . | nindent 6 }} 163 | {{- end }} 164 | securityContext: 165 | seccompProfile: 166 | type: RuntimeDefault 167 | volumes: 168 | - configMap: 169 | name: "{{ $fullName }}-apiserver-config" 170 | name: apiserver-config 171 | - secret: 172 | secretName: "{{ $fullName }}-pki-front-proxy-client" 173 | name: pki-front-proxy-client 174 | - secret: 175 | secretName: "{{ $fullName }}-pki-apiserver-server" 176 | name: pki-apiserver-server 177 | - secret: 178 | secretName: "{{ $fullName }}-pki-apiserver-etcd-client" 179 | name: pki-apiserver-etcd-client 180 | - secret: 181 | secretName: "{{ $fullName }}-pki-apiserver-kubelet-client" 182 | name: pki-apiserver-kubelet-client 183 | - secret: 184 | secretName: "{{ $fullName }}-pki-sa" 185 | name: pki-sa 186 | {{- if .Values.konnectivityServer.enabled }} 187 | - secret: 188 | secretName: "{{ $fullName }}-pki-konnectivity-client" 189 | name: pki-konnectivity-client 190 | {{- end }} 191 | {{- if and .Values.konnectivityServer.enabled (eq .Values.konnectivityServer.mode "GRPC") }} 192 | {{ template "kubernetes.konnectivityServer.volumes" . }} 193 | {{- end }} 194 | {{- with .Values.apiServer.extraVolumes }} 195 | {{- toYaml . | nindent 6 }} 196 | {{- end }} 197 | {{- end }} 198 | -------------------------------------------------------------------------------- /deploy/helm/kubernetes/templates/apiserver-service.yaml: -------------------------------------------------------------------------------- 1 | {{- if and .Values.apiServer.enabled .Values.apiServer.service.enabled }} 2 | {{- $fullName := include "kubernetes.fullname" . -}} 3 | --- 4 | apiVersion: v1 5 | kind: Service 6 | metadata: 7 | name: {{ $fullName }}-apiserver 8 | labels: 9 | app: {{ $fullName }}-apiserver 10 | {{- with .Values.apiServer.service.labels }} 11 | {{- toYaml . | nindent 4 }} 12 | {{- end }} 13 | annotations: 14 | {{- with .Values.apiServer.service.annotations }} 15 | {{- toYaml . | nindent 4 }} 16 | {{- end }} 17 | spec: 18 | type: {{ .Values.apiServer.service.type }} 19 | {{- with .Values.apiServer.service.loadBalancerIP }} 20 | loadBalancerIP: {{ . }} 21 | {{- end }} 22 | ports: 23 | - port: {{ .Values.apiServer.service.port }} 24 | name: client 25 | {{- with .Values.apiServer.service.nodePort }} 26 | nodePort: {{ . }} 27 | {{- end }} 28 | {{- if and .Values.konnectivityServer.enabled .Values.konnectivityServer.service.enabled (eq .Values.konnectivityServer.mode "GRPC") }} 29 | - port: {{ .Values.konnectivityServer.ports.agent }} 30 | name: agent 31 | {{- with .Values.konnectivityServer.service.nodePorts.client }} 32 | nodePort: {{ . }} 33 | {{- end }} 34 | {{- end }} 35 | selector: 36 | app: {{ $fullName }}-apiserver 37 | {{- end }} 38 | -------------------------------------------------------------------------------- /deploy/helm/kubernetes/templates/controller-manager-configmap.yaml: -------------------------------------------------------------------------------- 1 | {{- if .Values.controllerManager.enabled }} 2 | {{- $fullName := include "kubernetes.fullname" . -}} 3 | --- 4 | apiVersion: v1 5 | kind: ConfigMap 6 | metadata: 7 | name: {{ $fullName }}-controller-manager-conf 8 | data: 9 | controller-manager.conf: | 10 | apiVersion: v1 11 | clusters: 12 | - cluster: 13 | certificate-authority: /pki/controller-manager-client/ca.crt 14 | server: https://{{ $fullName }}-apiserver:{{ .Values.apiServer.service.port }} 15 | name: default-cluster 16 | contexts: 17 | - context: 18 | cluster: default-cluster 19 | namespace: default 20 | user: default-auth 21 | name: default-context 22 | current-context: default-context 23 | kind: Config 24 | preferences: {} 25 | users: 26 | - name: default-auth 27 | user: 28 | client-certificate: /pki/controller-manager-client/tls.crt 29 | client-key: /pki/controller-manager-client/tls.key 30 | {{- end }} 31 | -------------------------------------------------------------------------------- /deploy/helm/kubernetes/templates/controller-manager-deployment.yaml: -------------------------------------------------------------------------------- 1 | {{- if .Values.controllerManager.enabled }} 2 | {{- $fullName := include "kubernetes.fullname" . -}} 3 | --- 4 | apiVersion: apps/v1 5 | kind: Deployment 6 | metadata: 7 | name: "{{ $fullName }}-controller-manager" 8 | labels: 9 | app: "{{ $fullName }}-controller-manager" 10 | {{- with .Values.controllerManager.labels }} 11 | {{- toYaml . | nindent 4 }} 12 | {{- end }} 13 | {{- with .Values.controllerManager.annotations }} 14 | annotations: 15 | {{- toYaml . | nindent 4 }} 16 | {{- end }} 17 | spec: 18 | replicas: {{ .Values.controllerManager.replicaCount }} 19 | selector: 20 | matchLabels: 21 | app: "{{ $fullName }}-controller-manager" 22 | template: 23 | metadata: 24 | labels: 25 | app: "{{ $fullName }}-controller-manager" 26 | {{- with .Values.controllerManager.podLabels }} 27 | {{- toYaml . | nindent 8 }} 28 | {{- end }} 29 | {{- with .Values.controllerManager.podAnnotations }} 30 | annotations: 31 | {{- toYaml . | nindent 8 }} 32 | {{- end }} 33 | spec: 34 | {{- with .Values.controllerManager.nodeSelector }} 35 | nodeSelector: 36 | {{- toYaml . | nindent 8 }} 37 | {{- end }} 38 | {{- with .Values.controllerManager.tolerations }} 39 | tolerations: 40 | {{- toYaml . | nindent 6 }} 41 | {{- end }} 42 | {{- if or .Values.controllerManager.affinity .Values.controllerManager.podAntiAffinity }} 43 | affinity: 44 | {{- with .Values.controllerManager.affinity }} 45 | {{- toYaml . | nindent 8 }} 46 | {{- end }} 47 | {{- if eq .Values.controllerManager.podAntiAffinity "hard" }} 48 | podAntiAffinity: 49 | requiredDuringSchedulingIgnoredDuringExecution: 50 | - topologyKey: "{{ .Values.controllerManager.podAntiAffinityTopologyKey }}" 51 | labelSelector: 52 | matchLabels: 53 | app: {{ $fullName }}-controller-manager 54 | {{- else if eq .Values.controllerManager.podAntiAffinity "soft" }} 55 | podAntiAffinity: 56 | preferredDuringSchedulingIgnoredDuringExecution: 57 | - weight: 1 58 | podAffinityTerm: 59 | topologyKey: "{{ .Values.controllerManager.podAntiAffinityTopologyKey }}" 60 | labelSelector: 61 | matchLabels: 62 | app: {{ $fullName }}-controller-manager 63 | {{- end }} 64 | {{- end }} 65 | {{- with .Values.controllerManager.image.pullSecrets }} 66 | imagePullSecrets: 67 | {{- toYaml . | nindent 10 }} 68 | {{- end }} 69 | automountServiceAccountToken: false 70 | containers: 71 | - command: 72 | - kube-controller-manager 73 | - --authentication-kubeconfig=/etc/kubernetes/controller-manager.conf 74 | - --authorization-kubeconfig=/etc/kubernetes/controller-manager.conf 75 | - --bind-address=0.0.0.0 76 | - --client-ca-file=/pki/ca/tls.crt 77 | - --cluster-name=kubernetes 78 | - --cluster-signing-cert-file=/pki/ca/tls.crt 79 | - --cluster-signing-key-file=/pki/ca/tls.key 80 | - --controllers=*,bootstrapsigner,tokencleaner 81 | - --kubeconfig=/etc/kubernetes/controller-manager.conf 82 | - --leader-elect=true 83 | - --requestheader-client-ca-file=/pki/front-proxy-client/tls.crt 84 | - --root-ca-file=/pki/ca/tls.crt 85 | - --secure-port={{ .Values.controllerManager.port }} 86 | - --service-account-private-key-file=/pki/sa/tls.key 87 | - --use-service-account-credentials=true 88 | - --tls-cert-file=/pki/controller-manager-server/tls.crt 89 | - --tls-private-key-file=/pki/controller-manager-server/tls.key 90 | - --service-cluster-ip-range={{ .Values.networking.serviceSubnet }} 91 | {{ with .Values.networking.podSubnet }} 92 | - --allocate-node-cidrs=true 93 | - --cluster-cidr={{ . }} 94 | {{- end }} 95 | {{- range $key, $value := .Values.controllerManager.extraArgs }} 96 | - --{{ $key }}={{ $value }} 97 | {{- end }} 98 | {{- with .Values.controllerManager.image }} 99 | image: "{{ .repository }}{{ if .digest }}@{{ .digest }}{{ else }}:{{ .tag }}{{ end }}" 100 | imagePullPolicy: {{ .pullPolicy }} 101 | {{- end }} 102 | livenessProbe: 103 | failureThreshold: 8 104 | httpGet: 105 | path: /healthz 106 | port: {{ .Values.controllerManager.port }} 107 | scheme: HTTPS 108 | initialDelaySeconds: 15 109 | timeoutSeconds: 15 110 | name: kube-controller-manager 111 | resources: 112 | {{- toYaml .Values.controllerManager.resources | nindent 10 }} 113 | {{- with .Values.controllerManager.extraEnv }} 114 | env: 115 | {{- toYaml . | nindent 8 }} 116 | {{- end }} 117 | volumeMounts: 118 | - mountPath: /etc/kubernetes/ 119 | name: kubeconfig 120 | readOnly: true 121 | - mountPath: /pki/controller-manager-server 122 | name: pki-controller-manager-server 123 | - mountPath: /pki/controller-manager-client 124 | name: pki-controller-manager-client 125 | - mountPath: /pki/ca 126 | name: pki-ca 127 | - mountPath: /pki/front-proxy-client 128 | name: pki-front-proxy-client 129 | - mountPath: /pki/sa 130 | name: pki-sa 131 | {{- with .Values.controllerManager.extraVolumeMounts }} 132 | {{- toYaml . | nindent 8 }} 133 | {{- end }} 134 | {{- with .Values.controllerManager.sidecars }} 135 | {{- toYaml . | nindent 6 }} 136 | {{- end }} 137 | securityContext: 138 | seccompProfile: 139 | type: RuntimeDefault 140 | volumes: 141 | - configMap: 142 | name: "{{ $fullName }}-controller-manager-conf" 143 | name: kubeconfig 144 | - secret: 145 | secretName: "{{ $fullName }}-pki-controller-manager-server" 146 | name: pki-controller-manager-server 147 | - secret: 148 | secretName: "{{ $fullName }}-pki-controller-manager-client" 149 | name: pki-controller-manager-client 150 | - secret: 151 | secretName: "{{ $fullName }}-pki-ca" 152 | name: pki-ca 153 | - secret: 154 | secretName: "{{ $fullName }}-pki-front-proxy-client" 155 | name: pki-front-proxy-client 156 | - secret: 157 | secretName: "{{ $fullName }}-pki-sa" 158 | name: pki-sa 159 | {{- with .Values.controllerManager.extraVolumes }} 160 | {{- toYaml . | nindent 6 }} 161 | {{- end }} 162 | {{- end }} 163 | -------------------------------------------------------------------------------- /deploy/helm/kubernetes/templates/controller-manager-service.yaml: -------------------------------------------------------------------------------- 1 | {{- if and .Values.controllerManager.enabled .Values.controllerManager.service.enabled }} 2 | {{- $fullName := include "kubernetes.fullname" . -}} 3 | --- 4 | apiVersion: v1 5 | kind: Service 6 | metadata: 7 | name: {{ $fullName }}-controller-manager 8 | labels: 9 | app: {{ $fullName }}-controller-manager 10 | {{- with .Values.controllerManager.service.labels }} 11 | {{- toYaml . | nindent 4 }} 12 | {{- end }} 13 | {{- with .Values.controllerManager.service.annotations }} 14 | annotations: 15 | {{- toYaml . | nindent 4 }} 16 | {{- end }} 17 | spec: 18 | type: {{ .Values.controllerManager.service.type }} 19 | {{- with .Values.controllerManager.service.loadBalancerIP }} 20 | loadBalancerIP: {{ . }} 21 | {{- end }} 22 | ports: 23 | - port: {{ .Values.controllerManager.service.port }} 24 | name: client 25 | {{- with .Values.controllerManager.service.nodePort }} 26 | nodePort: {{ . }} 27 | {{- end }} 28 | selector: 29 | app: {{ $fullName }}-controller-manager 30 | {{- end }} 31 | -------------------------------------------------------------------------------- /deploy/helm/kubernetes/templates/etcd-backup-cronjob.yaml: -------------------------------------------------------------------------------- 1 | {{- if .Values.etcd.backup.enabled }} 2 | {{- $fullName := include "kubernetes.fullname" . -}} 3 | --- 4 | apiVersion: batch/v1 5 | kind: CronJob 6 | metadata: 7 | name: {{ $fullName }}-etcd-backup 8 | labels: 9 | app: {{ $fullName }}-etcd-backup 10 | {{- with .Values.etcd.backup.labels }} 11 | {{- toYaml . | nindent 4 }} 12 | {{- end }} 13 | {{- with .Values.etcd.backup.annotations }} 14 | annotations: 15 | {{- toYaml . | nindent 4 }} 16 | {{- end }} 17 | spec: 18 | schedule: "{{ .Values.etcd.backup.schedule }}" 19 | successfulJobsHistoryLimit: {{ .Values.etcd.backup.successfulJobsHistoryLimit }} 20 | failedJobsHistoryLimit: {{ .Values.etcd.backup.failedJobsHistoryLimit }} 21 | jobTemplate: 22 | metadata: 23 | labels: 24 | app: {{ $fullName }}-etcd-backup 25 | {{- with .Values.etcd.backup.labels }} 26 | {{- toYaml . | nindent 8 }} 27 | {{- end }} 28 | {{- with .Values.etcd.backup.annotations }} 29 | annotations: 30 | {{- toYaml . | nindent 8 }} 31 | {{- end }} 32 | spec: 33 | template: 34 | metadata: 35 | labels: 36 | app: {{ $fullName }}-etcd-backup 37 | {{- with .Values.etcd.backup.podLabels }} 38 | {{- toYaml . | nindent 12 }} 39 | {{- end }} 40 | {{- with .Values.etcd.backup.podAnnotations }} 41 | annotations: 42 | {{- toYaml . | nindent 12 }} 43 | {{- end }} 44 | spec: 45 | {{- with .Values.etcd.backup.nodeSelector }} 46 | nodeSelector: 47 | {{- toYaml . | nindent 12 }} 48 | {{- end }} 49 | {{- with .Values.etcd.backup.tolerations }} 50 | tolerations: 51 | {{- toYaml . | nindent 10 }} 52 | {{- end }} 53 | {{- if or .Values.etcd.backup.affinity .Values.etcd.backup.podAffinity }} 54 | affinity: 55 | {{- with .Values.etcd.backup.affinity }} 56 | {{- toYaml . | nindent 12 }} 57 | {{- end }} 58 | {{- if eq .Values.etcd.backup.podAffinity "hard" }} 59 | podAffinity: 60 | requiredDuringSchedulingIgnoredDuringExecution: 61 | - topologyKey: "{{ .Values.etcd.backup.podAffinityTopologyKey }}" 62 | labelSelector: 63 | matchLabels: 64 | app: {{ $fullName }}-etcd 65 | {{- else if eq .Values.etcd.backup.podAffinity "soft" }} 66 | podAffinity: 67 | preferredDuringSchedulingIgnoredDuringExecution: 68 | - weight: 1 69 | podAffinityTerm: 70 | topologyKey: "{{ .Values.etcd.backup.podAffinityTopologyKey }}" 71 | labelSelector: 72 | matchLabels: 73 | app: {{ $fullName }}-etcd 74 | {{- end }} 75 | {{- end }} 76 | 77 | {{- with .Values.etcd.image.pullSecrets }} 78 | imagePullSecrets: 79 | {{- toYaml . | nindent 10 }} 80 | {{- end }} 81 | automountServiceAccountToken: false 82 | restartPolicy: OnFailure 83 | containers: 84 | - command: 85 | - /bin/sh 86 | - -xc 87 | - | 88 | rtc() { while read k s v; do test "$k" = "rtc_$1" && echo "$v" && break; done