├── .gitignore ├── LICENSE ├── README.md ├── deploy.sh ├── docs └── k8sonk8s.png ├── kubeadm └── Dockerfile ├── manifests ├── apiserver │ ├── apiserver-deployment.yaml │ ├── apiserver-ingress.yaml │ └── apiserver-service.yaml ├── etcd │ ├── etcd-deployment.yaml │ └── etcd-service.yaml ├── kcm │ └── kube-controller-manager.yaml ├── on-k8s │ ├── kube-dns │ │ ├── kubedns-cm.yaml │ │ ├── kubedns-controller.yaml │ │ ├── kubedns-sa.yaml │ │ └── kubedns-svc.yaml │ ├── kube-flannel │ │ ├── kube-flannel-rbac.yml │ │ └── kube-flannel.yml │ └── kube-proxy │ │ └── kube-proxy.yaml └── scheduler │ └── kube-scheduler.yaml └── tls ├── ca-config.json ├── ca-csr.json ├── kube-apiserver-server-csr.json ├── kube-controller-manager-client-csr.json ├── kube-proxy-client-csr.json ├── kube-scheduler-client-csr.json ├── kubelet-client-csr.json ├── kubernetes-admin-user.csr.json └── make_certs.sh /.gitignore: -------------------------------------------------------------------------------- 1 | kubeconfig* 2 | *.pem 3 | *.csr 4 | tls/apiserver-admin-secret.yaml 5 | tls/apiserver-secret.yaml 6 | tls/controller-manager-secret.yaml 7 | tls/scheduler-secret.yaml 8 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "{}" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright {yyyy} {name of copyright owner} 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # k8s-on-k8s 2 | 3 | This project aims to build k8s _control planes_ on demand using an existing k8s 4 | cluster so you can easily test new versions, features or networking drivers without 5 | having to wait for an installer to support it. 6 | 7 | It can also be used for teaching purposes giving every student a control plane 8 | using as little resources as what a few _hyperkube_ and _etcd_ PODs consume. 9 | 10 | 11 | Theory of operation 12 | -------------- 13 | 14 | A _master_ or _infrastructure_ cluster provides control plane as a service to multiple 15 | delegated cluster administrators by granting each of them a writable namespace. 16 | 17 | The Kubernetes _hosted_ control plane is provisioned in that namespace just like any 18 | common containerized application using deployments, services and ingress native objects. 19 | 20 | The cluster administrator can then connect his worker nodes to that control plane 21 | and create the necessary RBAC bindings for his consumers with no change to the 22 | infrastructure cluster. 23 | 24 | A basic script creates the control plane in a *kubeadm* compatible way, in the sense 25 | you can use kubeadm to manage tokens for workers to join 26 | 27 | ![In a picture](docs/k8sonk8s.png) 28 | 29 | 30 | Current state 31 | -------------- 32 | It's a WIP. 33 | * A single etcd member with no persistence (until setup with operator) 34 | * A k8s control plane with RBAC enabled (API server, controller-manager and scheduler) 35 | * Kubeconfig files for the hosted cluster administrator 36 | * Flannel and kube-proxy daemon-sets to be instantiated on workers 37 | * Kube-dns on the first worker joining the cluster 38 | 39 | Also note this _10.199.199.199_ IP advertised in the API server manifest and setup 40 | locally on each worker's loopback interface as show below. This is a work around for in-cluster 41 | clients using the built-in kubernetes service to reach the API server (such as kube-flannel 42 | and kube-dns PODs) via a nodePort service. 43 | 44 | 45 | Local requirements to deploy _hosted_ control planes 46 | -------------- 47 | * cfssl 48 | * kubectl setup with the infrastructure cluster context 49 | 50 | 51 | K8s Infrastructure requirements 52 | -------------- 53 | * Access to a writable namespace 54 | * A functional ingress controller on the infra cluster 55 | 56 | 57 | Creating the hosted control plane 58 | -------------- 59 | Specify the desired API URL hostname that must resolve and route to the infrastructure 60 | cluster's ingress controller. This could be as a CNAME to the ingress controller 61 | of the infrastructure cluster. 62 | 63 | 64 | ``` 65 | $ ./deploy.sh -h kubernetes.foo-bar.com -n namespaceX 66 | CHECK: Access to cluster confirmed 67 | CHECK: API server host kubernetes.foo-bar.com resolves 68 | 2017/07/25 08:31:46 [INFO] generating a new CA key and certificate from CSR 69 | 2017/07/25 08:31:46 [INFO] generate received request 70 | 2017/07/25 08:31:46 [INFO] received CSR 71 | 2017/07/25 08:31:46 [INFO] generating key: rsa-2048 72 | 2017/07/25 08:31:46 [INFO] encoded CSR 73 | 2017/07/25 08:31:46 [INFO] signed certificate with serial number 343368285204006686908415438394591991039302384146 74 | 2017/07/25 08:31:46 [INFO] generate received request 75 | 2017/07/25 08:31:46 [INFO] received CSR 76 | 2017/07/25 08:31:46 [INFO] generating key: rsa-2048 77 | 2017/07/25 08:31:47 [INFO] encoded CSR 78 | 2017/07/25 08:31:47 [INFO] signed certificate with serial number 634677056508179731833979679679719784750411028857 79 | 2017/07/25 08:31:47 [INFO] generate received request 80 | 2017/07/25 08:31:47 [INFO] received CSR 81 | 2017/07/25 08:31:47 [INFO] generating key: rsa-2048 82 | 2017/07/25 08:31:47 [INFO] encoded CSR 83 | 2017/07/25 08:31:47 [INFO] signed certificate with serial number 34772891780328732259434788313932796585661614421 84 | ... 85 | secret "kube-apiserver" created 86 | secret "admin-kubeconfig" created 87 | secret "kube-controller-manager" created 88 | secret "kube-scheduler" created 89 | deployment "etcd" created 90 | service "etcd0" created 91 | deployment "kube-apiserver" created 92 | ingress "k8s-on-k8s" created 93 | service "apiserver" created 94 | deployment "kube-controller-manager" created 95 | deployment "kube-scheduler" created 96 | Giving a few seconds for the API server to start... 97 | Trying to connect to the hosted control plane... 98 | ......AVAILABLE ! 99 | Client Version: version.Info{Major:"1", Minor:"7", GitVersion:"v1.7.1", GitCommit:"1dc5c66f5dd61da08412a74221ecc79208c2165b", GitTreeState:"clean", BuildDate:"2017-07-14T02:00:46Z", GoVersion:"go1.8.3", Compiler:"gc", Platform:"linux/amd64"} 100 | Server Version: version.Info{Major:"1", Minor:"9", GitVersion:"v1.9.0", GitCommit:"925c127ec6b946659ad0fd596fa959be43f0cc05", GitTreeState:"clean", BuildDate:"2017-12-15T20:55:30Z", GoVersion:"go1.9.2", Compiler:"gc", Platform:"linux/amd64"} 101 | Deploying child cluster assets 102 | secret "kubeconfig-proxy" created 103 | daemonset "kube-proxy" created 104 | clusterrole "flannel" created 105 | clusterrolebinding "flannel" created 106 | serviceaccount "flannel" created 107 | configmap "kube-flannel-cfg" created 108 | daemonset "kube-flannel-ds" created 109 | configmap "kube-dns" created 110 | deployment "kube-dns" created 111 | serviceaccount "kube-dns" created 112 | service "kube-dns" created 113 | ``` 114 | 115 | 116 | Prepare the control plane for kubelet bootstraping 117 | -------------- 118 | ``` 119 | docker run -v ${PWD}/tls:/tmp/tls -it jfnadeau/kubeadm:1.9.0 kubeadm alpha phase bootstrap-token cluster-info --kubeconfig /tmp/tls/kubeconfig 120 | docker run -v ${PWD}/tls:/tmp/tls -it jfnadeau/kubeadm:1.9.0 kubeadm alpha phase bootstrap-token node allow-auto-approve --kubeconfig /tmp/tls/kubeconfig 121 | docker run -v ${PWD}/tls:/tmp/tls -it jfnadeau/kubeadm:1.9.0 kubeadm alpha phase bootstrap-token node allow-post-csrs --kubeconfig /tmp/tls/kubeconfig 122 | docker run -v ${PWD}/tls:/tmp/tls -it jfnadeau/kubeadm:1.9.0 kubeadm token create --kubeconfig /tmp/tls/kubeconfig 123 | 5a5312.564acf6a9824ca38 <--- Use this on the workers 124 | ``` 125 | 126 | Connecting a worker 127 | -------------- 128 | 129 | Follow the kubeadm installation process for your OS as described https://kubernetes.io/docs/setup/independent/install-kubeadm/ 130 | 131 | ``` 132 | kubeadm join --token 5a5312.564acf6a9824ca38 :443 --discovery-token-unsafe-skip-ca-verification 133 | ``` 134 | 135 | 136 | 137 | Using the _hosted_ cluster resources 138 | -------------- 139 | 140 | A _kubeconfig_ file with admin privileges was created under the tls directory. 141 | 142 | Use it to perform administrator tasks and to create the additional 143 | RBAC bindings for the end user (if any but you !) 144 | 145 | 146 | ``` 147 | $ kubectl get nodes --kubeconfig=tls/kubeconfig 148 | NAME STATUS AGE VERSION 149 | node1 Ready 12m v1.9.0 150 | ``` 151 | 152 | ``` 153 | $ kubectl --kubeconfig=tls/kubeconfig get pods -n kube-system -o wide 154 | NAME READY STATUS RESTARTS AGE IP NODE 155 | kube-dns-2177165803-19jgj 3/3 Running 0 3m 10.2.0.2 192.168.1.125 156 | kube-flannel-ds-29nwm 2/2 Running 1 3m 192.168.1.124 192.168.1.124 157 | kube-flannel-ds-btlg7 2/2 Running 1 3m 192.168.1.125 192.168.1.125 158 | kube-proxy-fz6s5 1/1 Running 0 3m 192.168.1.125 192.168.1.125 159 | kube-proxy-jtfk4 1/1 Running 0 3m 192.168.1.124 192.168.1.124 160 | 161 | ``` 162 | 163 | 164 | Delete hosted k8s resources 165 | -------------- 166 | ``` 167 | kubectl -n namespaceX delete deployment,svc,secrets,ingress --all 168 | ``` 169 | -------------------------------------------------------------------------------- /deploy.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | print_usage() { 4 | echo "Usage: ./deploy.sh -h [-p ] -n " 5 | exit 1 6 | } 7 | 8 | # Parse args 9 | [ -z "${1}" ] && print_usage 10 | OPTIND=1 11 | while getopts "h:p:n:" opt; do 12 | case "$opt" in 13 | h) APIHOST=${OPTARG} 14 | ;; 15 | p) APIPORT=${OPTARG} 16 | ;; 17 | n) NAMESPACE=${OPTARG} 18 | ;; 19 | esac 20 | done 21 | shift $((OPTIND-1)) 22 | [ "$1" = "--" ] && shift 23 | 24 | if [ -z "{APIHOST}" ] || [ -z "${NAMESPACE}" ] ; then 25 | print_usage 26 | fi 27 | 28 | # Make sure we have a working access to the desired NAMESPACE 29 | # by looking up the default service account 30 | # FIXME... doest mean we have priv to create stuff in here 31 | if $(which kubectl) get sa/default -n ${NAMESPACE} 1>/dev/null ; then 32 | echo "CHECK: Access to cluster confirmed" 33 | else 34 | exit 1 35 | fi 36 | 37 | # Generate TLS assets based on provided hostname 38 | if host ${APIHOST} 1>/dev/null ; then 39 | echo "CHECK: API server host ${APIHOST} resolves" 40 | else 41 | echo "Can't resolve ${APIHOST}... assuming ingress host will get created" 42 | fi 43 | 44 | # Add port if non default 45 | if [ ! -z "${APIPORT}" ] ; then 46 | APIHOST="${APIHOST}:${APIPORT}" 47 | fi 48 | 49 | # Create certs and kubeconfig 50 | cd ./tls 51 | ./make_certs.sh ${APIHOST} 52 | 53 | 54 | # Creating K8s secrets in infra cluster 55 | for i in apiserver-secret.yaml apiserver-admin-secret.yaml controller-manager-secret.yaml scheduler-secret.yaml ; do 56 | $(which kubectl) -n ${NAMESPACE} create -f $i 57 | done 58 | 59 | # Creating K8s assets in infra cluster 60 | cd ../manifests 61 | $(which kubectl) -n ${NAMESPACE} create -f etcd/ 62 | $(which kubectl) -n ${NAMESPACE} create -f apiserver/ 63 | $(which kubectl) -n ${NAMESPACE} create -f kcm/ 64 | $(which kubectl) -n ${NAMESPACE} create -f scheduler/ 65 | 66 | # Check we can connect ! 67 | cd ../ 68 | echo "Giving a 10 seconds for the API server to start..." 69 | sleep 10 70 | echo "Trying to connect to the hosted control plane..." 71 | for ((i = 0 ; i < 30 ; i++ )); do 72 | $(which kubectl) version --kubeconfig=tls/kubeconfig 1>/dev/null 2>/dev/null 73 | RES=$? 74 | if [ $RES -eq 0 ]; then 75 | echo "AVAILABLE !" ; 76 | $(which kubectl) version --kubeconfig=tls/kubeconfig 77 | break 78 | fi 79 | echo -n . 80 | sleep 5 81 | [[ $i -eq 15 ]] && echo "Timed out..." && exit 1 82 | done 83 | 84 | 85 | # Store kubeconfig as a secret for kube-proxy 86 | echo "Deploying child cluster assets" 87 | kubectl --kubeconfig=tls/kubeconfig -n kube-system create secret generic kubeconfig-proxy --from-file tls/kubeconfig-proxy 88 | kubectl --kubeconfig=tls/kubeconfig -n kube-system apply -f manifests/on-k8s/kube-proxy/ 89 | kubectl --kubeconfig=tls/kubeconfig -n kube-system apply -f manifests/on-k8s/kube-flannel/ 90 | kubectl --kubeconfig=tls/kubeconfig -n kube-system apply -f manifests/on-k8s/kube-dns/ 91 | -------------------------------------------------------------------------------- /docs/k8sonk8s.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/somejfn/k8s-on-k8s/517acc39ef66a82fda95d581d92428f43487bcdd/docs/k8sonk8s.png -------------------------------------------------------------------------------- /kubeadm/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM debian:stretch-slim 2 | 3 | ENV version 1.9.0-00 4 | 5 | RUN apt-get update \ 6 | && apt-get install -y apt-transport-https gnupg curl \ 7 | && curl -s https://packages.cloud.google.com/apt/doc/apt-key.gpg > /tmp/key.gpg \ 8 | && apt-key add /tmp/key.gpg \ 9 | && echo "deb http://apt.kubernetes.io/ kubernetes-xenial main" > /etc/apt/sources.list.d/kubernetes.list \ 10 | && apt-get update \ 11 | && alias systemctl=/bin/true \ 12 | && apt-get install -y kubelet=${version} kubeadm=${version} kubectl=${version} --allow-unauthenticated ; exit 0 13 | 14 | CMD ["/bin/sh"] 15 | -------------------------------------------------------------------------------- /manifests/apiserver/apiserver-deployment.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: extensions/v1beta1 2 | kind: Deployment 3 | metadata: 4 | labels: 5 | k8s-on-k8s: kube-apiserver 6 | name: kube-apiserver 7 | spec: 8 | replicas: 1 9 | selector: 10 | matchLabels: 11 | k8s-on-k8s: kube-apiserver 12 | tier: app 13 | template: 14 | metadata: 15 | labels: 16 | k8s-on-k8s: kube-apiserver 17 | tier: app 18 | spec: 19 | containers: 20 | - command: 21 | - /usr/bin/flock 22 | - --exclusive 23 | - --timeout=30 24 | - /var/lock/api-server.lock 25 | - /hyperkube 26 | - apiserver 27 | - --bind-address=0.0.0.0 28 | - --secure-port=443 29 | - --insecure-port=8080 30 | - --advertise-address=10.66.171.114 31 | - --etcd-servers=http://etcd0:2379 32 | - --allow-privileged=true 33 | - --service-cluster-ip-range=10.3.0.0/16 34 | - --admission-control=Initializers,NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,ResourceQuota 35 | - --tls-ca-file=/etc/kubernetes/secrets/ca.crt 36 | - --tls-cert-file=/etc/kubernetes/secrets/apiserver.crt 37 | - --tls-private-key-file=/etc/kubernetes/secrets/apiserver.key 38 | - --kubelet-client-certificate=/etc/kubernetes/secrets/apiserver.crt 39 | - --kubelet-client-key=/etc/kubernetes/secrets/apiserver.key 40 | - --client-ca-file=/etc/kubernetes/secrets/ca.crt 41 | - --authorization-mode=Node,RBAC 42 | - --cloud-provider= 43 | - --audit-log-path=/var/log/kubernetes/kube-apiserver-audit.log 44 | - --audit-log-maxage=30 45 | - --audit-log-maxbackup=3 46 | - --audit-log-maxsize=100 47 | - --requestheader-extra-headers-prefix=X-Remote-Extra- 48 | - --requestheader-allowed-names=front-proxy-client 49 | - --requestheader-group-headers=X-Remote-Group 50 | - --enable-bootstrap-token-auth=true 51 | - --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname 52 | image: gcr.io/google_containers/hyperkube:v1.9.0 53 | imagePullPolicy: IfNotPresent 54 | name: kube-apiserver 55 | resources: {} 56 | terminationMessagePath: /dev/termination-log 57 | terminationMessagePolicy: File 58 | ports: 59 | - name: http 60 | containerPort: 80 61 | protocol: TCP 62 | - name: https 63 | containerPort: 443 64 | protocol: TCP 65 | volumeMounts: 66 | - mountPath: /etc/ssl/certs 67 | name: ssl-certs-host 68 | readOnly: true 69 | - mountPath: /etc/kubernetes/secrets 70 | name: secrets 71 | readOnly: true 72 | - mountPath: /var/lock 73 | name: var-lock 74 | - mountPath: /var/log/kubernetes 75 | name: var-log-kubernetes 76 | dnsPolicy: ClusterFirst 77 | restartPolicy: Always 78 | terminationGracePeriodSeconds: 30 79 | volumes: 80 | - hostPath: 81 | path: /usr/share/ca-certificates 82 | name: ssl-certs-host 83 | - name: var-lock 84 | emptyDir: {} 85 | name: var-lock 86 | - name: var-log-kubernetes 87 | emptyDir: {} 88 | - name: secrets 89 | secret: 90 | defaultMode: 420 91 | secretName: kube-apiserver 92 | -------------------------------------------------------------------------------- /manifests/apiserver/apiserver-ingress.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: extensions/v1beta1 2 | kind: Ingress 3 | metadata: 4 | annotations: 5 | ingress.kubernetes.io/ssl-passthrough: "true" 6 | kubernetes.io/ingress.class: shared-nginx 7 | name: k8s-on-k8s 8 | spec: 9 | rules: 10 | - host: k8s.foo.somedomain.edu 11 | http: 12 | paths: 13 | - backend: 14 | serviceName: apiserver 15 | servicePort: 443 16 | tls: 17 | - hosts: 18 | - k8s.foo.somedomain.edu 19 | -------------------------------------------------------------------------------- /manifests/apiserver/apiserver-service.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Service 3 | metadata: 4 | labels: 5 | k8s-on-k8s: kube-apiserver 6 | name: apiserver 7 | spec: 8 | type: NodePort 9 | ports: 10 | - name: http 11 | port: 80 12 | protocol: TCP 13 | - name: https 14 | port: 443 15 | protocol: TCP 16 | selector: 17 | k8s-on-k8s: kube-apiserver 18 | sessionAffinity: None 19 | -------------------------------------------------------------------------------- /manifests/etcd/etcd-deployment.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: extensions/v1beta1 2 | kind: Deployment 3 | metadata: 4 | name: etcd 5 | spec: 6 | replicas: 1 7 | selector: 8 | matchLabels: 9 | k8s-on-k8s: etcd 10 | tier: app 11 | template: 12 | metadata: 13 | labels: 14 | k8s-on-k8s: etcd 15 | tier: app 16 | spec: 17 | hostname: etcd0 18 | containers: 19 | - name: etcd 20 | args: 21 | - /usr/local/bin/etcd 22 | - --name=node1 23 | - --data-dir=/etcd-data 24 | - --listen-peer-urls=http://0.0.0.0:2380 25 | - --listen-client-urls=http://0.0.0.0:2379 26 | - --initial-advertise-peer-urls=http://etcd0:2380 27 | - --initial-cluster=node1=http://etcd0:2380 28 | - --advertise-client-urls=http://etcd0:2379 29 | image: quay.io/coreos/etcd:v3.2.4 30 | imagePullPolicy: IfNotPresent 31 | livenessProbe: 32 | failureThreshold: 5 33 | httpGet: 34 | path: /health 35 | port: 2379 36 | scheme: HTTP 37 | initialDelaySeconds: 60 38 | periodSeconds: 10 39 | successThreshold: 1 40 | timeoutSeconds: 5 41 | ports: 42 | - containerPort: 2379 43 | name: client 44 | protocol: TCP 45 | - containerPort: 2380 46 | name: peer 47 | protocol: TCP 48 | readinessProbe: 49 | failureThreshold: 3 50 | httpGet: 51 | path: /health 52 | port: 2379 53 | scheme: HTTP 54 | initialDelaySeconds: 3 55 | periodSeconds: 10 56 | successThreshold: 1 57 | timeoutSeconds: 5 58 | resources: 59 | limits: 60 | memory: 250Mi 61 | requests: 62 | cpu: 250m 63 | memory: 100Mi 64 | terminationMessagePath: /dev/termination-log 65 | terminationMessagePolicy: File 66 | volumeMounts: 67 | - mountPath: /etcd-data 68 | name: etcd-data 69 | restartPolicy: Always 70 | terminationGracePeriodSeconds: 30 71 | volumes: 72 | - name: etcd-data 73 | emptyDir: {} 74 | -------------------------------------------------------------------------------- /manifests/etcd/etcd-service.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Service 3 | metadata: 4 | labels: 5 | k8s-on-k8s: etcd 6 | name: etcd0 7 | spec: 8 | clusterIP: None 9 | ports: 10 | - name: client 11 | port: 2379 12 | protocol: TCP 13 | - name: peer 14 | port: 2380 15 | protocol: TCP 16 | selector: 17 | k8s-on-k8s: etcd 18 | sessionAffinity: None 19 | -------------------------------------------------------------------------------- /manifests/kcm/kube-controller-manager.yaml: -------------------------------------------------------------------------------- 1 | kind: Deployment 2 | apiVersion: extensions/v1beta1 3 | metadata: 4 | name: kube-controller-manager 5 | labels: 6 | k8s-on-k8s: kube-controller-manager 7 | spec: 8 | replicas: 1 9 | selector: 10 | matchLabels: 11 | k8s-on-k8s: kube-controller-manager 12 | tier: app 13 | template: 14 | metadata: 15 | labels: 16 | k8s-on-k8s: kube-controller-manager 17 | tier: app 18 | spec: 19 | restartPolicy: Always 20 | schedulerName: default-scheduler 21 | securityContext: 22 | runAsUser: 65534 23 | runAsNonRoot: true 24 | containers: 25 | - resources: {} 26 | terminationMessagePath: /dev/termination-log 27 | name: kube-controller-manager 28 | command: 29 | - ./hyperkube 30 | - controller-manager 31 | - '--controllers=*,bootstrapsigner,tokencleaner' 32 | - '--cluster-signing-cert-file=/etc/kubernetes/secrets/ca.crt' 33 | - '--cluster-signing-key-file=/etc/kubernetes/secrets/ca.key' 34 | - '--kubeconfig=/etc/kubernetes/secrets/kubeconfig' 35 | - '--use-service-account-credentials' 36 | - '--allocate-node-cidrs=true' 37 | - '--configure-cloud-routes=false' 38 | - '--cluster-cidr=10.2.0.0/16' 39 | - '--root-ca-file=/etc/kubernetes/secrets/ca.crt' 40 | - '--service-account-private-key-file=/etc/kubernetes/secrets/service-account.key' 41 | - '--leader-elect=true' 42 | - '--cloud-provider=' 43 | livenessProbe: 44 | httpGet: 45 | path: /healthz 46 | port: 10252 47 | scheme: HTTP 48 | initialDelaySeconds: 15 49 | timeoutSeconds: 15 50 | periodSeconds: 10 51 | successThreshold: 1 52 | failureThreshold: 3 53 | imagePullPolicy: IfNotPresent 54 | volumeMounts: 55 | - name: secrets 56 | readOnly: true 57 | mountPath: /etc/kubernetes/secrets 58 | - name: ssl-host 59 | readOnly: true 60 | mountPath: /etc/ssl/certs 61 | terminationMessagePolicy: File 62 | image: gcr.io/google_containers/hyperkube:v1.9.0 63 | volumes: 64 | - name: secrets 65 | secret: 66 | secretName: kube-controller-manager 67 | defaultMode: 420 68 | - name: ssl-host 69 | hostPath: 70 | path: /usr/share/ca-certificates 71 | -------------------------------------------------------------------------------- /manifests/on-k8s/kube-dns/kubedns-cm.yaml: -------------------------------------------------------------------------------- 1 | # Copyright 2016 The Kubernetes Authors. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | apiVersion: v1 16 | kind: ConfigMap 17 | metadata: 18 | name: kube-dns 19 | namespace: kube-system 20 | labels: 21 | addonmanager.kubernetes.io/mode: EnsureExists 22 | -------------------------------------------------------------------------------- /manifests/on-k8s/kube-dns/kubedns-controller.yaml: -------------------------------------------------------------------------------- 1 | # Copyright 2016 The Kubernetes Authors. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | # Should keep target in cluster/addons/dns-horizontal-autoscaler/dns-horizontal-autoscaler.yaml 16 | # in sync with this file. 17 | 18 | # __MACHINE_GENERATED_WARNING__ 19 | 20 | apiVersion: extensions/v1beta1 21 | kind: Deployment 22 | metadata: 23 | name: kube-dns 24 | namespace: kube-system 25 | labels: 26 | k8s-app: kube-dns 27 | kubernetes.io/cluster-service: "true" 28 | addonmanager.kubernetes.io/mode: Reconcile 29 | spec: 30 | # replicas: not specified here: 31 | # 1. In order to make Addon Manager do not reconcile this replicas parameter. 32 | # 2. Default is 1. 33 | # 3. Will be tuned in real time if DNS horizontal auto-scaling is turned on. 34 | strategy: 35 | rollingUpdate: 36 | maxSurge: 10% 37 | maxUnavailable: 0 38 | selector: 39 | matchLabels: 40 | k8s-app: kube-dns 41 | template: 42 | metadata: 43 | labels: 44 | k8s-app: kube-dns 45 | annotations: 46 | scheduler.alpha.kubernetes.io/critical-pod: '' 47 | spec: 48 | tolerations: 49 | - key: "CriticalAddonsOnly" 50 | operator: "Exists" 51 | volumes: 52 | - name: kube-dns-config 53 | configMap: 54 | name: kube-dns 55 | optional: true 56 | containers: 57 | - name: kubedns 58 | image: gcr.io/google_containers/k8s-dns-kube-dns-amd64:1.14.4 59 | resources: 60 | # TODO: Set memory limits when we've profiled the container for large 61 | # clusters, then set request = limit to keep this container in 62 | # guaranteed class. Currently, this container falls into the 63 | # "burstable" category so the kubelet doesn't backoff from restarting it. 64 | limits: 65 | memory: 170Mi 66 | requests: 67 | cpu: 100m 68 | memory: 70Mi 69 | livenessProbe: 70 | httpGet: 71 | path: /healthcheck/kubedns 72 | port: 10054 73 | scheme: HTTP 74 | initialDelaySeconds: 60 75 | timeoutSeconds: 5 76 | successThreshold: 1 77 | failureThreshold: 5 78 | readinessProbe: 79 | httpGet: 80 | path: /readiness 81 | port: 8081 82 | scheme: HTTP 83 | # we poll on pod startup for the Kubernetes master service and 84 | # only setup the /readiness HTTP server once that's available. 85 | initialDelaySeconds: 3 86 | timeoutSeconds: 5 87 | args: 88 | - --domain=cluster.local 89 | - --dns-port=10053 90 | - --config-dir=/kube-dns-config 91 | - --v=2 92 | env: 93 | - name: PROMETHEUS_PORT 94 | value: "10055" 95 | ports: 96 | - containerPort: 10053 97 | name: dns-local 98 | protocol: UDP 99 | - containerPort: 10053 100 | name: dns-tcp-local 101 | protocol: TCP 102 | - containerPort: 10055 103 | name: metrics 104 | protocol: TCP 105 | volumeMounts: 106 | - name: kube-dns-config 107 | mountPath: /kube-dns-config 108 | - name: dnsmasq 109 | image: gcr.io/google_containers/k8s-dns-dnsmasq-nanny-amd64:1.14.4 110 | livenessProbe: 111 | httpGet: 112 | path: /healthcheck/dnsmasq 113 | port: 10054 114 | scheme: HTTP 115 | initialDelaySeconds: 60 116 | timeoutSeconds: 5 117 | successThreshold: 1 118 | failureThreshold: 5 119 | args: 120 | - -v=2 121 | - -logtostderr 122 | - -configDir=/etc/k8s/dns/dnsmasq-nanny 123 | - -restartDnsmasq=true 124 | - -- 125 | - -k 126 | - --cache-size=1000 127 | - --log-facility=- 128 | - --server=/cluster.local/127.0.0.1#10053 129 | - --server=/in-addr.arpa/127.0.0.1#10053 130 | - --server=/ip6.arpa/127.0.0.1#10053 131 | ports: 132 | - containerPort: 53 133 | name: dns 134 | protocol: UDP 135 | - containerPort: 53 136 | name: dns-tcp 137 | protocol: TCP 138 | # see: https://github.com/kubernetes/kubernetes/issues/29055 for details 139 | resources: 140 | requests: 141 | cpu: 150m 142 | memory: 20Mi 143 | volumeMounts: 144 | - name: kube-dns-config 145 | mountPath: /etc/k8s/dns/dnsmasq-nanny 146 | - name: sidecar 147 | image: gcr.io/google_containers/k8s-dns-sidecar-amd64:1.14.4 148 | livenessProbe: 149 | httpGet: 150 | path: /metrics 151 | port: 10054 152 | scheme: HTTP 153 | initialDelaySeconds: 60 154 | timeoutSeconds: 5 155 | successThreshold: 1 156 | failureThreshold: 5 157 | args: 158 | - --v=2 159 | - --logtostderr 160 | - --probe=kubedns,127.0.0.1:10053,kubernetes.default.svc.cluster.local,5,A 161 | - --probe=dnsmasq,127.0.0.1:53,kubernetes.default.svc.cluster.local,5,A 162 | ports: 163 | - containerPort: 10054 164 | name: metrics 165 | protocol: TCP 166 | resources: 167 | requests: 168 | memory: 20Mi 169 | cpu: 10m 170 | dnsPolicy: Default # Don't use cluster DNS. 171 | serviceAccountName: kube-dns 172 | -------------------------------------------------------------------------------- /manifests/on-k8s/kube-dns/kubedns-sa.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: ServiceAccount 3 | metadata: 4 | name: kube-dns 5 | namespace: kube-system 6 | labels: 7 | kubernetes.io/cluster-service: "true" 8 | addonmanager.kubernetes.io/mode: Reconcile 9 | -------------------------------------------------------------------------------- /manifests/on-k8s/kube-dns/kubedns-svc.yaml: -------------------------------------------------------------------------------- 1 | # Copyright 2016 The Kubernetes Authors. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | # __MACHINE_GENERATED_WARNING__ 16 | 17 | apiVersion: v1 18 | kind: Service 19 | metadata: 20 | name: kube-dns 21 | namespace: kube-system 22 | labels: 23 | k8s-app: kube-dns 24 | kubernetes.io/cluster-service: "true" 25 | addonmanager.kubernetes.io/mode: Reconcile 26 | kubernetes.io/name: "KubeDNS" 27 | spec: 28 | selector: 29 | k8s-app: kube-dns 30 | clusterIP: 10.3.0.10 31 | ports: 32 | - name: dns 33 | port: 53 34 | protocol: UDP 35 | - name: dns-tcp 36 | port: 53 37 | protocol: TCP 38 | -------------------------------------------------------------------------------- /manifests/on-k8s/kube-flannel/kube-flannel-rbac.yml: -------------------------------------------------------------------------------- 1 | # Create the clusterrole and clusterrolebinding: 2 | # $ kubectl create -f kube-flannel-rbac.yml 3 | # Create the pod using the same namespace used by the flannel serviceaccount: 4 | # $ kubectl create --namespace kube-system -f kube-flannel.yml 5 | --- 6 | kind: ClusterRole 7 | apiVersion: rbac.authorization.k8s.io/v1beta1 8 | metadata: 9 | name: flannel 10 | rules: 11 | - apiGroups: 12 | - "" 13 | resources: 14 | - pods 15 | verbs: 16 | - get 17 | - apiGroups: 18 | - "" 19 | resources: 20 | - nodes 21 | verbs: 22 | - list 23 | - watch 24 | - apiGroups: 25 | - "" 26 | resources: 27 | - nodes/status 28 | verbs: 29 | - patch 30 | --- 31 | kind: ClusterRoleBinding 32 | apiVersion: rbac.authorization.k8s.io/v1beta1 33 | metadata: 34 | name: flannel 35 | roleRef: 36 | apiGroup: rbac.authorization.k8s.io 37 | kind: ClusterRole 38 | name: flannel 39 | subjects: 40 | - kind: ServiceAccount 41 | name: flannel 42 | namespace: kube-system 43 | -------------------------------------------------------------------------------- /manifests/on-k8s/kube-flannel/kube-flannel.yml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: v1 3 | kind: ServiceAccount 4 | metadata: 5 | name: flannel 6 | namespace: kube-system 7 | --- 8 | kind: ConfigMap 9 | apiVersion: v1 10 | metadata: 11 | name: kube-flannel-cfg 12 | namespace: kube-system 13 | labels: 14 | tier: node 15 | app: flannel 16 | data: 17 | cni-conf.json: | 18 | { 19 | "name": "cbr0", 20 | "type": "flannel", 21 | "delegate": { 22 | "isDefaultGateway": true 23 | } 24 | } 25 | net-conf.json: | 26 | { 27 | "Network": "10.2.0.0/16", 28 | "Backend": { 29 | "Type": "vxlan" 30 | } 31 | } 32 | --- 33 | apiVersion: extensions/v1beta1 34 | kind: DaemonSet 35 | metadata: 36 | name: kube-flannel-ds 37 | namespace: kube-system 38 | labels: 39 | tier: node 40 | app: flannel 41 | spec: 42 | template: 43 | metadata: 44 | labels: 45 | tier: node 46 | app: flannel 47 | spec: 48 | hostNetwork: true 49 | nodeSelector: 50 | beta.kubernetes.io/arch: amd64 51 | tolerations: 52 | - key: node-role.kubernetes.io/master 53 | operator: Exists 54 | effect: NoSchedule 55 | serviceAccountName: flannel 56 | containers: 57 | - name: kube-flannel 58 | image: quay.io/coreos/flannel:v0.8.0-amd64 59 | command: [ "/opt/bin/flanneld", "--ip-masq", "--kube-subnet-mgr"] 60 | securityContext: 61 | privileged: true 62 | env: 63 | - name: POD_NAME 64 | valueFrom: 65 | fieldRef: 66 | fieldPath: metadata.name 67 | - name: POD_NAMESPACE 68 | valueFrom: 69 | fieldRef: 70 | fieldPath: metadata.namespace 71 | volumeMounts: 72 | - name: run 73 | mountPath: /run 74 | - name: flannel-cfg 75 | mountPath: /etc/kube-flannel/ 76 | - name: install-cni 77 | image: quay.io/coreos/flannel:v0.8.0-amd64 78 | command: [ "/bin/sh", "-c", "set -e -x; cp -f /etc/kube-flannel/cni-conf.json /etc/cni/net.d/10-flannel.conf; while true; do sleep 3600; done" ] 79 | volumeMounts: 80 | - name: cni 81 | mountPath: /etc/cni/net.d 82 | - name: flannel-cfg 83 | mountPath: /etc/kube-flannel/ 84 | volumes: 85 | - name: run 86 | hostPath: 87 | path: /run 88 | - name: cni 89 | hostPath: 90 | path: /etc/cni/net.d 91 | - name: flannel-cfg 92 | configMap: 93 | name: kube-flannel-cfg 94 | -------------------------------------------------------------------------------- /manifests/on-k8s/kube-proxy/kube-proxy.yaml: -------------------------------------------------------------------------------- 1 | kind: DaemonSet 2 | apiVersion: extensions/v1beta1 3 | metadata: 4 | name: kube-proxy 5 | namespace: kube-system 6 | labels: 7 | k8s-app: kube-proxy 8 | tier: node 9 | spec: 10 | selector: 11 | matchLabels: 12 | k8s-app: kube-proxy 13 | tier: node 14 | template: 15 | metadata: 16 | creationTimestamp: null 17 | labels: 18 | k8s-app: kube-proxy 19 | tier: node 20 | annotations: 21 | scheduler.alpha.kubernetes.io/critical-pod: '' 22 | spec: 23 | restartPolicy: Always 24 | schedulerName: default-scheduler 25 | hostNetwork: true 26 | terminationGracePeriodSeconds: 30 27 | securityContext: {} 28 | containers: 29 | - resources: {} 30 | terminationMessagePath: /dev/termination-log 31 | name: kube-proxy 32 | command: 33 | - ./hyperkube 34 | - proxy 35 | - '--kubeconfig=/etc/kubernetes/secrets/kubeconfig-proxy' 36 | - '--proxy-mode=iptables' 37 | - '--hostname-override=$(NODE_NAME)' 38 | - '--cluster-cidr=10.2.0.0/16' 39 | env: 40 | - name: NODE_NAME 41 | valueFrom: 42 | fieldRef: 43 | apiVersion: v1 44 | fieldPath: spec.nodeName 45 | securityContext: 46 | privileged: true 47 | imagePullPolicy: IfNotPresent 48 | volumeMounts: 49 | - name: ssl-certs-host 50 | readOnly: true 51 | mountPath: /etc/ssl/certs 52 | - name: secrets 53 | readOnly: true 54 | mountPath: /etc/kubernetes/secrets 55 | terminationMessagePolicy: File 56 | image: 'gcr.io/google_containers/hyperkube:v1.7.2' 57 | volumes: 58 | - name: ssl-certs-host 59 | hostPath: 60 | path: /usr/share/ca-certificates 61 | - name: secrets 62 | secret: 63 | secretName: kubeconfig-proxy 64 | defaultMode: 420 65 | dnsPolicy: ClusterFirst 66 | updateStrategy: 67 | type: RollingUpdate 68 | rollingUpdate: 69 | maxUnavailable: 1 70 | templateGeneration: 1 71 | -------------------------------------------------------------------------------- /manifests/scheduler/kube-scheduler.yaml: -------------------------------------------------------------------------------- 1 | kind: Deployment 2 | apiVersion: extensions/v1beta1 3 | metadata: 4 | name: kube-scheduler 5 | labels: 6 | k8s-on-k8s: kube-scheduler 7 | spec: 8 | replicas: 1 9 | selector: 10 | matchLabels: 11 | k8s-on-k8s: kube-scheduler 12 | tier: app 13 | template: 14 | metadata: 15 | labels: 16 | k8s-on-k8s: kube-scheduler 17 | tier: app 18 | spec: 19 | restartPolicy: Always 20 | schedulerName: default-scheduler 21 | securityContext: 22 | runAsUser: 65534 23 | runAsNonRoot: true 24 | containers: 25 | - resources: {} 26 | terminationMessagePath: /dev/termination-log 27 | name: kube-scheduler 28 | command: 29 | - ./hyperkube 30 | - scheduler 31 | - '--kubeconfig=/etc/kubernetes/secrets/kubeconfig' 32 | - '--leader-elect=true' 33 | livenessProbe: 34 | httpGet: 35 | path: /healthz 36 | port: 10251 37 | scheme: HTTP 38 | initialDelaySeconds: 15 39 | timeoutSeconds: 15 40 | periodSeconds: 10 41 | successThreshold: 1 42 | failureThreshold: 3 43 | imagePullPolicy: IfNotPresent 44 | volumeMounts: 45 | - name: secrets 46 | readOnly: true 47 | mountPath: /etc/kubernetes/secrets 48 | - name: ssl-host 49 | readOnly: true 50 | mountPath: /etc/ssl/certs 51 | terminationMessagePolicy: File 52 | image: gcr.io/google_containers/hyperkube:v1.9.0 53 | volumes: 54 | - name: secrets 55 | secret: 56 | secretName: kube-scheduler 57 | defaultMode: 420 58 | - name: ssl-host 59 | hostPath: 60 | path: /usr/share/ca-certificates 61 | -------------------------------------------------------------------------------- /tls/ca-config.json: -------------------------------------------------------------------------------- 1 | { 2 | "signing": { 3 | "default": { 4 | "expiry": "8760h" 5 | }, 6 | "profiles": { 7 | "server": { 8 | "usages": [ 9 | "signing", 10 | "key encipherment", 11 | "server auth" 12 | ], 13 | "expiry": "8760h" 14 | }, 15 | "client": { 16 | "usages": [ 17 | "signing", 18 | "key encipherment", 19 | "client auth" 20 | ], 21 | "expiry": "8760h" 22 | } 23 | } 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /tls/ca-csr.json: -------------------------------------------------------------------------------- 1 | { 2 | "CN": "Kubernetes", 3 | "key": { 4 | "algo": "rsa", 5 | "size": 2048 6 | }, 7 | "names": [ 8 | { 9 | "C": "US", 10 | "L": "Portland", 11 | "O": "Kubernetes", 12 | "OU": "CA", 13 | "ST": "Oregon" 14 | } 15 | ] 16 | } 17 | -------------------------------------------------------------------------------- /tls/kube-apiserver-server-csr.json: -------------------------------------------------------------------------------- 1 | { 2 | "CN": "kube-apiserver", 3 | "hosts": [ 4 | "127.0.0.1", 5 | "10.3.0.1", 6 | "kubernetes", 7 | "k8s.foo.somedomain.edu" 8 | ], 9 | "key": { 10 | "algo": "rsa", 11 | "size": 2048 12 | }, 13 | "names": [ 14 | { 15 | "C": "US", 16 | "L": "Portland", 17 | "O": "Kubernetes", 18 | "OU": "API Server", 19 | "ST": "Oregon" 20 | } 21 | ] 22 | } 23 | -------------------------------------------------------------------------------- /tls/kube-controller-manager-client-csr.json: -------------------------------------------------------------------------------- 1 | { 2 | "CN": "system:kube-controller-manager", 3 | "key": { 4 | "algo": "rsa", 5 | "size": 2048 6 | }, 7 | "names": [ 8 | { 9 | "C": "US", 10 | "L": "Portland", 11 | "O": "system:kube-controller-manager", 12 | "OU": "Controller Manager", 13 | "ST": "Oregon" 14 | } 15 | ] 16 | } 17 | -------------------------------------------------------------------------------- /tls/kube-proxy-client-csr.json: -------------------------------------------------------------------------------- 1 | { 2 | "CN": "system:kube-proxy", 3 | "hosts": [""], 4 | "key": { 5 | "algo": "rsa", 6 | "size": 2048 7 | }, 8 | "names": [ 9 | { 10 | "C": "US", 11 | "L": "Portland", 12 | "O": "system:kube-proxy", 13 | "OU": "Proxy", 14 | "ST": "Oregon" 15 | } 16 | ] 17 | } 18 | -------------------------------------------------------------------------------- /tls/kube-scheduler-client-csr.json: -------------------------------------------------------------------------------- 1 | { 2 | "CN": "system:kube-scheduler", 3 | "key": { 4 | "algo": "rsa", 5 | "size": 2048 6 | }, 7 | "names": [ 8 | { 9 | "C": "US", 10 | "L": "Portland", 11 | "O": "system:kube-scheduler", 12 | "OU": "Scheduler", 13 | "ST": "Oregon" 14 | } 15 | ] 16 | } 17 | -------------------------------------------------------------------------------- /tls/kubelet-client-csr.json: -------------------------------------------------------------------------------- 1 | { 2 | "CN": "system:nodes", 3 | "hosts": [""], 4 | "key": { 5 | "algo": "rsa", 6 | "size": 2048 7 | }, 8 | "names": [ 9 | { 10 | "C": "US", 11 | "L": "Portland", 12 | "O": "system:nodes", 13 | "OU": "Kubelet", 14 | "ST": "Oregon" 15 | } 16 | ] 17 | } 18 | -------------------------------------------------------------------------------- /tls/kubernetes-admin-user.csr.json: -------------------------------------------------------------------------------- 1 | { 2 | "CN": "admin", 3 | "hosts": [""], 4 | "key": { 5 | "algo": "rsa", 6 | "size": 2048 7 | }, 8 | "names": [ 9 | { 10 | "C": "US", 11 | "L": "Portland", 12 | "O": "system:masters", 13 | "OU": "Cluster Admins", 14 | "ST": "Oregon" 15 | } 16 | ] 17 | } 18 | -------------------------------------------------------------------------------- /tls/make_certs.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | APIURL=$1 4 | 5 | [ -z "${APIURL}" ] && echo "Usage: make_cert.sh apihost[:port]" && exit 1 6 | APIHOST=$(echo ${APIURL} | cut -f1 -d":") 7 | 8 | # Init CA 9 | cfssl gencert -initca ca-csr.json | cfssljson -bare ca 10 | 11 | # Apiserver 12 | cat < kube-apiserver-server-csr.json 13 | { 14 | "CN": "kube-apiserver", 15 | "hosts": [ 16 | "127.0.0.1", 17 | "10.3.0.1", 18 | "kubernetes", 19 | "${APIHOST}" 20 | ], 21 | "key": { 22 | "algo": "rsa", 23 | "size": 2048 24 | }, 25 | "names": [ 26 | { 27 | "C": "US", 28 | "L": "Portland", 29 | "O": "Kubernetes", 30 | "OU": "API Server", 31 | "ST": "Oregon" 32 | } 33 | ] 34 | } 35 | EOF 36 | 37 | cfssl gencert \ 38 | -ca=ca.pem \ 39 | -ca-key=ca-key.pem \ 40 | -config=ca-config.json \ 41 | -profile=server \ 42 | kube-apiserver-server-csr.json | cfssljson -bare kube-apiserver-server 43 | 44 | 45 | # Scheduler 46 | cfssl gencert \ 47 | -ca=ca.pem \ 48 | -ca-key=ca-key.pem \ 49 | -config=ca-config.json \ 50 | -profile=client \ 51 | kube-scheduler-client-csr.json | cfssljson -bare kube-scheduler-client 52 | 53 | # KCM 54 | cfssl gencert \ 55 | -ca=ca.pem \ 56 | -ca-key=ca-key.pem \ 57 | -config=ca-config.json \ 58 | -profile=client \ 59 | kube-controller-manager-client-csr.json | cfssljson -bare kube-controller-manager-client 60 | 61 | # Kube-proxy 62 | cfssl gencert \ 63 | -ca=ca.pem \ 64 | -ca-key=ca-key.pem \ 65 | -config=ca-config.json \ 66 | -profile=client \ 67 | kube-proxy-client-csr.json | cfssljson -bare kube-proxy-client 68 | 69 | 70 | # Kubelet 71 | cfssl gencert \ 72 | -ca=ca.pem \ 73 | -ca-key=ca-key.pem \ 74 | -config=ca-config.json \ 75 | -profile=client \ 76 | kubelet-client-csr.json | cfssljson -bare kubelet-client 77 | 78 | # Admin certs 79 | cfssl gencert \ 80 | -ca=ca.pem \ 81 | -ca-key=ca-key.pem \ 82 | -config=ca-config.json \ 83 | -profile=client \ 84 | kubernetes-admin-user.csr.json | cfssljson -bare kubernetes-admin-user 85 | 86 | 87 | 88 | cat < apiserver-secret.yaml 89 | apiVersion: v1 90 | data: 91 | apiserver.crt: $(cat kube-apiserver-server.pem | base64 | tr -d '\n') 92 | apiserver.key: $(cat kube-apiserver-server-key.pem | base64 | tr -d '\n') 93 | ca.crt: $(cat ca.pem | base64 | tr -d '\n') 94 | ca.key: $(cat ca-key.pem | base64 | tr -d '\n') 95 | etcd-ca.crt: "" 96 | etcd-client.crt: "" 97 | etcd-client.key: "" 98 | kind: Secret 99 | metadata: 100 | name: kube-apiserver 101 | type: Opaque 102 | EOF 103 | 104 | 105 | # Gen kubeconfig file for the admin users 106 | cat < kubeconfig 107 | apiVersion: v1 108 | kind: Config 109 | clusters: 110 | - name: local 111 | cluster: 112 | server: https://${APIURL} 113 | certificate-authority-data: $(cat ca.pem | base64 | tr -d '\n') 114 | users: 115 | - name: admin 116 | user: 117 | client-certificate-data: $(cat kubernetes-admin-user.pem | base64 | tr -d '\n') 118 | client-key-data: $(cat kubernetes-admin-user-key.pem | base64 | tr -d '\n') 119 | contexts: 120 | - context: 121 | cluster: local 122 | user: admin 123 | EOF 124 | 125 | 126 | # Store kubeconfig as a secret 127 | cat < apiserver-admin-secret.yaml 128 | apiVersion: v1 129 | data: 130 | kubeconfig: $(cat kubeconfig | base64 | tr -d '\n') 131 | kind: Secret 132 | metadata: 133 | name: admin-kubeconfig 134 | type: Opaque 135 | EOF 136 | 137 | 138 | # Gen kubeconfig file for KCM 139 | cat < kubeconfig-kcm 140 | apiVersion: v1 141 | kind: Config 142 | clusters: 143 | - name: local 144 | cluster: 145 | server: https://${APIURL} 146 | certificate-authority-data: $(cat ca.pem | base64 | tr -d '\n') 147 | users: 148 | - name: kcm 149 | user: 150 | client-certificate-data: $(cat kube-controller-manager-client.pem | base64 | tr -d '\n') 151 | client-key-data: $(cat kube-controller-manager-client-key.pem | base64 | tr -d '\n') 152 | contexts: 153 | - context: 154 | cluster: local 155 | user: kcm 156 | EOF 157 | 158 | 159 | # Controller manager secret 160 | cat < controller-manager-secret.yaml 161 | kind: Secret 162 | apiVersion: v1 163 | metadata: 164 | name: kube-controller-manager 165 | data: 166 | ca.crt: $(cat ca.pem | base64 | tr -d '\n') 167 | ca.key: $(cat ca-key.pem | base64 | tr -d '\n') 168 | service-account.key: $(cat kube-apiserver-server-key.pem | base64 | tr -d '\n') 169 | kubeconfig: $(cat kubeconfig-kcm | base64 | tr -d '\n') 170 | type: Opaque 171 | EOF 172 | 173 | 174 | 175 | # Gen kubeconfig file for scheduler 176 | cat < kubeconfig-scheduler 177 | apiVersion: v1 178 | kind: Config 179 | clusters: 180 | - name: local 181 | cluster: 182 | server: https://${APIURL} 183 | certificate-authority-data: $(cat ca.pem | base64 | tr -d '\n') 184 | users: 185 | - name: scheduler 186 | user: 187 | client-certificate-data: $(cat kube-scheduler-client.pem | base64 | tr -d '\n') 188 | client-key-data: $(cat kube-scheduler-client-key.pem | base64 | tr -d '\n') 189 | contexts: 190 | - context: 191 | cluster: local 192 | user: scheduler 193 | EOF 194 | 195 | 196 | # Controller manager secret 197 | cat < scheduler-secret.yaml 198 | kind: Secret 199 | apiVersion: v1 200 | metadata: 201 | name: kube-scheduler 202 | data: 203 | ca.crt: $(cat ca.pem | base64 | tr -d '\n') 204 | kubeconfig: $(cat kubeconfig-scheduler | base64 | tr -d '\n') 205 | type: Opaque 206 | EOF 207 | 208 | 209 | # Gen kubeconfig file for the kubelets 210 | cat < kubeconfig-kubelets 211 | apiVersion: v1 212 | kind: Config 213 | clusters: 214 | - name: local 215 | cluster: 216 | server: https://${APIURL} 217 | certificate-authority-data: $(cat ca.pem | base64 | tr -d '\n') 218 | users: 219 | - name: admin 220 | user: 221 | client-certificate-data: $(cat kubelet-client.pem | base64 | tr -d '\n') 222 | client-key-data: $(cat kubelet-client-key.pem | base64 | tr -d '\n') 223 | contexts: 224 | - context: 225 | cluster: local 226 | user: admin 227 | EOF 228 | 229 | 230 | # Gen kubeconfig file for proxy 231 | cat < kubeconfig-proxy 232 | apiVersion: v1 233 | kind: Config 234 | clusters: 235 | - name: local 236 | cluster: 237 | server: https://${APIURL} 238 | certificate-authority-data: $(cat ca.pem | base64 | tr -d '\n') 239 | users: 240 | - name: proxy 241 | user: 242 | client-certificate-data: $(cat kube-proxy-client.pem | base64 | tr -d '\n') 243 | client-key-data: $(cat kube-proxy-client-key.pem | base64 | tr -d '\n') 244 | contexts: 245 | - context: 246 | cluster: local 247 | user: proxy 248 | EOF 249 | --------------------------------------------------------------------------------