├── .gitignore ├── LICENSE ├── README.md ├── deployments ├── federation-apiserver.yaml └── federation-controller-manager.yaml ├── ingress └── nginx.yaml ├── labs ├── 01-cluster-bootstrap.md ├── 02-cluster-dns-managed-zone.md ├── 03-download-an-updated-kubectl-client.md ├── 04-provision-federation-apiserver.md ├── 05-provision-federation-controller-manager.md ├── 06-adding-clusters.md ├── 07-federated-nginx-service.md ├── 08-federated-secrets.md ├── 09-cleaning-up.md └── images │ └── googledns.png ├── ns └── federation.yaml ├── pvc └── federation-apiserver-etcd.yaml ├── rs ├── nginx-us.yaml └── nginx.yaml └── services ├── federation-apiserver.yaml └── nginx.yaml /.gitignore: -------------------------------------------------------------------------------- 1 | clusters/ 2 | kubeconfigs/ 3 | known-tokens.csv 4 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | 2 | Apache License 3 | Version 2.0, January 2004 4 | http://www.apache.org/licenses/ 5 | 6 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 7 | 8 | 1. Definitions. 9 | 10 | "License" shall mean the terms and conditions for use, reproduction, 11 | and distribution as defined by Sections 1 through 9 of this document. 12 | 13 | "Licensor" shall mean the copyright owner or entity authorized by 14 | the copyright owner that is granting the License. 15 | 16 | "Legal Entity" shall mean the union of the acting entity and all 17 | other entities that control, are controlled by, or are under common 18 | control with that entity. For the purposes of this definition, 19 | "control" means (i) the power, direct or indirect, to cause the 20 | direction or management of such entity, whether by contract or 21 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 22 | outstanding shares, or (iii) beneficial ownership of such entity. 23 | 24 | "You" (or "Your") shall mean an individual or Legal Entity 25 | exercising permissions granted by this License. 26 | 27 | "Source" form shall mean the preferred form for making modifications, 28 | including but not limited to software source code, documentation 29 | source, and configuration files. 30 | 31 | "Object" form shall mean any form resulting from mechanical 32 | transformation or translation of a Source form, including but 33 | not limited to compiled object code, generated documentation, 34 | and conversions to other media types. 35 | 36 | "Work" shall mean the work of authorship, whether in Source or 37 | Object form, made available under the License, as indicated by a 38 | copyright notice that is included in or attached to the work 39 | (an example is provided in the Appendix below). 40 | 41 | "Derivative Works" shall mean any work, whether in Source or Object 42 | form, that is based on (or derived from) the Work and for which the 43 | editorial revisions, annotations, elaborations, or other modifications 44 | represent, as a whole, an original work of authorship. For the purposes 45 | of this License, Derivative Works shall not include works that remain 46 | separable from, or merely link (or bind by name) to the interfaces of, 47 | the Work and Derivative Works thereof. 48 | 49 | "Contribution" shall mean any work of authorship, including 50 | the original version of the Work and any modifications or additions 51 | to that Work or Derivative Works thereof, that is intentionally 52 | submitted to Licensor for inclusion in the Work by the copyright owner 53 | or by an individual or Legal Entity authorized to submit on behalf of 54 | the copyright owner. For the purposes of this definition, "submitted" 55 | means any form of electronic, verbal, or written communication sent 56 | to the Licensor or its representatives, including but not limited to 57 | communication on electronic mailing lists, source code control systems, 58 | and issue tracking systems that are managed by, or on behalf of, the 59 | Licensor for the purpose of discussing and improving the Work, but 60 | excluding communication that is conspicuously marked or otherwise 61 | designated in writing by the copyright owner as "Not a Contribution." 62 | 63 | "Contributor" shall mean Licensor and any individual or Legal Entity 64 | on behalf of whom a Contribution has been received by Licensor and 65 | subsequently incorporated within the Work. 66 | 67 | 2. Grant of Copyright License. Subject to the terms and conditions of 68 | this License, each Contributor hereby grants to You a perpetual, 69 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 70 | copyright license to reproduce, prepare Derivative Works of, 71 | publicly display, publicly perform, sublicense, and distribute the 72 | Work and such Derivative Works in Source or Object form. 73 | 74 | 3. Grant of Patent License. Subject to the terms and conditions of 75 | this License, each Contributor hereby grants to You a perpetual, 76 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 77 | (except as stated in this section) patent license to make, have made, 78 | use, offer to sell, sell, import, and otherwise transfer the Work, 79 | where such license applies only to those patent claims licensable 80 | by such Contributor that are necessarily infringed by their 81 | Contribution(s) alone or by combination of their Contribution(s) 82 | with the Work to which such Contribution(s) was submitted. If You 83 | institute patent litigation against any entity (including a 84 | cross-claim or counterclaim in a lawsuit) alleging that the Work 85 | or a Contribution incorporated within the Work constitutes direct 86 | or contributory patent infringement, then any patent licenses 87 | granted to You under this License for that Work shall terminate 88 | as of the date such litigation is filed. 89 | 90 | 4. Redistribution. You may reproduce and distribute copies of the 91 | Work or Derivative Works thereof in any medium, with or without 92 | modifications, and in Source or Object form, provided that You 93 | meet the following conditions: 94 | 95 | (a) You must give any other recipients of the Work or 96 | Derivative Works a copy of this License; and 97 | 98 | (b) You must cause any modified files to carry prominent notices 99 | stating that You changed the files; and 100 | 101 | (c) You must retain, in the Source form of any Derivative Works 102 | that You distribute, all copyright, patent, trademark, and 103 | attribution notices from the Source form of the Work, 104 | excluding those notices that do not pertain to any part of 105 | the Derivative Works; and 106 | 107 | (d) If the Work includes a "NOTICE" text file as part of its 108 | distribution, then any Derivative Works that You distribute must 109 | include a readable copy of the attribution notices contained 110 | within such NOTICE file, excluding those notices that do not 111 | pertain to any part of the Derivative Works, in at least one 112 | of the following places: within a NOTICE text file distributed 113 | as part of the Derivative Works; within the Source form or 114 | documentation, if provided along with the Derivative Works; or, 115 | within a display generated by the Derivative Works, if and 116 | wherever such third-party notices normally appear. The contents 117 | of the NOTICE file are for informational purposes only and 118 | do not modify the License. You may add Your own attribution 119 | notices within Derivative Works that You distribute, alongside 120 | or as an addendum to the NOTICE text from the Work, provided 121 | that such additional attribution notices cannot be construed 122 | as modifying the License. 123 | 124 | You may add Your own copyright statement to Your modifications and 125 | may provide additional or different license terms and conditions 126 | for use, reproduction, or distribution of Your modifications, or 127 | for any such Derivative Works as a whole, provided Your use, 128 | reproduction, and distribution of the Work otherwise complies with 129 | the conditions stated in this License. 130 | 131 | 5. Submission of Contributions. Unless You explicitly state otherwise, 132 | any Contribution intentionally submitted for inclusion in the Work 133 | by You to the Licensor shall be under the terms and conditions of 134 | this License, without any additional terms or conditions. 135 | Notwithstanding the above, nothing herein shall supersede or modify 136 | the terms of any separate license agreement you may have executed 137 | with Licensor regarding such Contributions. 138 | 139 | 6. Trademarks. This License does not grant permission to use the trade 140 | names, trademarks, service marks, or product names of the Licensor, 141 | except as required for reasonable and customary use in describing the 142 | origin of the Work and reproducing the content of the NOTICE file. 143 | 144 | 7. Disclaimer of Warranty. Unless required by applicable law or 145 | agreed to in writing, Licensor provides the Work (and each 146 | Contributor provides its Contributions) on an "AS IS" BASIS, 147 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 148 | implied, including, without limitation, any warranties or conditions 149 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 150 | PARTICULAR PURPOSE. You are solely responsible for determining the 151 | appropriateness of using or redistributing the Work and assume any 152 | risks associated with Your exercise of permissions under this License. 153 | 154 | 8. Limitation of Liability. In no event and under no legal theory, 155 | whether in tort (including negligence), contract, or otherwise, 156 | unless required by applicable law (such as deliberate and grossly 157 | negligent acts) or agreed to in writing, shall any Contributor be 158 | liable to You for damages, including any direct, indirect, special, 159 | incidental, or consequential damages of any character arising as a 160 | result of this License or out of the use or inability to use the 161 | Work (including but not limited to damages for loss of goodwill, 162 | work stoppage, computer failure or malfunction, or any and all 163 | other commercial damages or losses), even if such Contributor 164 | has been advised of the possibility of such damages. 165 | 166 | 9. Accepting Warranty or Additional Liability. While redistributing 167 | the Work or Derivative Works thereof, You may choose to offer, 168 | and charge a fee for, acceptance of support, warranty, indemnity, 169 | or other liability obligations and/or rights consistent with this 170 | License. However, in accepting such obligations, You may act only 171 | on Your own behalf and on Your sole responsibility, not on behalf 172 | of any other Contributor, and only if You agree to indemnify, 173 | defend, and hold each Contributor harmless for any liability 174 | incurred by, or claims asserted against, such Contributor by reason 175 | of your accepting any such warranty or additional liability. 176 | 177 | END OF TERMS AND CONDITIONS 178 | 179 | APPENDIX: How to apply the Apache License to your work. 180 | 181 | To apply the Apache License to your work, attach the following 182 | boilerplate notice, with the fields enclosed by brackets "[]" 183 | replaced with your own identifying information. (Don't include 184 | the brackets!) The text should be enclosed in the appropriate 185 | comment syntax for the file format. We also recommend that a 186 | file or class name and description of purpose be included on the 187 | same "printed page" as the copyright notice for easier 188 | identification within third-party archives. 189 | 190 | Copyright [yyyy] [name of copyright owner] 191 | 192 | Licensed under the Apache License, Version 2.0 (the "License"); 193 | you may not use this file except in compliance with the License. 194 | You may obtain a copy of the License at 195 | 196 | http://www.apache.org/licenses/LICENSE-2.0 197 | 198 | Unless required by applicable law or agreed to in writing, software 199 | distributed under the License is distributed on an "AS IS" BASIS, 200 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 201 | See the License for the specific language governing permissions and 202 | limitations under the License. 203 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Kubernetes Cluster Federation (The Hard Way) 2 | 3 | This tutorial will walk you through setting up a Kubernetes cluster federation composed of four Kubernetes clusters across multiple GCP regions. 4 | 5 | This guide is not for people looking for a fully automated command to bring up a Kubernetes cluster federation. If that's you then check out [Setting up Cluster Federation with Kubefed](https://kubernetes.io/docs/tutorials/federation/set-up-cluster-federation-kubefed/). 6 | 7 | This tutorial is optimized for learning, which means taking the long route to help people understand each task required to bootstrap a Kubernetes cluster federation. This tutorial requires access to [Google Compute Engine](https://cloud.google.com/compute). 8 | 9 | ## Prerequisites 10 | 11 | ### Clone This Repository 12 | 13 | Checkout this repository which contains the Kubernetes configs required to provision a federated control plane. 14 | 15 | ``` 16 | git clone https://github.com/kelseyhightower/kubernetes-cluster-federation.git 17 | ``` 18 | 19 | ``` 20 | cd kubernetes-cluster-federation 21 | ``` 22 | 23 | ## Bootstrapping Tasks 24 | 25 | This tutorial will leverage [Google Container Engine](https://cloud.google.com/container-engine) to provision four Kubernetes clusters across multiple GCP regions. 26 | 27 | * [Provision Kubernetes Clusters](labs/01-cluster-bootstrap.md) 28 | * [Create GCE DNS Managed Zone](labs/02-cluster-dns-managed-zone.md) 29 | * [Download an Updated Kubernetes Client](labs/03-download-an-updated-kubectl-client.md) 30 | 31 | ## Provision Federated Control Plane 32 | 33 | The federated control plane must run in a Kubernetes host cluster which has access to a set of cluster configurations and secrets for accessing them. The following labs will walk you through provisioning a federated control plane in the `us-central1-b` cluster. 34 | 35 | * [Provision the Federated API Server](labs/04-provision-federation-apiserver.md) 36 | * [Provision the Federated Controller Manager](labs/05-provision-federation-controller-manager.md) 37 | * [Adding Clusters to the Federation](labs/06-adding-clusters.md) 38 | 39 | ## Running Federated Workloads 40 | 41 | * [Federated NGINX Service](labs/07-federated-nginx-service.md) 42 | * [Federated Secrets](labs/08-federated-secrets.md) 43 | 44 | ## Cleaning Up 45 | 46 | * [Delete Federated Cluster](labs/09-cleaning-up.md) 47 | -------------------------------------------------------------------------------- /deployments/federation-apiserver.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: extensions/v1beta1 2 | kind: Deployment 3 | metadata: 4 | name: federation-apiserver 5 | namespace: federation 6 | labels: 7 | app: federated-cluster 8 | spec: 9 | template: 10 | metadata: 11 | name: federation-apiserver 12 | labels: 13 | app: federated-cluster 14 | module: federation-apiserver 15 | spec: 16 | containers: 17 | - name: apiserver 18 | image: gcr.io/google_containers/hyperkube-amd64:v1.6.2 19 | env: 20 | - name: ADVERTISE_ADDRESS 21 | valueFrom: 22 | configMapKeyRef: 23 | name: federated-apiserver 24 | key: advertise-address 25 | command: 26 | - /hyperkube 27 | - federation-apiserver 28 | - --bind-address=0.0.0.0 29 | - --etcd-servers=http://localhost:2379 30 | - --secure-port=443 31 | - --advertise-address=$(ADVERTISE_ADDRESS) 32 | - --token-auth-file=/srv/kubernetes/known-tokens.csv 33 | ports: 34 | - containerPort: 443 35 | name: https 36 | - containerPort: 8080 37 | name: local 38 | volumeMounts: 39 | - name: federation-apiserver-secrets 40 | mountPath: /srv/kubernetes/ 41 | readOnly: true 42 | - name: etcd 43 | image: quay.io/coreos/etcd:v3.0.7 44 | command: 45 | - "/usr/local/bin/etcd" 46 | args: 47 | - "--data-dir=/var/lib/etcd" 48 | volumeMounts: 49 | - name: etcd-data 50 | mountPath: /var/lib/etcd 51 | volumes: 52 | - name: federation-apiserver-secrets 53 | secret: 54 | secretName: federation-apiserver-secrets 55 | - name: etcd-data 56 | persistentVolumeClaim: 57 | claimName: federation-apiserver-etcd 58 | -------------------------------------------------------------------------------- /deployments/federation-controller-manager.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: extensions/v1beta1 2 | kind: Deployment 3 | metadata: 4 | name: federation-controller-manager 5 | namespace: federation 6 | labels: 7 | app: federated-cluster 8 | spec: 9 | template: 10 | metadata: 11 | name: federation-controller-manager 12 | labels: 13 | app: federated-cluster 14 | module: federation-controller-manager 15 | spec: 16 | volumes: 17 | - name: ssl 18 | hostPath: 19 | path: /etc/ssl 20 | containers: 21 | - name: controller-manager 22 | image: gcr.io/google_containers/hyperkube-amd64:v1.6.2 23 | env: 24 | - name: POD_NAMESPACE 25 | valueFrom: 26 | fieldRef: 27 | fieldPath: metadata.namespace 28 | - name: ZONE_ID 29 | valueFrom: 30 | configMapKeyRef: 31 | name: federation-controller-manager 32 | key: zone-id 33 | - name: ZONE_NAME 34 | valueFrom: 35 | configMapKeyRef: 36 | name: federation-controller-manager 37 | key: zone-name 38 | args: 39 | - /hyperkube 40 | - federation-controller-manager 41 | - --master=https://federation-apiserver:443 42 | - --dns-provider=google-clouddns 43 | - --federation-name=federation 44 | - --zone-id=$(ZONE_ID) 45 | - --zone-name=$(ZONE_NAME) 46 | - --v=2 47 | volumeMounts: 48 | - name: ssl 49 | readOnly: true 50 | mountPath: /etc/ssl 51 | ports: 52 | - containerPort: 443 53 | name: https 54 | - containerPort: 8080 55 | name: local 56 | -------------------------------------------------------------------------------- /ingress/nginx.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: extensions/v1beta1 2 | kind: Ingress 3 | metadata: 4 | annotations: 5 | kubernetes.io/ingress.global-static-ip-name: federation 6 | name: nginx 7 | spec: 8 | backend: 9 | serviceName: nginx 10 | servicePort: 80 11 | -------------------------------------------------------------------------------- /labs/01-cluster-bootstrap.md: -------------------------------------------------------------------------------- 1 | # Provision Kubernetes Clusters 2 | 3 | This tutorial will walk you through setting up Kubernetes clusters in four GCP regions using GKE. 4 | 5 | ## Create Clusters 6 | 7 | Use the `gcloud container clusters create` command to create a Kubernetes clusters in the following zones: 8 | 9 | * asia-east1-b 10 | * europe-west1-b 11 | * us-east1-b 12 | * us-central1-b 13 | 14 | Run the following commands in a separate terminal for parallel execution. 15 | 16 | Create the `asia-east1-b` cluster: 17 | 18 | ``` 19 | gcloud beta container clusters create asia-east1-b \ 20 | --cluster-version 1.6.2 \ 21 | --zone asia-east1-b \ 22 | --scopes "cloud-platform,storage-ro,logging-write,monitoring-write,service-control,service-management,https://www.googleapis.com/auth/ndev.clouddns.readwrite" 23 | ``` 24 | 25 | Create the `europe-west1-b` cluster: 26 | 27 | ``` 28 | gcloud beta container clusters create europe-west1-b \ 29 | --cluster-version 1.6.2 \ 30 | --zone=europe-west1-b \ 31 | --scopes "cloud-platform,storage-ro,logging-write,monitoring-write,service-control,service-management,https://www.googleapis.com/auth/ndev.clouddns.readwrite" 32 | ``` 33 | 34 | Create the `us-east1-b` cluster: 35 | 36 | ``` 37 | gcloud beta container clusters create us-east1-b \ 38 | --cluster-version 1.6.2 \ 39 | --zone=us-east1-b \ 40 | --scopes "cloud-platform,storage-ro,logging-write,monitoring-write,service-control,service-management,https://www.googleapis.com/auth/ndev.clouddns.readwrite" 41 | ``` 42 | 43 | Create the `us-central1-b` cluster: 44 | 45 | ``` 46 | gcloud beta container clusters create us-central1-b \ 47 | --cluster-version 1.6.2 \ 48 | --zone=us-central1-b \ 49 | --scopes "cloud-platform,storage-ro,logging-write,monitoring-write,service-control,service-management,https://www.googleapis.com/auth/ndev.clouddns.readwrite" 50 | ``` 51 | 52 | Save the cluster credentials: 53 | 54 | ``` 55 | for cluster in asia-east1-b europe-west1-b us-east1-b us-central1-b; do 56 | gcloud container clusters get-credentials ${cluster} \ 57 | --zone ${cluster} 58 | done 59 | ``` 60 | 61 | ## Create Cluster Contexts 62 | 63 | ``` 64 | GCP_PROJECT=$(gcloud config list --format='value(core.project)') 65 | ``` 66 | 67 | Create context aliases: 68 | 69 | ``` 70 | for cluster in asia-east1-b europe-west1-b us-east1-b us-central1-b; do 71 | kubectl config set-context ${cluster} \ 72 | --cluster=gke_${GCP_PROJECT}_${cluster}_${cluster} \ 73 | --user=gke_${GCP_PROJECT}_${cluster}_${cluster} 74 | done 75 | ``` 76 | 77 | Create the host cluster context: 78 | 79 | ``` 80 | HOST_CLUSTER=us-central1-b 81 | ``` 82 | 83 | ``` 84 | kubectl config set-context host-cluster \ 85 | --cluster=gke_${GCP_PROJECT}_${HOST_CLUSTER}_${HOST_CLUSTER} \ 86 | --user=gke_${GCP_PROJECT}_${HOST_CLUSTER}_${HOST_CLUSTER} \ 87 | --namespace=federation 88 | ``` 89 | 90 | 91 | ### Verify 92 | 93 | ``` 94 | gcloud container clusters list 95 | ``` 96 | 97 | ``` 98 | NAME ZONE MASTER_VERSION MASTER_IP MACHINE_TYPE NODE_VERSION NUM_NODES STATUS 99 | asia-east1-b asia-east1-b 1.6.2 XXX.XXX.XXX.XXX n1-standard-1 1.6.2 3 RUNNING 100 | europe-west1-b europe-west1-b 1.6.2 XXX.XXX.XX.X n1-standard-1 1.6.2 3 RUNNING 101 | us-central1-b us-central1-b 1.6.2 XXX.XXX.XXX.XX n1-standard-1 1.6.2 3 RUNNING 102 | us-east1-b us-east1-b 1.6.2 XXX.XXX.XXX.XX n1-standard-1 1.6.2 3 RUNNING 103 | ``` 104 | -------------------------------------------------------------------------------- /labs/02-cluster-dns-managed-zone.md: -------------------------------------------------------------------------------- 1 | # Cluster DNS Managed Zone 2 | 3 | Kubernetes federated services have the ability to manage external DNS entries based on services created across a federated set of Kubernetes clusters. In this lab you will setup a [Google DNS managed zone](https://cloud.google.com/dns/zones) to hold the DNS entries. Kubernetes will support other external DNS providers using a plugin based system on the Federated Controller Manager. 4 | 5 | ## Create a Google DNS Managed Zone 6 | 7 | The follow command will create a DNS zone named `federation`. In a production setup a valid managed zone backed by a registered DNS domain should be used. 8 | 9 | ``` 10 | gcloud dns managed-zones create federation \ 11 | --description "Kubernetes federation testing" \ 12 | --dns-name federation.localdomain 13 | ``` 14 | -------------------------------------------------------------------------------- /labs/03-download-an-updated-kubectl-client.md: -------------------------------------------------------------------------------- 1 | # Download an updated kubectl Client 2 | 3 | This guide will walk you through downloading an updated kubectl client. kubectl version 1.6.1+ is required to work with a Federated Kubernetes control plane. 4 | 5 | ### Darwin 6 | ``` 7 | curl -O https://storage.googleapis.com/kubernetes-release/release/v1.6.2/bin/darwin/amd64/kubectl 8 | chmod a+x kubectl 9 | sudo cp kubectl /usr/local/bin/kubectl 10 | ``` 11 | 12 | ### Linux 13 | 14 | ``` 15 | curl -O https://storage.googleapis.com/kubernetes-release/release/v1.6.2/bin/linux/amd64/kubectl 16 | chmod a+x kubectl 17 | sudo cp kubectl /usr/local/bin/kubectl 18 | ``` 19 | 20 | ### Verify 21 | 22 | ``` 23 | /usr/local/bin/kubectl version 24 | ``` 25 | -------------------------------------------------------------------------------- /labs/04-provision-federation-apiserver.md: -------------------------------------------------------------------------------- 1 | # Provision Federated API Server 2 | 3 | The Federated API Server will run in the us-central1 cluster. 4 | 5 | The federated controller manager must be able to locate the federated API server when running on the host cluster. 6 | 7 | ## Prerequisites 8 | 9 | ``` 10 | kubectl config use-context host-cluster 11 | ``` 12 | 13 | ## Create the Federation Namespace 14 | 15 | The Kubernetes federation control plane will run in the federation namespace. Create the federation namespace using kubectl: 16 | 17 | ``` 18 | kubectl create -f ns/federation.yaml 19 | ``` 20 | 21 | ## Create the Federated API Server Service 22 | 23 | ``` 24 | kubectl create -f services/federation-apiserver.yaml 25 | ``` 26 | 27 | Wait until the `EXTERNAL-IP` is populated as it will be required to configure the federation-controller-manager. 28 | 29 | ``` 30 | kubectl get services 31 | ``` 32 | 33 | ``` 34 | NAME CLUSTER-IP EXTERNAL-IP PORT(S) AGE 35 | federation-apiserver XX.XXX.XXX.XX XX.XXX.XX.XX 443/TCP 1m 36 | ``` 37 | 38 | ## Create the Federation API Server Secret 39 | 40 | In this section you will create a set of credentials to limit access to the federated API server. 41 | 42 | ``` 43 | FEDERATION_TOKEN=$(head -c 16 /dev/urandom | od -An -t x | tr -d ' ') 44 | ``` 45 | 46 | ``` 47 | cat > known-tokens.csv < kubeconfigs/federation-apiserver/kubeconfig 57 | ``` 58 | 59 | #### Create the Federated API Server Secret 60 | 61 | Switch to the host cluster context and create the `federation-apiserver-kubeconfig`, which holds the kubeconfig for the federated API server used by the Federated Controller Manager. 62 | 63 | ``` 64 | kubectl config use-context host-cluster 65 | ``` 66 | 67 | ``` 68 | kubectl create secret generic federation-apiserver-kubeconfig \ 69 | --from-file=kubeconfigs/federation-apiserver/kubeconfig 70 | ``` 71 | 72 | Verify 73 | 74 | ``` 75 | kubectl describe secrets federation-apiserver-kubeconfig 76 | ``` 77 | 78 | ### Deploy the Federated Controller Manager 79 | 80 | ``` 81 | DNS_ZONE_NAME=$(gcloud dns managed-zones describe federation --format='value(dnsName)') 82 | ``` 83 | 84 | ``` 85 | DNS_ZONE_ID=$(gcloud dns managed-zones describe federation --format='value(id)') 86 | ``` 87 | 88 | ``` 89 | kubectl create configmap federation-controller-manager \ 90 | --from-literal=zone-id=${DNS_ZONE_ID} \ 91 | --from-literal=zone-name=${DNS_ZONE_NAME} 92 | ``` 93 | 94 | ``` 95 | kubectl get configmap federation-controller-manager -o yaml 96 | ``` 97 | 98 | ``` 99 | kubectl create -f deployments/federation-controller-manager.yaml 100 | ``` 101 | 102 | Wait for the `federation-controller-manager` pod to be running. 103 | 104 | ``` 105 | kubectl get pods 106 | ``` 107 | 108 | ### Configure kube-dns with federated DNS support 109 | 110 | In order for automatic DNS failover to work kube-dns must be configured to support federation. 111 | 112 | ``` 113 | kubectl config use-context federation-cluster 114 | ``` 115 | 116 | ``` 117 | mkdir -p configmaps 118 | ``` 119 | 120 | ``` 121 | cat > configmaps/kube-dns.yaml < clusters/${cluster}.yaml < Notice there are no `nginx-us` pods running in the `europe-west1-b` or `asia-east1-b` clusters: 123 | 124 | 125 | ## Federated NGINX Service 126 | 127 | Create a federated service object in the `federation-cluster` context. 128 | 129 | ``` 130 | kubectl config use-context federation-cluster 131 | ``` 132 | 133 | ``` 134 | kubectl create -f services/nginx.yaml 135 | ``` 136 | 137 | Wait until the nginx service is propagated across all 4 clusters and the federated service is updated with the details. Currently this can take up to 5 mins to complete. 138 | 139 | ### Verify 140 | 141 | Describe the federated nginx service. 142 | 143 | ``` 144 | kubectl describe services nginx 145 | ``` 146 | 147 | ``` 148 | Name: nginx 149 | Namespace: default 150 | Labels: app=nginx 151 | Selector: app=nginx 152 | Type: LoadBalancer 153 | IP: 154 | LoadBalancer Ingress: 104.155.179.91, 104.199.198.18, 104.196.155.68, 23.251.129.13 155 | Port: http 80/TCP 156 | Endpoints: 157 | Session Affinity: None 158 | No events. 159 | ``` 160 | 161 | ### List Services 162 | 163 | 164 | ``` 165 | for cluster in ${CLUSTERS}; do 166 | echo "" 167 | echo "${cluster}" 168 | kubectl --context=${cluster} describe services nginx 169 | done 170 | ``` 171 | 172 | ``` 173 | Name: nginx 174 | Namespace: default 175 | Labels: app=nginx 176 | Selector: app=nginx 177 | Type: LoadBalancer 178 | IP: 10.87.255.25 179 | LoadBalancer Ingress: XXX.XXX.XXX.XX 180 | Port: http 80/TCP 181 | NodePort: http 30330/TCP 182 | Endpoints: 10.84.1.5:80 183 | Session Affinity: None 184 | ``` 185 | 186 | 187 | ### Review Cloud DNS Console 188 | 189 | The Federated controller manager creates DNS entries in the configured zone. 190 | 191 | ![Google Cloud DNS](images/googledns.png) 192 | 193 | 194 | -------------------------------------------------------------------------------- /labs/08-federated-secrets.md: -------------------------------------------------------------------------------- 1 | # Federated Secrets 2 | 3 | Kubernetes supports the federation of secrets across multiple clusters. Federated secrets are automatically created on new clusters added to the federation. 4 | 5 | ## Prerequisites 6 | 7 | ``` 8 | kubectl config use-context federation-cluster 9 | ``` 10 | 11 | ``` 12 | CLUSTERS="asia-east1-b europe-west1-b us-east1-b us-central1-b" 13 | ``` 14 | 15 | ## Federate Secrets 16 | 17 | The following command will create the `federated` secret across all 4 clusters: 18 | 19 | ``` 20 | kubectl create secret generic federated --from-literal=password=foo 21 | ``` 22 | 23 | ``` 24 | secret "federated" created 25 | ``` 26 | 27 | ### Verify 28 | 29 | List the `federated` secret 30 | 31 | ``` 32 | for cluster in ${CLUSTERS}; do 33 | echo "" 34 | echo "${cluster}" 35 | kubectl --context=${cluster} get secrets 36 | done 37 | ``` 38 | -------------------------------------------------------------------------------- /labs/09-cleaning-up.md: -------------------------------------------------------------------------------- 1 | # Cleaning Up 2 | 3 | ## Kubernetes Resources 4 | 5 | ``` 6 | kubectl --context=federation-cluster delete services nginx 7 | ``` 8 | 9 | ``` 10 | kubectl --context=federation-cluster delete ing nginx 11 | ``` 12 | 13 | ``` 14 | CLUSTERS="asia-east1-b europe-west1-b us-east1-b us-central1-b" 15 | ``` 16 | 17 | ``` 18 | kubectl --context=federation-cluster delete clusters ${CLUSTERS[*]} 19 | ``` 20 | 21 | ``` 22 | kubectl config use-context host-cluster 23 | ``` 24 | 25 | ``` 26 | kubectl delete ns federation 27 | ``` 28 | 29 | ## DNS Managed Zone 30 | 31 | The managed zone must be empty before you can delete it. Visit the Cloud DNS console and delete all resource records before running the following command: 32 | 33 | ``` 34 | gcloud dns managed-zones delete federation 35 | ``` 36 | 37 | ## GKE Clusters 38 | 39 | Delete the four GKE clusters: 40 | 41 | ``` 42 | gcloud container clusters delete asia-east1-b -q --zone=asia-east1-b 43 | ``` 44 | 45 | ``` 46 | gcloud container clusters delete europe-west1-b -q --zone=europe-west1-b 47 | ``` 48 | 49 | ``` 50 | gcloud container clusters delete us-central1-b -q --zone=us-central1-b 51 | ``` 52 | 53 | ``` 54 | gcloud container clusters delete us-east1-b -q --zone=us-east1-b 55 | ``` 56 | -------------------------------------------------------------------------------- /labs/images/googledns.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kelseyhightower/kubernetes-cluster-federation/0606fbe0348530ed45c9f69fd5fd5c7bd1632018/labs/images/googledns.png -------------------------------------------------------------------------------- /ns/federation.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Namespace 3 | metadata: 4 | name: federation 5 | -------------------------------------------------------------------------------- /pvc/federation-apiserver-etcd.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: PersistentVolumeClaim 3 | metadata: 4 | name: federation-apiserver-etcd 5 | annotations: 6 | volume.alpha.kubernetes.io/storage-class: "yes" 7 | namespace: federation 8 | labels: 9 | app: federated-cluster 10 | spec: 11 | accessModes: 12 | - ReadWriteOnce 13 | resources: 14 | requests: 15 | storage: 10Gi 16 | -------------------------------------------------------------------------------- /rs/nginx-us.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: extensions/v1beta1 2 | kind: ReplicaSet 3 | metadata: 4 | name: nginx-us 5 | annotations: 6 | federation.kubernetes.io/replica-set-preferences: | 7 | { 8 | "rebalance": true, 9 | "clusters": { 10 | "us-east1-b": { 11 | "minReplicas": 2, 12 | "maxReplicas": 4, 13 | "weight": 1 14 | }, 15 | "us-central1-b": { 16 | "minReplicas": 2, 17 | "maxReplicas": 4, 18 | "weight": 1 19 | } 20 | } 21 | } 22 | spec: 23 | replicas: 4 24 | template: 25 | metadata: 26 | labels: 27 | region: nginx-us 28 | spec: 29 | containers: 30 | - name: nginx 31 | image: nginx:1.10 32 | resources: 33 | requests: 34 | cpu: 100m 35 | memory: 100Mi 36 | -------------------------------------------------------------------------------- /rs/nginx.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: extensions/v1beta1 2 | kind: ReplicaSet 3 | metadata: 4 | name: nginx 5 | spec: 6 | replicas: 4 7 | template: 8 | metadata: 9 | labels: 10 | app: nginx 11 | spec: 12 | containers: 13 | - name: nginx 14 | image: nginx:1.10 15 | resources: 16 | requests: 17 | cpu: 100m 18 | memory: 100Mi 19 | -------------------------------------------------------------------------------- /services/federation-apiserver.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Service 3 | metadata: 4 | name: federation-apiserver 5 | namespace: federation 6 | labels: 7 | app: federated-cluster 8 | spec: 9 | type: LoadBalancer 10 | selector: 11 | app: federated-cluster 12 | module: federation-apiserver 13 | ports: 14 | - name: https 15 | protocol: TCP 16 | port: 443 17 | targetPort: 443 18 | -------------------------------------------------------------------------------- /services/nginx.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Service 3 | metadata: 4 | labels: 5 | app: nginx 6 | name: nginx 7 | spec: 8 | ports: 9 | - port: 80 10 | protocol: TCP 11 | targetPort: 80 12 | name: http 13 | selector: 14 | app: nginx 15 | type: NodePort 16 | --------------------------------------------------------------------------------