├── img
    ├── godday-ns.png
    └── fed-cluster.png
├── us-east
    ├── k
    ├── delete.sh
    └── create.sh
├── us-west
    ├── k
    ├── delete.sh
    └── create.sh
├── ap-northeast
    ├── k
    ├── delete.sh
    └── create.sh
├── patches
    ├── us-east-pod.yml
    ├── us-west-pod.yml
    └── kops-cluster-patch.yml
├── .gitignore
├── 0-create-hosted-domain.sh
├── 1-create-clusters.sh
├── 2-init-helm.sh
├── 5-join-us-east.sh
├── 6-join-us-west.sh
├── 4-join-ap-northeast.sh
├── 3-install-federation.sh
├── 14-create-nginx2-rsp.sh
├── dns-record.json
├── 7-create-fns.sh
├── 10-create-nginx-fsvc.sh
├── 12-create-nginx-domain-and-record.sh
├── 9-patch-nginx-pod.sh
├── 99-purge.sh
├── 13-create-nginx2-fdeploy.sh
├── 8-create-nginx-fdeploy.sh
├── .env.sample
├── 11-deploy-external-dns.sh
└── README.md


/img/godday-ns.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/kairen/aws-k8s-federation/HEAD/img/godday-ns.png


--------------------------------------------------------------------------------
/img/fed-cluster.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/kairen/aws-k8s-federation/HEAD/img/fed-cluster.png


--------------------------------------------------------------------------------
/us-east/k:
--------------------------------------------------------------------------------
1 | #!/bin/bash
2 | 
3 | source .env
4 | set -eux
5 | 
6 | kubectl --context=${US_EAST_CONTEXT} $@
7 | 


--------------------------------------------------------------------------------
/us-west/k:
--------------------------------------------------------------------------------
1 | #!/bin/bash
2 | 
3 | source .env
4 | set -eux
5 | 
6 | kubectl --context=${US_WEST_CONTEXT} $@
7 | 


--------------------------------------------------------------------------------
/ap-northeast/k:
--------------------------------------------------------------------------------
1 | #!/bin/bash
2 | 
3 | source .env
4 | set -eux
5 | 
6 | kubectl --context=${AP_NORTHEAST_CONTEXT} $@
7 | 


--------------------------------------------------------------------------------
/patches/us-east-pod.yml:
--------------------------------------------------------------------------------
1 | spec:
2 |   containers:
3 |   - name: nginx
4 |     image: kairen/nginx:magic
5 |     
6 |       


--------------------------------------------------------------------------------
/patches/us-west-pod.yml:
--------------------------------------------------------------------------------
1 | spec:
2 |   containers:
3 |   - name: nginx
4 |     image: kairen/nginx:slowpoke
5 |     
6 |       


--------------------------------------------------------------------------------
/.gitignore:
--------------------------------------------------------------------------------
1 | .DS_Store
2 | dns-record.json-d
3 | .env
4 | fed-controller-patch.yml
5 | 
6 | # ignore all files generated by JetBrains tools
7 | .idea
8 | 


--------------------------------------------------------------------------------
/0-create-hosted-domain.sh:
--------------------------------------------------------------------------------
1 | #!/bin/bash
2 | 
3 | source .env
4 | set -eux
5 | 
6 | aws route53 create-hosted-zone \
7 |   --name ${DOMAIN_NAME} \
8 |   --caller-reference $(date '+%Y-%m-%d-%H:%M')
9 | 


--------------------------------------------------------------------------------
/1-create-clusters.sh:
--------------------------------------------------------------------------------
 1 | #!/bin/bash
 2 | 
 3 | source .env
 4 | set -eux
 5 | 
 6 | # Start to create k8s cluster on aws
 7 | ./ap-northeast/create.sh
 8 | ./us-east/create.sh
 9 | ./us-west/create.sh
10 | 


--------------------------------------------------------------------------------
/us-east/delete.sh:
--------------------------------------------------------------------------------
 1 | #!/bin/bash
 2 | 
 3 | source .env
 4 | set -eux
 5 | 
 6 | kops delete cluster \
 7 |  --name=${US_EAST_CONTEXT} \
 8 |  --state=s3://${US_EAST_BUCKET_NAME} --yes
 9 | 
10 | aws s3 rb s3://${US_EAST_BUCKET_NAME} --force
11 | 


--------------------------------------------------------------------------------
/us-west/delete.sh:
--------------------------------------------------------------------------------
 1 | #!/bin/bash
 2 | 
 3 | source .env
 4 | set -eux
 5 | 
 6 | kops delete cluster \
 7 |  --name=${US_WEST_CONTEXT} \
 8 |  --state=s3://${US_WEST_BUCKET_NAME} --yes
 9 | 
10 | aws s3 rb s3://${US_WEST_BUCKET_NAME} --force
11 | 


--------------------------------------------------------------------------------
/ap-northeast/delete.sh:
--------------------------------------------------------------------------------
 1 | #!/bin/bash
 2 | 
 3 | source .env
 4 | set -eux
 5 | 
 6 | kops delete cluster \
 7 |  --name=${AP_NORTHEAST_CONTEXT} \
 8 |  --state=s3://${AP_NORTHEAST_BUCKET_NAME} --yes
 9 | 
10 | aws s3 rb s3://${AP_NORTHEAST_BUCKET_NAME} --force
11 | 


--------------------------------------------------------------------------------
/2-init-helm.sh:
--------------------------------------------------------------------------------
 1 | #!/bin/bash
 2 | 
 3 | source .env
 4 | set -eux
 5 | 
 6 | # Switch to master cluster
 7 | kubectl config use-context ${FED_CONTEXT}
 8 | 
 9 | # Create RBAC for Helm
10 | kubectl -n kube-system create sa tiller
11 | kubectl create clusterrolebinding tiller --clusterrole cluster-admin --serviceaccount=kube-system:tiller
12 | helm init --service-account tiller
13 | 


--------------------------------------------------------------------------------
/5-join-us-east.sh:
--------------------------------------------------------------------------------
 1 | #!/bin/bash
 2 | 
 3 | source .env
 4 | set -eux
 5 | 
 6 | # Switch to the fed cluster context
 7 | kubectl config use-context ${FED_CONTEXT}
 8 | 
 9 | # Join the us-east cluster to the Federation
10 | kubefedctl join us-east \
11 |   --host-cluster-context=${AP_NORTHEAST_CONTEXT} \
12 |   --cluster-context=${US_EAST_CONTEXT} \
13 |   --v=2 
14 | 
15 | # Check cluster by kubectl
16 | kubectl -n kube-federation-system describe kubefedclusters us-east


--------------------------------------------------------------------------------
/6-join-us-west.sh:
--------------------------------------------------------------------------------
 1 | #!/bin/bash
 2 | 
 3 | source .env
 4 | set -eux
 5 | 
 6 | # Switch to the fed cluster context
 7 | kubectl config use-context ${FED_CONTEXT}
 8 | 
 9 | # Join the us-west cluster to the Federation
10 | kubefedctl join us-west \
11 |   --host-cluster-context=${AP_NORTHEAST_CONTEXT} \
12 |   --cluster-context=${US_WEST_CONTEXT} \
13 |   --v=2 
14 | 
15 | # Check cluster by kubectl
16 | kubectl -n kube-federation-system describe kubefedclusters us-west


--------------------------------------------------------------------------------
/4-join-ap-northeast.sh:
--------------------------------------------------------------------------------
 1 | #!/bin/bash
 2 | 
 3 | source .env
 4 | set -eux
 5 | 
 6 | # Switch to the fed cluster context
 7 | kubectl config use-context ${FED_CONTEXT}
 8 | 
 9 | # Join the ap-northeast cluster to the Federation
10 | kubefedctl join ap-northeast \
11 |   --host-cluster-context=${AP_NORTHEAST_CONTEXT} \
12 |   --cluster-context=${AP_NORTHEAST_CONTEXT} \
13 |   --v=2 
14 | 
15 | # Check cluster by kubectl
16 | kubectl -n kube-federation-system describe kubefedclusters ap-northeast


--------------------------------------------------------------------------------
/3-install-federation.sh:
--------------------------------------------------------------------------------
 1 | #!/bin/bash
 2 | 
 3 | source .env
 4 | set -eux
 5 | 
 6 | # Switch to master cluster
 7 | kubectl config use-context ${FED_CONTEXT}
 8 | 
 9 | # Deploy the Federation control plane to the host cluster
10 | git clone https://github.com/kubernetes-sigs/kubefed.git -b v0.1.0-rc2
11 | cd kubefed
12 | 
13 | helm install charts/kubefed \
14 |   --name kubefed \
15 |   --namespace kube-federation-system \
16 |   --set controllermanager.tag=v0.1.0-rc2 
17 | 
18 | # Remove unnecessary files
19 | cd ../
20 | rm -rf kubefed
21 | 
22 | kubectl -n kube-federation-system get po -o wide


--------------------------------------------------------------------------------
/14-create-nginx2-rsp.sh:
--------------------------------------------------------------------------------
 1 | #!/bin/bash
 2 | 
 3 | source .env
 4 | set -eux
 5 | 
 6 | # Switch to the fed cluster context
 7 | kubectl config use-context ${FED_CONTEXT}
 8 | 
 9 | # Create a nginx2 rsp
10 | cat <<EOF | kubectl apply -f -
11 | apiVersion: scheduling.kubefed.k8s.io/v1alpha1
12 | kind: ReplicaSchedulingPreference
13 | metadata:
14 |   name: nginx2
15 |   namespace: test
16 | spec:
17 |   targetKind: FederatedDeployment
18 |   totalReplicas: 15
19 |   clusters:
20 |     "*":
21 |       weight: 2
22 |     ap-northeast:
23 |       minReplicas: 1
24 |       maxReplicas: 3
25 |       weight: 1
26 | EOF
27 | 
28 | 


--------------------------------------------------------------------------------
/dns-record.json:
--------------------------------------------------------------------------------
 1 | {
 2 |   "Comment": "Create/Update a latency-based CNAME record for a federated Deployment",
 3 |   "Changes": [
 4 |     {
 5 |       "Action": "UPSERT",
 6 |       "ResourceRecordSet": {
 7 |         "Name": "nginx.k8s.example.com",
 8 |         "Type": "CNAME",
 9 |         "Region": "ap-northeast-1",
10 |         "TTL": 300,
11 |         "SetIdentifier": "ap-northeast-1",
12 |         "ResourceRecords": [
13 |           {
14 |             "Value": " a197b40c9438c11e8a95a068c009ec4b-1437356461.ap-northeast-1.elb.amazonaws.com "
15 |           }
16 |         ]
17 |       }
18 |     }
19 |   ]
20 | }
21 | 


--------------------------------------------------------------------------------
/7-create-fns.sh:
--------------------------------------------------------------------------------
 1 | #!/bin/bash
 2 | 
 3 | source .env
 4 | set -eux
 5 | 
 6 | # Switch to the fed cluster context
 7 | kubectl config use-context ${FED_CONTEXT}
 8 | 
 9 | # Create a namespace called `test`
10 | cat <<EOF | kubectl apply -f -
11 | apiVersion: v1
12 | kind: Namespace
13 | metadata:
14 |   name: test
15 | EOF
16 | 
17 | # Create a federated namespace
18 | cat <<EOF | kubectl apply -f -
19 | apiVersion: types.kubefed.k8s.io/v1beta1
20 | kind: FederatedNamespace
21 | metadata:
22 |   name: test
23 |   namespace: test
24 | spec:
25 |   placement:
26 |     clusters:
27 |     - name: ap-northeast
28 |     - name: us-east
29 |     - name: us-west
30 | EOF
31 | 


--------------------------------------------------------------------------------
/10-create-nginx-fsvc.sh:
--------------------------------------------------------------------------------
 1 | #!/bin/bash
 2 | 
 3 | source .env
 4 | set -eux
 5 | 
 6 | # Switch to the fed cluster context
 7 | kubectl config use-context ${FED_CONTEXT}
 8 | 
 9 | # Create a service for NGINX deployment
10 | cat <<EOF | kubectl apply -f -
11 | apiVersion: types.kubefed.k8s.io/v1beta1
12 | kind: FederatedService
13 | metadata:
14 |   name: nginx
15 |   namespace: test
16 | spec:
17 |   template:
18 |     spec:
19 |       selector:
20 |         app: nginx
21 |       type: LoadBalancer
22 |       ports:
23 |         - name: http
24 |           port: 80
25 |   placement:
26 |     clusters:
27 |     - name: ap-northeast
28 |     - name: us-east
29 |     - name: us-west
30 | EOF
31 | 


--------------------------------------------------------------------------------
/12-create-nginx-domain-and-record.sh:
--------------------------------------------------------------------------------
 1 | #!/bin/bash
 2 | 
 3 | source .env
 4 | set -eux
 5 | 
 6 | # Switch to the fed cluster context
 7 | kubectl config use-context ${FED_CONTEXT}
 8 | 
 9 | # Create a domain and service dns record
10 | cat <<EOF | kubectl apply -f -
11 | apiVersion: multiclusterdns.kubefed.k8s.io/v1alpha1
12 | kind: Domain
13 | metadata:
14 |   name: test-domain
15 |   namespace: kube-federation-system
16 | domain: ${DOMAIN_NAME}
17 | EOF
18 | 
19 | # Create a service dns record
20 | cat <<EOF | kubectl apply -f -
21 | apiVersion: multiclusterdns.kubefed.k8s.io/v1alpha1
22 | kind: ServiceDNSRecord
23 | metadata:
24 |   name: nginx
25 |   namespace: test
26 | spec:
27 |   domainRef: test-domain
28 |   recordTTL: 300
29 | EOF
30 | 


--------------------------------------------------------------------------------
/9-patch-nginx-pod.sh:
--------------------------------------------------------------------------------
 1 | #!/bin/bash
 2 | 
 3 | source .env
 4 | set -eux
 5 | 
 6 | # Switch to the us-east cluster context
 7 | kubectl config use-context ${US_EAST_CONTEXT}
 8 | 
 9 | # Patch the us-east  pods
10 | PODS=$(kubectl -n test get po --no-headers "-o=custom-columns=NAME:.metadata.name")
11 | 
12 | for POD in ${PODS}; do
13 |   kubectl -n test patch pod ${POD} --patch "$(cat patches/us-east-pod.yml)"
14 | done
15 | 
16 | # Switch to the us-west cluster context
17 | kubectl config use-context ${US_WEST_CONTEXT}
18 | 
19 | # Patch the us-west pods
20 | PODS=$(kubectl -n test get po --no-headers "-o=custom-columns=NAME:.metadata.name")
21 | 
22 | for POD in ${PODS}; do
23 |   kubectl -n test patch pod ${POD} --patch "$(cat patches/us-west-pod.yml)"
24 | done


--------------------------------------------------------------------------------
/99-purge.sh:
--------------------------------------------------------------------------------
 1 | #!/bin/bash
 2 | 
 3 | source .env
 4 | set -eux
 5 | 
 6 | # Delete clusters
 7 | ./ap-northeast/delete.sh
 8 | ./us-east/delete.sh
 9 | ./us-west/delete.sh
10 | 
11 | ID=$(echo ${HOSTED_ZONE_ID} | sed 's/\/hostedzone\///')
12 | 
13 | # Delete hosted domain name
14 | aws route53 list-resource-record-sets \
15 |   --hosted-zone-id ${ID} |
16 | jq -c '.ResourceRecordSets[]' |
17 | while read -r resourcerecordset; do
18 |   read -r name type <<<$(echo $(jq -r '.Name,.Type' <<<"$resourcerecordset"))
19 |   if [ $type != "NS" -a $type != "SOA" ]; then
20 |     aws route53 change-resource-record-sets \
21 |       --hosted-zone-id ${ID} \
22 |       --change-batch '{"Changes":[{"Action":"DELETE","ResourceRecordSet":
23 |           '"$resourcerecordset"'
24 |         }]}' \
25 |       --output text --query 'ChangeInfo.Id'
26 |   fi
27 | done
28 | 
29 | aws route53 delete-hosted-zone --id ${ID}
30 | 


--------------------------------------------------------------------------------
/13-create-nginx2-fdeploy.sh:
--------------------------------------------------------------------------------
 1 | #!/bin/bash
 2 | 
 3 | source .env
 4 | set -eux
 5 | 
 6 | # Switch to the fed cluster context
 7 | kubectl config use-context ${FED_CONTEXT}
 8 | 
 9 | # Create a nginx2 deployment
10 | cat <<EOF | kubectl apply -f -
11 | apiVersion: types.kubefed.k8s.io/v1beta1
12 | kind: FederatedDeployment
13 | metadata:
14 |   name: nginx2
15 |   namespace: test
16 | spec:
17 |   template:
18 |     metadata:
19 |       labels:
20 |         app: nginx2
21 |     spec:
22 |       replicas: 1
23 |       selector:
24 |         matchLabels:
25 |           app: nginx2
26 |       template:
27 |         metadata:
28 |           labels:
29 |             app: nginx2
30 |         spec:
31 |           containers:
32 |           - image: nginx
33 |             name: nginx
34 |   placement:
35 |     clusters:
36 |     - name: ap-northeast
37 |     - name: us-east
38 |     - name: us-west
39 | EOF
40 | 
41 | 


--------------------------------------------------------------------------------
/patches/kops-cluster-patch.yml:
--------------------------------------------------------------------------------
 1 | spec:
 2 |   additionalPolicies:
 3 |     node: |
 4 |       [
 5 |         {
 6 |           "Effect": "Allow",
 7 |           "Action": [
 8 |             "route53:ChangeResourceRecordSets",
 9 |             "route53:ListResourceRecordSets",
10 |             "route53:GetHostedZone"
11 |           ],
12 |           "Resource": [
13 |             "arn:aws:route53:::hostedzone/*"
14 |           ]
15 |         },
16 |         {
17 |           "Effect": "Allow",
18 |           "Action": [
19 |             "route53:GetChange"
20 |           ],
21 |           "Resource": [
22 |             "arn:aws:route53:::change/*"
23 |           ]
24 |         },
25 |         {
26 |           "Effect": "Allow",
27 |           "Action": [
28 |             "route53:ListHostedZones"
29 |           ],
30 |           "Resource": [
31 |             "*"
32 |           ]
33 |         },
34 |         {
35 |             "Effect": "Allow",
36 |             "Action": "*",
37 |             "Resource": "*"
38 |         }
39 |       ]


--------------------------------------------------------------------------------
/8-create-nginx-fdeploy.sh:
--------------------------------------------------------------------------------
 1 | #!/bin/bash
 2 | 
 3 | source .env
 4 | set -eux
 5 | 
 6 | # Switch to the fed cluster context
 7 | kubectl config use-context ${FED_CONTEXT}
 8 | 
 9 | # Create a nginx deployment
10 | cat <<EOF | kubectl apply -f -
11 | apiVersion: types.kubefed.k8s.io/v1beta1
12 | kind: FederatedDeployment
13 | metadata:
14 |   name: nginx
15 |   namespace: test
16 | spec:
17 |   template:
18 |     metadata:
19 |       labels:
20 |         app: nginx
21 |     spec:
22 |       replicas: 1
23 |       selector:
24 |         matchLabels:
25 |           app: nginx
26 |       template:
27 |         metadata:
28 |           labels:
29 |             app: nginx
30 |         spec:
31 |           containers:
32 |           - image: nginx
33 |             name: nginx
34 |   placement:
35 |     clusters:
36 |     - name: ap-northeast
37 |     - name: us-east
38 |     - name: us-west
39 |   overrides:
40 |   - clusterName: us-east
41 |     clusterOverrides:
42 |     - path: spec.replicas
43 |       value: 2
44 |     - path: spec.template.spec.containers[0].image
45 |       value: kairen/nginx:magic
46 |   - clusterName: us-west
47 |     clusterOverrides:
48 |     - path: spec.replicas
49 |       value: 3
50 | EOF
51 | 
52 | 


--------------------------------------------------------------------------------
/.env.sample:
--------------------------------------------------------------------------------
 1 | # Kubernetes version
 2 | export KUBERNETES_VERSION="1.14.1"
 3 | 
 4 | # Your domain name and envs
 5 | export DOMAIN_NAME="k8s.example.com"
 6 | 
 7 | # Regions and zones
 8 | export US_WEST_REGION="us-west-2"
 9 | export US_EAST_REGION="us-east-2"
10 | export AP_NORTHEAST_REGION="ap-northeast-1"
11 | export ZONE="a"
12 | export US_WEST_ZONE="${US_WEST_REGION}${ZONE}"
13 | export US_EAST_ZONE="${US_WEST_REGION}${ZONE}"
14 | export AP_NORTHEAST_ZONE="${US_WEST_REGION}${ZONE}"
15 | 
16 | # Cluster contexts
17 | export US_WEST_CONTEXT="us-west.${DOMAIN_NAME}"
18 | export US_EAST_CONTEXT="us-east.${DOMAIN_NAME}"
19 | export AP_NORTHEAST_CONTEXT="ap-northeast.${DOMAIN_NAME}"
20 | export FED_CONTEXT="${AP_NORTHEAST_CONTEXT}"
21 | 
22 | # S3 buckets
23 | export US_WEST_BUCKET_NAME="us-west-k8s"
24 | export US_EAST_BUCKET_NAME="us-east-k8s"
25 | export AP_NORTHEAST_BUCKET_NAME="ap-northeast-k8s"
26 | 
27 | # Get domain name id
28 | export HOSTED_ZONE_ID=$(aws route53 list-hosted-zones \
29 |     | jq -r '.HostedZones[] | select(.Name=="'${DOMAIN_NAME}'.") | .Id')
30 | 
31 | # Federaion master and node size, and node count
32 | export MASTER_FLAVOR="t2.micro"
33 | export NODE_FLAVOR="t2.micro"
34 | export MASTER_COUNT=1
35 | export NODE_COUNT=2
36 | 
37 | # Federation deploy and service name
38 | export DNS_RECORD_PREFIX="nginx"
39 | export SERVICE_NAME="nginx"
40 | 


--------------------------------------------------------------------------------
/11-deploy-external-dns.sh:
--------------------------------------------------------------------------------
 1 | #!/bin/bash
 2 | 
 3 | source .env
 4 | set -eux
 5 | 
 6 | # Switch to the fed cluster context
 7 | kubectl config use-context ${FED_CONTEXT}
 8 | 
 9 | # Create externalDNS for Route53
10 | cat <<EOF | kubectl apply -f -
11 | apiVersion: v1
12 | kind: ServiceAccount
13 | metadata:
14 |   name: external-dns
15 |   namespace: kube-federation-system
16 | ---
17 | apiVersion: rbac.authorization.k8s.io/v1beta1
18 | kind: ClusterRole
19 | metadata:
20 |   name: external-dns
21 | rules:
22 | - apiGroups: [""]
23 |   resources: ["services"]
24 |   verbs: ["get","watch","list"]
25 | - apiGroups: [""]
26 |   resources: ["pods"]
27 |   verbs: ["get","watch","list"]
28 | - apiGroups: ["extensions"] 
29 |   resources: ["ingresses"] 
30 |   verbs: ["get","watch","list"]
31 | - apiGroups: [""]
32 |   resources: ["nodes"]
33 |   verbs: ["list", "get", "watch"]
34 | - apiGroups: ["multiclusterdns.kubefed.k8s.io"]
35 |   resources: ["*"]
36 |   verbs: ["*"]
37 | ---
38 | apiVersion: rbac.authorization.k8s.io/v1beta1
39 | kind: ClusterRoleBinding
40 | metadata:
41 |   name: external-dns-viewer
42 | roleRef:
43 |   apiGroup: rbac.authorization.k8s.io
44 |   kind: ClusterRole
45 |   name: external-dns
46 | subjects:
47 | - kind: ServiceAccount
48 |   name: external-dns
49 |   namespace: kube-federation-system
50 | ---
51 | apiVersion: apps/v1
52 | kind: Deployment
53 | metadata:
54 |   name: external-dns
55 |   namespace: kube-federation-system
56 |   labels:
57 |     k8s-app: external-dns
58 | spec:
59 |   replicas: 1
60 |   selector:
61 |     matchLabels:
62 |       k8s-app: external-dns
63 |   template:
64 |     metadata:
65 |       labels:
66 |         k8s-app: external-dns
67 |     spec:
68 |       serviceAccountName: external-dns
69 |       tolerations:
70 |       - key: CriticalAddonsOnly
71 |         operator: Exists
72 |       - effect: NoSchedule
73 |         key: node-role.kubernetes.io/master
74 |       containers:
75 |       - name: external-dns
76 |         image: registry.opensource.zalan.do/teapot/external-dns:v0.5.9
77 |         args:
78 |         - --source=service
79 |         - --source=ingress
80 |         - --domain-filter=${DOMAIN_NAME}
81 |         - --provider=aws
82 |         - --policy=upsert-only
83 |         - --aws-zone-type=public
84 |         - --registry=txt
85 |         - --txt-owner-id=${HOSTED_ZONE_ID}
86 |         - --log-level=debug
87 |         - --txt-prefix=cname
88 |         - --source=crd
89 |         - --crd-source-apiversion=multiclusterdns.kubefed.k8s.io/v1alpha1
90 |         - --crd-source-kind=DNSEndpoint
91 | EOF


--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
 1 | # Setup Kubernetes Federation `V2` on AWS
 2 | In this lab, we will deploy the Kubernetes cluster in three different AWS regions, and then setup Federation between clusters. When federation has been created we will create a federated deployment and service for NGINX. And finally, we will create latency-based DNS records in Route 53, one for each cluster region.
 3 | 
 4 | ![](/img/fed-cluster.png)
 5 | 
 6 | ## Prerequisites
 7 | * We need to deploy Kubernetes cluster in different AWS regions. e.g.:
 8 |   * US West: **Oregon(us-west-2)**
 9 |   * US East: **Ohio(us-east-2)**
10 |   * Asia: **Tokyo(ap-northeast-1)**
11 | * Install the following tools on the host:
12 |   * [kubectl](https://kubernetes.io/docs/tasks/tools/install-kubectl/): Using kubectl, you can inspect cluster resources; create, delete, and update components.
13 |   * [helm](https://helm.sh/docs/using_helm/#installing-helm): Using helm to install federation-v2 components by the [chart](https://github.com/kubernetes-sigs/kubefed/tree/master/charts/kubefed).
14 |   * [kubefedctl](https://github.com/kubernetes-sigs/kubefed): Helps you to join/unjon clusters from an existing federation control plane.
15 |   * [kops](https://github.com/kubernetes/kops): Production Grade K8s Installation, Upgrades, and Management.
16 |   
17 |   > Federation v2 can be deployed to and manage clusters running Kubernetes `v1.11` or greater.
18 | 
19 |   * [aws](https://aws.amazon.com/cli/?nc1=h_ls): The AWS Command Line Interface (CLI) is a unified tool to manage your AWS services.
20 | * We will be using Amazon AWS as the IaaS provider:
21 |   * IAM: Provide identity and access management.
22 |   * EC2: The Kubernetes cluster instances.
23 |   * ELB: Kubernetes service load balancer.
24 |   * Route53: Public domain for Kubernetes API, Service, ..., etc.
25 |   * S3: Store Kops state.
26 |   * VPC: Provide cluster network.
27 | * Godaddy domain name or register from Route53
28 | 
29 | ## Quick Start
30 | For the execution of the labs, you need set your env in `.env` file：
31 | ```sh
32 | $ cp .env.sample .env
33 | $ vim .env
34 | ```
35 | 
36 | First create a hostedzone using `0-create-hosted-domain.sh`:
37 | ```sh
38 | $ ./0-create-hosted-domain.sh
39 | # output like this
40 | {
41 |     "HostedZone": {
42 |         "ResourceRecordSetCount": 2,
43 |         "CallerReference": "2018-04-19-11:24",
44 |         "Config": {
45 |             "PrivateZone": false
46 |         },
47 |         "Id": "/hostedzone/Z363YQ27EUQU4S",
48 |         "Name": "k8s.xxxx.com."
49 |     },
50 |     "DelegationSet": {
51 |         "NameServers": [
52 |             "ns-431.awsdns-49.org",
53 |             "ns-1341.awsdns-00.com",
54 |             "ns-134.awsdns-42.co.uk",
55 |             "ns-1131.awsdns-62.net"
56 |         ]
57 |     },
58 |     "Location": "https://route53.amazonaws.com/2013-04-01/hostedzone/Z363YQ27EUQU4S",
59 |     "ChangeInfo": {
60 |         "Status": "PENDING",
61 |         "SubmittedAt": "2018-04-19T03:24:17.638Z",
62 |         "Id": "/change/CTCT89X4F01LM"
63 |     }
64 | }
65 | 
66 | $ aws route53 list-hosted-zones
67 | ```
68 | 
69 | Add `NameServers` into Godaddy, like this:
70 | 
71 | ![](/img/godday-ns.png)
72 | 
73 | Now follow the scripts to setup your federation cluster.
74 | 


--------------------------------------------------------------------------------
/us-west/create.sh:
--------------------------------------------------------------------------------
  1 | #!/bin/bash
  2 | 
  3 | source .env
  4 | set -eux
  5 | 
  6 | aws s3 mb s3://${US_WEST_BUCKET_NAME} --region ${US_WEST_REGION}
  7 | 
  8 | cat <<EOF | kops create --state=s3://${US_WEST_BUCKET_NAME} -f -
  9 | apiVersion: kops/v1alpha2
 10 | kind: Cluster
 11 | metadata:
 12 |   name: ${US_WEST_CONTEXT}
 13 | spec:
 14 |   additionalPolicies:
 15 |     master: |
 16 |       [
 17 |         {
 18 |             "Action": "elasticloadbalancing:*",
 19 |             "Resource": "*",
 20 |             "Effect": "Allow"
 21 |         }
 22 |       ]
 23 |   api:
 24 |     dns: {}
 25 |   authorization:
 26 |     rbac: {}
 27 |   channel: stable
 28 |   cloudProvider: aws
 29 |   configBase: s3://${US_WEST_BUCKET_NAME}/${US_WEST_CONTEXT}
 30 |   dnsZone: ${DOMAIN_NAME}
 31 |   etcdClusters:
 32 |   - cpuRequest: 200m
 33 |     etcdMembers:
 34 |     - instanceGroup: master-${US_WEST_ZONE}
 35 |       name: a
 36 |     memoryRequest: 100Mi
 37 |     name: main
 38 |   - cpuRequest: 100m
 39 |     etcdMembers:
 40 |     - instanceGroup: master-${US_WEST_ZONE}
 41 |       name: a
 42 |     memoryRequest: 100Mi
 43 |     name: events
 44 |   iam:
 45 |     allowContainerRegistry: true
 46 |     legacy: false
 47 |   kubelet:
 48 |     anonymousAuth: false
 49 |   kubernetesApiAccess:
 50 |   - 0.0.0.0/0
 51 |   kubernetesVersion: ${KUBERNETES_VERSION}
 52 |   masterPublicName: api.${US_WEST_CONTEXT}
 53 |   networkCIDR: 172.20.0.0/16
 54 |   networking:
 55 |     kubenet: {}
 56 |   nonMasqueradeCIDR: 100.64.0.0/10
 57 |   sshAccess:
 58 |   - 0.0.0.0/0
 59 |   subnets:
 60 |   - cidr: 172.20.32.0/19
 61 |     name: ${US_WEST_ZONE}
 62 |     type: Public
 63 |     zone: ${US_WEST_ZONE}
 64 |   topology:
 65 |     dns:
 66 |       type: Public
 67 |     masters: public
 68 |     nodes: public
 69 | ---
 70 | apiVersion: kops/v1alpha2
 71 | kind: InstanceGroup
 72 | metadata:
 73 |   labels:
 74 |     kops.k8s.io/cluster: ${US_WEST_CONTEXT}
 75 |   name: master-${US_WEST_ZONE}
 76 | spec:
 77 |   image: kope.io/k8s-1.11-debian-stretch-amd64-hvm-ebs-2018-08-17
 78 |   machineType: ${MASTER_FLAVOR}
 79 |   maxSize: ${MASTER_COUNT}
 80 |   minSize: ${MASTER_COUNT}
 81 |   nodeLabels:
 82 |     kops.k8s.io/instancegroup: master-${US_WEST_ZONE}
 83 |   role: Master
 84 |   subnets:
 85 |   - ${US_WEST_ZONE}
 86 | ---
 87 | apiVersion: kops/v1alpha2
 88 | kind: InstanceGroup
 89 | metadata:
 90 |   labels:
 91 |     kops.k8s.io/cluster: ${US_WEST_CONTEXT}
 92 |   name: nodes
 93 | spec:
 94 |   image: kope.io/k8s-1.11-debian-stretch-amd64-hvm-ebs-2018-08-17
 95 |   machineType: ${NODE_FLAVOR}
 96 |   maxSize: ${NODE_COUNT}
 97 |   minSize: ${NODE_COUNT}
 98 |   nodeLabels:
 99 |     kops.k8s.io/instancegroup: nodes
100 |   role: Node
101 |   subnets:
102 |   - ${US_WEST_ZONE}
103 | EOF
104 | 
105 | kops create secret --name ${US_WEST_CONTEXT} \
106 |   --state=s3://${US_WEST_BUCKET_NAME} \
107 |   sshpublickey admin -i ~/.ssh/id_rsa.pub
108 | 
109 | # kops create cluster \
110 | #   --name=${US_WEST_CONTEXT} \
111 | #   --state=s3://${US_WEST_BUCKET_NAME} \
112 | #   --zones="${US_WEST_ZONE}" \
113 | #   --master-size=${MASTER_FLAVOR} \
114 | #   --node-size=${NODE_FLAVOR} \
115 | #   --node-count=${NODE_COUNT} \
116 | #   --dns-zone=${DOMAIN_NAME}
117 | 
118 | kops update cluster ${US_WEST_CONTEXT} \
119 |   --state=s3://${US_WEST_BUCKET_NAME} \
120 |   --yes
121 | 


--------------------------------------------------------------------------------
/us-east/create.sh:
--------------------------------------------------------------------------------
  1 | #!/bin/bash
  2 | 
  3 | source .env
  4 | set -eux
  5 | 
  6 | aws s3 mb s3://${US_EAST_BUCKET_NAME} --region ${US_EAST_REGION}
  7 | 
  8 | cat <<EOF | kops create --state=s3://${US_EAST_BUCKET_NAME} -f -
  9 | apiVersion: kops/v1alpha2
 10 | kind: Cluster
 11 | metadata:
 12 |   name: ${US_EAST_CONTEXT}
 13 | spec:
 14 |   additionalPolicies:
 15 |     master: |
 16 |       [
 17 |         {
 18 |             "Action": "elasticloadbalancing:*",
 19 |             "Resource": "*",
 20 |             "Effect": "Allow"
 21 |         }
 22 |       ]
 23 |   api:
 24 |     dns: {}
 25 |   authorization:
 26 |     rbac: {}
 27 |   channel: stable
 28 |   cloudProvider: aws
 29 |   configBase: s3://${US_EAST_BUCKET_NAME}/${US_EAST_CONTEXT}
 30 |   dnsZone: ${DOMAIN_NAME}
 31 |   etcdClusters:
 32 |   - cpuRequest: 200m
 33 |     etcdMembers:
 34 |     - instanceGroup: master-${US_EAST_ZONE}
 35 |       name: a
 36 |     memoryRequest: 100Mi
 37 |     name: main
 38 |   - cpuRequest: 100m
 39 |     etcdMembers:
 40 |     - instanceGroup: master-${US_EAST_ZONE}
 41 |       name: a
 42 |     memoryRequest: 100Mi
 43 |     name: events
 44 |   iam:
 45 |     allowContainerRegistry: true
 46 |     legacy: false
 47 |   kubelet:
 48 |     anonymousAuth: false
 49 |   kubernetesApiAccess:
 50 |   - 0.0.0.0/0
 51 |   kubernetesVersion: ${KUBERNETES_VERSION}
 52 |   masterPublicName: api.${US_EAST_CONTEXT}
 53 |   networkCIDR: 172.20.0.0/16
 54 |   networking:
 55 |     kubenet: {}
 56 |   nonMasqueradeCIDR: 100.64.0.0/10
 57 |   sshAccess:
 58 |   - 0.0.0.0/0
 59 |   subnets:
 60 |   - cidr: 172.20.32.0/19
 61 |     name: ${US_EAST_ZONE}
 62 |     type: Public
 63 |     zone: ${US_EAST_ZONE}
 64 |   topology:
 65 |     dns:
 66 |       type: Public
 67 |     masters: public
 68 |     nodes: public
 69 | ---
 70 | apiVersion: kops/v1alpha2
 71 | kind: InstanceGroup
 72 | metadata:
 73 |   labels:
 74 |     kops.k8s.io/cluster: ${US_EAST_CONTEXT}
 75 |   name: master-${US_EAST_ZONE}
 76 | spec:
 77 |   image: kope.io/k8s-1.11-debian-stretch-amd64-hvm-ebs-2018-08-17
 78 |   machineType: ${MASTER_FLAVOR}
 79 |   maxSize: ${MASTER_COUNT}
 80 |   minSize: ${MASTER_COUNT}
 81 |   nodeLabels:
 82 |     kops.k8s.io/instancegroup: master-${US_EAST_ZONE}
 83 |   role: Master
 84 |   subnets:
 85 |   - ${US_EAST_ZONE}
 86 | ---
 87 | apiVersion: kops/v1alpha2
 88 | kind: InstanceGroup
 89 | metadata:
 90 |   labels:
 91 |     kops.k8s.io/cluster: ${US_EAST_CONTEXT}
 92 |   name: nodes
 93 | spec:
 94 |   image: kope.io/k8s-1.11-debian-stretch-amd64-hvm-ebs-2018-08-17
 95 |   machineType: ${NODE_FLAVOR}
 96 |   maxSize: ${NODE_COUNT}
 97 |   minSize: ${NODE_COUNT}
 98 |   nodeLabels:
 99 |     kops.k8s.io/instancegroup: nodes
100 |   role: Node
101 |   subnets:
102 |   - ${US_EAST_ZONE}
103 | EOF
104 | 
105 | kops create secret --name ${US_EAST_CONTEXT} \
106 |   --state=s3://${US_EAST_BUCKET_NAME} \
107 |   sshpublickey admin -i ~/.ssh/id_rsa.pub
108 | 
109 | # kops create cluster \
110 | #   --name=${US_EAST_CONTEXT} \
111 | #   --state=s3://${US_EAST_BUCKET_NAME} \
112 | #   --zones="${US_EAST_ZONE}" \
113 | #   --master-size=${MASTER_FLAVOR}} \
114 | #   --node-size=${NODE_FLAVOR} \
115 | #   --node-count=${NODE_COUNT} \
116 | #   --dns-zone=${DOMAIN_NAME}
117 | 
118 | kops update cluster ${US_EAST_CONTEXT} \
119 |   --state=s3://${US_EAST_BUCKET_NAME} \
120 |   --yes
121 | 


--------------------------------------------------------------------------------
/ap-northeast/create.sh:
--------------------------------------------------------------------------------
  1 | #!/bin/bash
  2 | 
  3 | source .env
  4 | set -eux
  5 | 
  6 | aws s3 mb s3://${AP_NORTHEAST_BUCKET_NAME} --region ${AP_NORTHEAST_REGION}
  7 | 
  8 | cat <<EOF | kops create --state=s3://${AP_NORTHEAST_BUCKET_NAME} -f -
  9 | apiVersion: kops/v1alpha2
 10 | kind: Cluster
 11 | metadata:
 12 |   name: ${AP_NORTHEAST_CONTEXT}
 13 | spec:
 14 |   additionalPolicies:
 15 |     master: |
 16 |       [
 17 |         {
 18 |             "Action": "elasticloadbalancing:*",
 19 |             "Resource": "*",
 20 |             "Effect": "Allow"
 21 |         }
 22 |       ]
 23 |     node: |
 24 |       [
 25 |         {
 26 |           "Effect": "Allow",
 27 |           "Action": [
 28 |             "route53:ChangeResourceRecordSets",
 29 |             "route53:ListResourceRecordSets",
 30 |             "route53:GetHostedZone"
 31 |           ],
 32 |           "Resource": [
 33 |             "arn:aws:route53:::hostedzone/*"
 34 |           ]
 35 |         },
 36 |         {
 37 |           "Effect": "Allow",
 38 |           "Action": [
 39 |             "route53:GetChange"
 40 |           ],
 41 |           "Resource": [
 42 |             "arn:aws:route53:::change/*"
 43 |           ]
 44 |         },
 45 |         {
 46 |           "Effect": "Allow",
 47 |           "Action": [
 48 |             "route53:ListHostedZones"
 49 |           ],
 50 |           "Resource": [
 51 |             "*"
 52 |           ]
 53 |         },
 54 |         {
 55 |             "Effect": "Allow",
 56 |             "Action": "*",
 57 |             "Resource": "*"
 58 |         }
 59 |       ]
 60 |   api:
 61 |     dns: {}
 62 |   authorization:
 63 |     rbac: {}
 64 |   channel: stable
 65 |   cloudProvider: aws
 66 |   configBase: s3://${AP_NORTHEAST_BUCKET_NAME}/${AP_NORTHEAST_CONTEXT}
 67 |   dnsZone: ${DOMAIN_NAME}
 68 |   etcdClusters:
 69 |   - cpuRequest: 200m
 70 |     etcdMembers:
 71 |     - instanceGroup: master-${AP_NORTHEAST_ZONE}
 72 |       name: a
 73 |     memoryRequest: 100Mi
 74 |     name: main
 75 |   - cpuRequest: 100m
 76 |     etcdMembers:
 77 |     - instanceGroup: master-${AP_NORTHEAST_ZONE}
 78 |       name: a
 79 |     memoryRequest: 100Mi
 80 |     name: events
 81 |   iam:
 82 |     allowContainerRegistry: true
 83 |     legacy: false
 84 |   kubelet:
 85 |     anonymousAuth: false
 86 |   kubernetesApiAccess:
 87 |   - 0.0.0.0/0
 88 |   kubernetesVersion: ${KUBERNETES_VERSION}
 89 |   masterPublicName: api.${AP_NORTHEAST_CONTEXT}
 90 |   networkCIDR: 172.20.0.0/16
 91 |   networking:
 92 |     kubenet: {}
 93 |   nonMasqueradeCIDR: 100.64.0.0/10
 94 |   sshAccess:
 95 |   - 0.0.0.0/0
 96 |   subnets:
 97 |   - cidr: 172.20.32.0/19
 98 |     name: ${AP_NORTHEAST_ZONE}
 99 |     type: Public
100 |     zone: ${AP_NORTHEAST_ZONE}
101 |   topology:
102 |     dns:
103 |       type: Public
104 |     masters: public
105 |     nodes: public
106 | ---
107 | apiVersion: kops/v1alpha2
108 | kind: InstanceGroup
109 | metadata:
110 |   labels:
111 |     kops.k8s.io/cluster: ${AP_NORTHEAST_CONTEXT}
112 |   name: master-${AP_NORTHEAST_ZONE}
113 | spec:
114 |   image: kope.io/k8s-1.11-debian-stretch-amd64-hvm-ebs-2018-08-17
115 |   machineType: ${MASTER_FLAVOR}
116 |   maxSize: ${MASTER_COUNT}
117 |   minSize: ${MASTER_COUNT}
118 |   nodeLabels:
119 |     kops.k8s.io/instancegroup: master-${AP_NORTHEAST_ZONE}
120 |   role: Master
121 |   subnets:
122 |   - ${AP_NORTHEAST_ZONE}
123 | ---
124 | apiVersion: kops/v1alpha2
125 | kind: InstanceGroup
126 | metadata:
127 |   labels:
128 |     kops.k8s.io/cluster: ${AP_NORTHEAST_CONTEXT}
129 |   name: nodes
130 | spec:
131 |   image: kope.io/k8s-1.11-debian-stretch-amd64-hvm-ebs-2018-08-17
132 |   machineType: ${NODE_FLAVOR}
133 |   maxSize: ${NODE_COUNT}
134 |   minSize: ${NODE_COUNT}
135 |   nodeLabels:
136 |     kops.k8s.io/instancegroup: nodes
137 |   role: Node
138 |   subnets:
139 |   - ${AP_NORTHEAST_ZONE}
140 | EOF
141 | 
142 | kops create secret --name ${AP_NORTHEAST_CONTEXT} \
143 |   --state=s3://${AP_NORTHEAST_BUCKET_NAME} \
144 |   sshpublickey admin -i ~/.ssh/id_rsa.pub
145 | 
146 | # kops create cluster \
147 | #   --name=${AP_NORTHEAST_CONTEXT} \
148 | #   --state=s3://${AP_NORTHEAST_BUCKET_NAME} \
149 | #   --zones="${AP_NORTHEAST_ZONE}" \
150 | #   --master-size=${MASTER_FLAVOR} \
151 | #   --node-size=${NODE_FLAVOR} \
152 | #   --node-count=${NODE_COUNT} \
153 | #   --dns-zone=${DOMAIN_NAME}
154 | 
155 | kops update cluster ${AP_NORTHEAST_CONTEXT} \
156 |   --state=s3://${AP_NORTHEAST_BUCKET_NAME} \
157 |   --yes
158 | 


--------------------------------------------------------------------------------