├── README.md ├── common └── util.sh ├── demo.sh ├── federated-ingress ├── federatedingress.yaml ├── federatedreplicaset.yaml ├── federatedservice.yaml └── ingressdns.yaml ├── federated-service ├── federatedreplicaset.yaml ├── federatedservice.yaml ├── production-domain.yaml └── servicedns.yaml └── globaldns ├── cleanup.sh ├── config ├── coredns-chart-values.yaml ├── federatedconfigmap-coredns.yaml ├── federatedconfigmap-kubedns.yaml ├── globaldns-coredns-external-dns-chart-values.yaml └── globaldns-google-external-dns-chart-values.yaml └── start-dns-servers.sh /README.md: -------------------------------------------------------------------------------- 1 | # DNS based cross-cluster service discovery demo 2 | 3 | Follow the step below to bring federation and member clusters on minikube 4 | 5 | ## Prerequisite 6 | - Ensure to bring-up atleast 2 kubernetes clusters say "cluster1" & "cluster2" 7 | (Provisioning LoadBalancer service should be possible in these clusters). 8 | 9 | Note: the cluster which hosts federation should be >=v1.11.0 10 | 11 | ## Sequence 12 | - 1. Attach Region & Zone labels to cluster nodes (for e.g. in minikube env, do as below) 13 | ``` 14 | $ kubectl --context cluster1 label node minikube \ 15 | failure-domain.beta.kubernetes.io/region="us" \ 16 | failure-domain.beta.kubernetes.io/zone="us1" 17 | $ kubectl --context cluster2 label node minikube \ 18 | failure-domain.beta.kubernetes.io/region="eu" \ 19 | failure-domain.beta.kubernetes.io/zone="eu1" 20 | ``` 21 | 22 | - 2. Bringup federation control plane in cluster "cluster1" and join "cluster1" & "cluster2" clusters to federation 23 | ``` 24 | $ cd ${GOPATH}/src/github.com/kubernetes-sigs/federation-v2 25 | $ kubectl config use-context cluster1 26 | $ ./scripts/deploy-federation-latest.sh cluster2 27 | $ cd - 28 | ``` 29 | 30 | - 3. Start Global-DNS programmer (based on CoreDNS provider & External-DNS) 31 | ``` 32 | # DEMO_AUTO_RUN=true ./globaldns/start-dns-servers.sh cluster1 33 | ``` 34 | 35 | - 4. Make aware the in-cluster dns about federations & the global-dns server. 36 | ``` 37 | # apply the suitable federatedconfigmap in globaldns/config/ 38 | # don't forget to configure the global-dns server as upstream nameserver in case of CoreDNS 39 | ``` 40 | 41 | - 5. Finally the demo 42 | ``` 43 | # ./demo.sh cluster1 cluster1 cluster2 44 | ``` 45 | -------------------------------------------------------------------------------- /common/util.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # Copyright 2018 The Kubernetes Authors. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | readonly reset=$(tput sgr0) 17 | readonly green=$(tput bold; tput setaf 2) 18 | readonly yellow=$(tput bold; tput setaf 3) 19 | readonly blue=$(tput bold; tput setaf 6) 20 | readonly timeout=$(if [ "$(uname)" == "Darwin" ]; then echo "1"; else echo "0.1"; fi) 21 | 22 | function desc() { 23 | maybe_first_prompt 24 | echo "$blue# $@$reset" 25 | prompt 26 | } 27 | 28 | function prompt() { 29 | echo -n "$yellow\$ $reset" 30 | } 31 | 32 | started="" 33 | function maybe_first_prompt() { 34 | if [ -z "$started" ]; then 35 | prompt 36 | started=true 37 | fi 38 | } 39 | 40 | # After a `run` this variable will hold the stdout of the command that was run. 41 | # If the command was interactive, this will likely be garbage. 42 | DEMO_RUN_STDOUT="" 43 | 44 | function run() { 45 | maybe_first_prompt 46 | rate=25 47 | if [ -n "$DEMO_RUN_FAST" ]; then 48 | rate=1000 49 | fi 50 | echo "$green$@$reset" | pv -qL $rate 51 | if [ -n "$DEMO_RUN_FAST" ]; then 52 | sleep 0.5 53 | fi 54 | if [ -z "$DEMO_AUTO_RUN" ]; then 55 | read -s option 56 | fi 57 | if [ "${option}" == "s" ]; then 58 | prompt 59 | return 0 60 | fi 61 | OFILE="$(mktemp -t $(basename $0).XXXXXX)" 62 | script -eq -c "$1" -f "$OFILE" 63 | r=$? 64 | read -d '' -t "${timeout}" -n 10000 # clear stdin 65 | DEMO_RUN_STDOUT="$(tail -n +2 $OFILE | sed 's/\r//g')" 66 | prompt 67 | if [ -z "$DEMO_AUTO_RUN" ]; then 68 | read -s option 69 | fi 70 | if [ "${option}" == "r" ]; then 71 | run "$1" 72 | fi 73 | return $r 74 | } 75 | 76 | function relative() { 77 | for arg; do 78 | echo "$(realpath $(dirname $(which $0)))/$arg" | sed "s|$(realpath $(pwd))|.|" 79 | done 80 | } 81 | 82 | trap "echo" EXIT 83 | -------------------------------------------------------------------------------- /demo.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | base_dir="$( cd "$(dirname "$0")/.." && pwd )" 4 | base_dir=${base_dir##$(pwd)/} 5 | base_dir=. 6 | 7 | source ${base_dir}/common/util.sh 8 | 9 | if [[ $# -ne 3 ]]; then 10 | echo "usage: $0 " 11 | exit 1 12 | fi 13 | 14 | Cluster=$1 15 | C1=$2 16 | C2=$3 17 | 18 | read -s 19 | clear 20 | 21 | # Demo 22 | kubectl config use-context ${Cluster} 23 | 24 | run "kubectl apply -f ${base_dir}/federated-service" 25 | 26 | run "kubectl --context=${C1} get pods" 27 | while [ "2" != "$(kubectl --context=${C1} get rs fr1 -o jsonpath="{.status.availableReplicas}")" ]; do 28 | sleep 3; 29 | done 30 | run "kubectl --context=${C1} get ep" 31 | 32 | run "kubectl --context=${C2} get pods" 33 | while [ "2" != "$(kubectl --context=${C2} get rs fr1 -o jsonpath="{.status.availableReplicas}")" ]; do 34 | sleep 3; 35 | done 36 | run "kubectl --context=${C2} get ep" 37 | 38 | run "kubectl run dnstools --rm --restart=Never -i --image=infoblox/dnstools --command -- curl -s fs1.default.production" 39 | 40 | run "kubectl patch federatedreplicasetplacements fr1 -p '{\"spec\":{\"clusternames\":[\"${C2}\"]}}'" 41 | 42 | run "kubectl run dnstools --rm --restart=Never -i --image=infoblox/dnstools --command -- curl -s fs1.default.production" 43 | -------------------------------------------------------------------------------- /federated-ingress/federatedingress.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: primitives.federation.k8s.io/v1alpha1 2 | kind: FederatedIngress 3 | metadata: 4 | name: fi1 5 | spec: 6 | template: 7 | spec: 8 | rules: 9 | - host: foo.fing.f8n.org 10 | http: 11 | paths: 12 | - backend: 13 | serviceName: fs1 14 | servicePort: 80 15 | --- 16 | apiVersion: primitives.federation.k8s.io/v1alpha1 17 | kind: FederatedIngressPlacement 18 | metadata: 19 | name: fi1 20 | spec: 21 | clusternames: 22 | - cluster1 23 | - cluster2 24 | -------------------------------------------------------------------------------- /federated-ingress/federatedreplicaset.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: primitives.federation.k8s.io/v1alpha1 2 | kind: FederatedReplicaSet 3 | metadata: 4 | name: fr1 5 | spec: 6 | template: 7 | spec: 8 | replicas: 1 9 | selector: 10 | matchLabels: 11 | app: demo 12 | template: 13 | metadata: 14 | name: demo 15 | labels: 16 | app: demo 17 | spec: 18 | containers: 19 | - name: demo 20 | image: shashidharatd/fed-sd-demo 21 | ports: 22 | - containerPort: 80 23 | env: 24 | - name: MY_NODE_NAME 25 | valueFrom: 26 | fieldRef: 27 | fieldPath: spec.nodeName 28 | - name: MY_POD_NAME 29 | valueFrom: 30 | fieldRef: 31 | fieldPath: metadata.name 32 | - name: MY_POD_NAMESPACE 33 | valueFrom: 34 | fieldRef: 35 | fieldPath: metadata.namespace 36 | - name: MY_POD_IP 37 | valueFrom: 38 | fieldRef: 39 | fieldPath: status.podIP 40 | - name: MY_POD_SERVICE_ACCOUNT 41 | valueFrom: 42 | fieldRef: 43 | fieldPath: spec.serviceAccountName 44 | --- 45 | apiVersion: primitives.federation.k8s.io/v1alpha1 46 | kind: FederatedReplicaSetPlacement 47 | metadata: 48 | name: fr1 49 | spec: 50 | clusterNames: 51 | - cluster1 52 | - cluster2 53 | -------------------------------------------------------------------------------- /federated-ingress/federatedservice.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: primitives.federation.k8s.io/v1alpha1 2 | kind: FederatedService 3 | metadata: 4 | name: fs1 5 | spec: 6 | template: 7 | spec: 8 | selector: 9 | app: demo 10 | ports: 11 | - protocol: TCP 12 | port: 80 13 | targetPort: 80 14 | --- 15 | apiVersion: primitives.federation.k8s.io/v1alpha1 16 | kind: FederatedServicePlacement 17 | metadata: 18 | name: fs1 19 | spec: 20 | clusterNames: 21 | - cluster1 22 | - cluster2 23 | -------------------------------------------------------------------------------- /federated-ingress/ingressdns.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: multiclusterdns.federation.k8s.io/v1alpha1 2 | kind: IngressDNSRecord 3 | metadata: 4 | name: fi1 5 | spec: 6 | hosts: 7 | - foo.fing.f8n.org 8 | recordTTL: 180 9 | -------------------------------------------------------------------------------- /federated-service/federatedreplicaset.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: primitives.federation.k8s.io/v1alpha1 2 | kind: FederatedReplicaSet 3 | metadata: 4 | name: fr1 5 | spec: 6 | template: 7 | spec: 8 | replicas: 1 9 | selector: 10 | matchLabels: 11 | app: demo 12 | template: 13 | metadata: 14 | name: demo 15 | labels: 16 | app: demo 17 | spec: 18 | containers: 19 | - name: demo 20 | image: shashidharatd/fed-sd-demo 21 | ports: 22 | - containerPort: 80 23 | env: 24 | - name: MY_NODE_NAME 25 | valueFrom: 26 | fieldRef: 27 | fieldPath: spec.nodeName 28 | - name: MY_POD_NAME 29 | valueFrom: 30 | fieldRef: 31 | fieldPath: metadata.name 32 | - name: MY_POD_NAMESPACE 33 | valueFrom: 34 | fieldRef: 35 | fieldPath: metadata.namespace 36 | - name: MY_POD_IP 37 | valueFrom: 38 | fieldRef: 39 | fieldPath: status.podIP 40 | - name: MY_POD_SERVICE_ACCOUNT 41 | valueFrom: 42 | fieldRef: 43 | fieldPath: spec.serviceAccountName 44 | --- 45 | apiVersion: primitives.federation.k8s.io/v1alpha1 46 | kind: FederatedReplicaSetPlacement 47 | metadata: 48 | name: fr1 49 | spec: 50 | clusterSelector: {} 51 | --- 52 | apiVersion: primitives.federation.k8s.io/v1alpha1 53 | kind: FederatedReplicaSetOverride 54 | metadata: 55 | name: fr1 56 | spec: 57 | overrides: 58 | - clusterName: cluster1 59 | clusterOverrides: 60 | - path: spec.replicas 61 | value: 1 62 | - clusterName: cluster2 63 | clusterOverrides: 64 | - path: spec.replicas 65 | value: 2 66 | -------------------------------------------------------------------------------- /federated-service/federatedservice.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: primitives.federation.k8s.io/v1alpha1 2 | kind: FederatedService 3 | metadata: 4 | name: fs1 5 | spec: 6 | template: 7 | spec: 8 | selector: 9 | app: demo 10 | ports: 11 | - protocol: TCP 12 | port: 80 13 | targetPort: 80 14 | type: LoadBalancer 15 | --- 16 | apiVersion: primitives.federation.k8s.io/v1alpha1 17 | kind: FederatedServicePlacement 18 | metadata: 19 | name: fs1 20 | spec: 21 | clusterSelector: {} 22 | -------------------------------------------------------------------------------- /federated-service/production-domain.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: multiclusterdns.federation.k8s.io/v1alpha1 2 | kind: Domain 3 | metadata: 4 | name: production 5 | namespace: federation-system 6 | domain: prod.f8n.org 7 | -------------------------------------------------------------------------------- /federated-service/servicedns.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: multiclusterdns.federation.k8s.io/v1alpha1 2 | kind: ServiceDNSRecord 3 | metadata: 4 | name: fs1 5 | spec: 6 | domainRef: production 7 | enableWeightedTargets: true 8 | dnsPrefix: mall 9 | -------------------------------------------------------------------------------- /globaldns/cleanup.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | base_dir="$( cd "$(dirname "$0")/../.." && pwd )" 4 | base_dir=${base_dir##$(pwd)/} 5 | 6 | source ${base_dir}/common/util.sh 7 | 8 | helm delete --purge coredns 9 | helm delete --purge globaldns 10 | -------------------------------------------------------------------------------- /globaldns/config/coredns-chart-values.yaml: -------------------------------------------------------------------------------- 1 | image: 2 | tag: "1.2.5" 3 | isClusterService: false 4 | service: 5 | type: "LoadBalancer" 6 | 7 | servers: 8 | - zones: 9 | - zone: . 10 | port: 53 11 | plugins: 12 | - name: health 13 | - name: errors 14 | - name: log 15 | - name: etcd 16 | parameters: "f8n.org." 17 | configBlock: |- 18 | endpoint "http://etcd.federation-system:2379" 19 | -------------------------------------------------------------------------------- /globaldns/config/federatedconfigmap-coredns.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: primitives.federation.k8s.io/v1alpha1 2 | kind: FederatedConfigMap 3 | metadata: 4 | name: coredns 5 | namespace: kube-system 6 | spec: 7 | template: 8 | metadata: 9 | name: coredns 10 | namespace: kube-system 11 | data: 12 | Corefile: | 13 | .:53 { 14 | errors 15 | health 16 | kubernetes cluster.local in-addr.arpa ip6.arpa { 17 | pods insecure 18 | upstream 19 | fallthrough in-addr.arpa ip6.arpa 20 | } 21 | prometheus :9153 22 | proxy . /etc/resolv.conf 23 | cache 30 24 | reload 25 | federation cluster.local { 26 | production prod.f8n.org 27 | staging stage.f8n.org 28 | upstream 29 | } 30 | } 31 | --- 32 | apiVersion: primitives.federation.k8s.io/v1alpha1 33 | kind: FederatedConfigMapPlacement 34 | metadata: 35 | name: coredns 36 | namespace: kube-system 37 | spec: 38 | clusterSelector: {} 39 | -------------------------------------------------------------------------------- /globaldns/config/federatedconfigmap-kubedns.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: primitives.federation.k8s.io/v1alpha1 2 | kind: FederatedConfigMap 3 | metadata: 4 | name: coredns 5 | namespace: kube-system 6 | spec: 7 | template: 8 | metadata: 9 | name: kube-dns 10 | namespace: kube-system 11 | labels: 12 | addonmanager.kubernetes.io/mode: EnsureExists 13 | data: 14 | federations: production=prod.f8n.org,staging=stage.f8n.org 15 | --- 16 | apiVersion: primitives.federation.k8s.io/v1alpha1 17 | kind: FederatedConfigMapPlacement 18 | metadata: 19 | name: kube-dns 20 | namespace: kube-system 21 | spec: 22 | clusterSelector: {} 23 | -------------------------------------------------------------------------------- /globaldns/config/globaldns-coredns-external-dns-chart-values.yaml: -------------------------------------------------------------------------------- 1 | provider: coredns 2 | 3 | image: 4 | name: quay.io/shashidharatd/external-dns 5 | tag: v0.5.8-dev 6 | 7 | sources: 8 | - crd 9 | 10 | extraArgs: 11 | crd-source-apiversion: multiclusterdns.federation.k8s.io/v1alpha1 12 | crd-source-kind: DNSEndpoint 13 | 14 | extraEnv: 15 | ETCD_URLS: "http://etcd.federation-system:2379" 16 | 17 | txt-owner-id: "baaz" 18 | -------------------------------------------------------------------------------- /globaldns/config/globaldns-google-external-dns-chart-values.yaml: -------------------------------------------------------------------------------- 1 | provider: google 2 | 3 | google: 4 | project: "fedv2-218905" 5 | serviceAccountSecret: "gcloud-config" 6 | 7 | sources: 8 | - crd 9 | 10 | extraArgs: 11 | crd-source-apiversion: multiclusterdns.federation.k8s.io/v1alpha1 12 | crd-source-kind: DNSEndpoint 13 | 14 | registry: "noop" 15 | -------------------------------------------------------------------------------- /globaldns/start-dns-servers.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | base_dir="$( cd "$(dirname "$0")/../.." && pwd )" 4 | base_dir=${base_dir##$(pwd)/} 5 | base_dir=. 6 | 7 | source ${base_dir}/common/util.sh 8 | 9 | if [[ $# -lt 1 ]]; then 10 | echo "usage: $0 " 11 | exit 1 12 | fi 13 | 14 | Cluster=$1 15 | Mode="v1" 16 | if [[ -n "$2" ]]; then 17 | Mode=$2 18 | fi 19 | NS="federation-system" 20 | 21 | read -s 22 | clear 23 | 24 | function wait_for_pods_to_be_ready() { 25 | Deployment=$1 26 | while [ "0" == "$(kubectl -n ${NS} get deployments ${Deployment} -o jsonpath="{.status.availableReplicas}")" ]; do 27 | echo "Waiting for ${Deployment} pod to be Running" 28 | sleep 3; 29 | done 30 | run "kubectl -n ${NS} get pods" 31 | } 32 | 33 | kubectl config use-context ${Cluster} 34 | 35 | run "helm init" 36 | 37 | run "helm version 2>/dev/null" 38 | while [ $? -ne 0 ]; do 39 | sleep 3 40 | helm version 2>/dev/null 41 | done 42 | 43 | run "# Start ETCD server. Used as backend for CoreDNS server" 44 | run "kubectl -n ${NS} run etcd --image=quay.io/coreos/etcd:v3.3 --env="ETCD_LISTEN_CLIENT_URLS=http://0.0.0.0:2379" --env="ETCD_ADVERTISE_CLIENT_URLS=http://etcd.${NS}:2379" --port=2379 --expose" 45 | wait_for_pods_to_be_ready "etcd" 46 | 47 | 48 | run "# Start CoreDNS server (Global DNS)" 49 | run "helm install --namespace ${NS} --name coredns -f ${base_dir}/globaldns/config/coredns-chart-values.yaml stable/coredns" 50 | wait_for_pods_to_be_ready "coredns-coredns" 51 | 52 | run "# Start External-DNS" 53 | run "helm install --namespace ${NS} --name globaldns -f ${base_dir}/globaldns/config/globaldns-coredns-external-dns-chart-values.yaml stable/external-dns" 54 | wait_for_pods_to_be_ready "globaldns-external-dns" 55 | 56 | run "kubectl -n ${NS} get pods" 57 | 58 | global_dns_server=$(kubectl -n federation-system get svc coredns-coredns -o jsonpath={.status.loadBalancer.ingress[0].ip}) 59 | 60 | echo "global_dns_server=${global_dns_server}" --------------------------------------------------------------------------------