├── 30service.yml ├── bootstrap ├── pv.sh ├── pvc.yml └── pv-template.yml ├── zookeeper ├── service.yml ├── bootstrap │ ├── pv.sh │ ├── pvc.yml │ └── pv-template.yml ├── test.sh ├── init │ ├── Makefile │ ├── Dockerfile │ ├── on-change.sh │ ├── install.sh │ └── on-start.sh ├── README.md └── zookeeper.yaml ├── 20dns.yml ├── test ├── 99testclient.yml ├── 21consumer-test1.yml ├── 11topic-create-test1.yml ├── 12topic-create-test2.yml └── test.sh ├── 50kafka.yml └── README.md /30service.yml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: v1 3 | kind: Service 4 | metadata: 5 | name: kafka 6 | namespace: kafka 7 | spec: 8 | ports: 9 | - port: 9092 10 | selector: 11 | app: kafka 12 | -------------------------------------------------------------------------------- /bootstrap/pv.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | echo "Creating persistant volume using hostPath volumes in the current Dir" 4 | 5 | dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && cd .. && pwd )" 6 | path="$dir/data" 7 | cat bootstrap/pv-template.yml | sed "s|/tmp/k8s-data|$path|" | kubectl create -f - 8 | -------------------------------------------------------------------------------- /zookeeper/service.yml: -------------------------------------------------------------------------------- 1 | # the headless service is for PetSet DNS, this one is for clients 2 | apiVersion: v1 3 | kind: Service 4 | metadata: 5 | name: zookeeper 6 | namespace: kafka 7 | spec: 8 | ports: 9 | - port: 2181 10 | name: client 11 | selector: 12 | app: zk 13 | -------------------------------------------------------------------------------- /zookeeper/bootstrap/pv.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | echo "Creating persistant volume using hostPath volumes in the current Dir" 3 | dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && cd .. && pwd )" 4 | path="$dir/data" 5 | cat zookeeper/bootstrap/pv-template.yml | sed "s|/tmp/k8s-data|$path|" | kubectl create -f - 6 | -------------------------------------------------------------------------------- /20dns.yml: -------------------------------------------------------------------------------- 1 | # A headless service to create DNS records 2 | --- 3 | apiVersion: v1 4 | kind: Service 5 | metadata: 6 | annotations: 7 | service.alpha.kubernetes.io/tolerate-unready-endpoints: "true" 8 | name: broker 9 | namespace: kafka 10 | spec: 11 | ports: 12 | - port: 9092 13 | # [podname].broker.kafka.svc.cluster.local 14 | clusterIP: None 15 | selector: 16 | app: kafka 17 | -------------------------------------------------------------------------------- /test/99testclient.yml: -------------------------------------------------------------------------------- 1 | # Kafka image without the service, so you can run ./bin/ stuff 2 | # kubectl exec -ti testclient -- /bin/bash 3 | apiVersion: v1 4 | kind: Pod 5 | metadata: 6 | name: testclient 7 | namespace: kafka 8 | spec: 9 | containers: 10 | - name: kafka 11 | image: solsson/kafka:0.10.0.1 12 | command: 13 | - sh 14 | - -c 15 | - "exec tail -f /dev/null" 16 | -------------------------------------------------------------------------------- /test/21consumer-test1.yml: -------------------------------------------------------------------------------- 1 | apiVersion: extensions/v1beta1 2 | kind: Deployment 3 | metadata: 4 | name: consumer-test1 5 | namespace: kafka 6 | spec: 7 | replicas: 1 8 | template: 9 | metadata: 10 | labels: 11 | app: consumer 12 | scope: test 13 | topic: test1 14 | spec: 15 | containers: 16 | - name: kafka 17 | image: solsson/kafka:0.10.0.1 18 | command: 19 | - ./bin/kafka-console-consumer.sh 20 | - --zookeeper 21 | - zookeeper:2181 22 | - --topic 23 | - test1 24 | - --from-beginning 25 | -------------------------------------------------------------------------------- /test/11topic-create-test1.yml: -------------------------------------------------------------------------------- 1 | apiVersion: batch/v1 2 | kind: Job 3 | metadata: 4 | name: topic-create-test1 5 | namespace: kafka 6 | spec: 7 | template: 8 | metadata: 9 | name: topic-create-test1 10 | spec: 11 | containers: 12 | - name: kafka 13 | image: solsson/kafka:0.10.0.1 14 | command: 15 | - ./bin/kafka-topics.sh 16 | - --zookeeper 17 | - zookeeper:2181 18 | - --create 19 | - --topic 20 | - test1 21 | - --partitions 22 | - "1" 23 | - --replication-factor 24 | - "1" 25 | restartPolicy: Never 26 | -------------------------------------------------------------------------------- /test/12topic-create-test2.yml: -------------------------------------------------------------------------------- 1 | apiVersion: batch/v1 2 | kind: Job 3 | metadata: 4 | name: topic-create-test2 5 | namespace: kafka 6 | spec: 7 | template: 8 | metadata: 9 | name: topic-create-test2 10 | spec: 11 | containers: 12 | - name: kafka 13 | image: solsson/kafka:0.10.0.1 14 | command: 15 | - ./bin/kafka-topics.sh 16 | - --zookeeper 17 | - zookeeper:2181 18 | - --create 19 | - --topic 20 | - test2 21 | - --partitions 22 | - "1" 23 | - --replication-factor 24 | - "3" 25 | restartPolicy: Never 26 | -------------------------------------------------------------------------------- /zookeeper/bootstrap/pvc.yml: -------------------------------------------------------------------------------- 1 | --- 2 | kind: PersistentVolumeClaim 3 | apiVersion: v1 4 | metadata: 5 | name: datadir-zoo-0 6 | namespace: kafka 7 | spec: 8 | accessModes: 9 | - ReadWriteOnce 10 | resources: 11 | requests: 12 | storage: 100Mi 13 | --- 14 | kind: PersistentVolumeClaim 15 | apiVersion: v1 16 | metadata: 17 | name: datadir-zoo-1 18 | namespace: kafka 19 | spec: 20 | accessModes: 21 | - ReadWriteOnce 22 | resources: 23 | requests: 24 | storage: 100Mi 25 | --- 26 | kind: PersistentVolumeClaim 27 | apiVersion: v1 28 | metadata: 29 | name: datadir-zoo-2 30 | namespace: kafka 31 | spec: 32 | accessModes: 33 | - ReadWriteOnce 34 | resources: 35 | requests: 36 | storage: 100Mi 37 | -------------------------------------------------------------------------------- /zookeeper/test.sh: -------------------------------------------------------------------------------- 1 | #! /bin/bash 2 | 3 | # Copyright 2016 The Kubernetes Authors All rights reserved. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | kubectl exec zoo-0 -- /opt/zookeeper/bin/zkCli.sh create /foo bar; 18 | kubectl exec zoo-2 -- /opt/zookeeper/bin/zkCli.sh get /foo; 19 | 20 | -------------------------------------------------------------------------------- /zookeeper/init/Makefile: -------------------------------------------------------------------------------- 1 | # Copyright 2016 The Kubernetes Authors All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | all: push 16 | 17 | TAG = 0.1 18 | PREFIX = gcr.io/google_containers/zookeeper-install 19 | 20 | container: 21 | docker build -t $(PREFIX):$(TAG) . 22 | 23 | push: container 24 | gcloud docker push $(PREFIX):$(TAG) 25 | 26 | clean: 27 | docker rmi $(PREFIX):$(TAG) 28 | -------------------------------------------------------------------------------- /bootstrap/pvc.yml: -------------------------------------------------------------------------------- 1 | --- 2 | kind: PersistentVolumeClaim 3 | apiVersion: v1 4 | metadata: 5 | name: datadir-kafka-0 6 | namespace: kafka 7 | spec: 8 | accessModes: 9 | - ReadWriteOnce 10 | resources: 11 | requests: 12 | storage: 100Mi 13 | --- 14 | kind: PersistentVolumeClaim 15 | apiVersion: v1 16 | metadata: 17 | name: datadir-kafka-1 18 | namespace: kafka 19 | spec: 20 | accessModes: 21 | - ReadWriteOnce 22 | resources: 23 | requests: 24 | storage: 100Mi 25 | --- 26 | kind: PersistentVolumeClaim 27 | apiVersion: v1 28 | metadata: 29 | name: datadir-kafka-2 30 | namespace: kafka 31 | spec: 32 | accessModes: 33 | - ReadWriteOnce 34 | resources: 35 | requests: 36 | storage: 100Mi 37 | --- 38 | kind: PersistentVolumeClaim 39 | apiVersion: v1 40 | metadata: 41 | name: datadir-kafka-3 42 | namespace: kafka 43 | spec: 44 | accessModes: 45 | - ReadWriteOnce 46 | resources: 47 | requests: 48 | storage: 100Mi 49 | --- 50 | kind: PersistentVolumeClaim 51 | apiVersion: v1 52 | metadata: 53 | name: datadir-kafka-4 54 | namespace: kafka 55 | spec: 56 | accessModes: 57 | - ReadWriteOnce 58 | resources: 59 | requests: 60 | storage: 100Mi 61 | -------------------------------------------------------------------------------- /zookeeper/init/Dockerfile: -------------------------------------------------------------------------------- 1 | # Copyright 2016 The Kubernetes Authors All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | # TODO: get rid of bash dependency and switch to plain busybox. 16 | # The tar in busybox also doesn't seem to understand compression. 17 | FROM debian:jessie 18 | MAINTAINER Prashanth.B 19 | 20 | RUN apt-get update && apt-get install -y wget netcat 21 | 22 | ADD on-start.sh / 23 | ADD on-change.sh / 24 | # See contrib/pets/peer-finder for details 25 | RUN wget -qO /peer-finder https://storage.googleapis.com/kubernetes-release/pets/peer-finder 26 | 27 | ADD install.sh / 28 | RUN chmod -c 755 /install.sh /on-start.sh /on-change.sh /peer-finder 29 | Entrypoint ["/install.sh"] 30 | -------------------------------------------------------------------------------- /50kafka.yml: -------------------------------------------------------------------------------- 1 | apiVersion: apps/v1beta1 2 | kind: StatefulSet 3 | metadata: 4 | name: kafka 5 | namespace: kafka 6 | spec: 7 | serviceName: "broker" 8 | replicas: 3 9 | template: 10 | metadata: 11 | labels: 12 | app: kafka 13 | annotations: 14 | pod.alpha.kubernetes.io/initialized: "true" 15 | pod.alpha.kubernetes.io/init-containers: '[ 16 | ]' 17 | spec: 18 | containers: 19 | - name: broker 20 | image: solsson/kafka:0.10.0.1 21 | ports: 22 | - containerPort: 9092 23 | command: 24 | - sh 25 | - -c 26 | - "./bin/kafka-server-start.sh config/server.properties --override broker.id=$(hostname | awk -F'-' '{print $2}')" 27 | volumeMounts: 28 | - name: datadir 29 | mountPath: /opt/kafka/data 30 | # - name: conf 31 | # mountPath: /opt/kafka/config/server.properties 32 | #volumes: 33 | #- name: conf 34 | # configMap: 35 | # name: conf-d 36 | volumeClaimTemplates: 37 | - metadata: 38 | name: datadir 39 | namespace: kafka 40 | annotations: 41 | volume.alpha.kubernetes.io/storage-class: anything 42 | spec: 43 | accessModes: [ "ReadWriteOnce" ] 44 | resources: 45 | requests: 46 | storage: 100Mi 47 | -------------------------------------------------------------------------------- /bootstrap/pv-template.yml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: v1 3 | kind: PersistentVolume 4 | metadata: 5 | name: datadir-kafka-0 6 | labels: 7 | app: kafka 8 | petindex: "0" 9 | spec: 10 | accessModes: 11 | - ReadWriteOnce 12 | capacity: 13 | storage: 100Mi 14 | hostPath: 15 | path: /tmp/k8s-data/datadir-kafka-0 16 | --- 17 | apiVersion: v1 18 | kind: PersistentVolume 19 | metadata: 20 | name: datadir-kafka-1 21 | labels: 22 | app: kafka 23 | petindex: "1" 24 | spec: 25 | accessModes: 26 | - ReadWriteOnce 27 | capacity: 28 | storage: 100Mi 29 | hostPath: 30 | path: /tmp/k8s-data/datadir-kafka-1 31 | --- 32 | apiVersion: v1 33 | kind: PersistentVolume 34 | metadata: 35 | name: datadir-kafka-2 36 | labels: 37 | app: kafka 38 | petindex: "2" 39 | spec: 40 | accessModes: 41 | - ReadWriteOnce 42 | capacity: 43 | storage: 100Mi 44 | hostPath: 45 | path: /tmp/k8s-data/datadir-kafka-2 46 | --- 47 | apiVersion: v1 48 | kind: PersistentVolume 49 | metadata: 50 | name: datadir-kafka-3 51 | labels: 52 | app: kafka 53 | petindex: "3" 54 | spec: 55 | accessModes: 56 | - ReadWriteOnce 57 | capacity: 58 | storage: 100Mi 59 | hostPath: 60 | path: /tmp/k8s-data/datadir-kafka-3 61 | --- 62 | apiVersion: v1 63 | kind: PersistentVolume 64 | metadata: 65 | name: datadir-kafka-4 66 | labels: 67 | app: kafka 68 | petindex: "4" 69 | spec: 70 | accessModes: 71 | - ReadWriteOnce 72 | capacity: 73 | storage: 100Mi 74 | hostPath: 75 | path: /tmp/k8s-data/datadir-kafka-4 76 | -------------------------------------------------------------------------------- /zookeeper/bootstrap/pv-template.yml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: v1 3 | kind: PersistentVolume 4 | metadata: 5 | name: datadir-zoo-0 6 | namespace: kafka 7 | labels: 8 | app: zk 9 | petindex: "0" 10 | spec: 11 | accessModes: 12 | - ReadWriteOnce 13 | capacity: 14 | storage: 100Mi 15 | hostPath: 16 | path: /tmp/k8s-data/datadir-zoo-0 17 | --- 18 | apiVersion: v1 19 | kind: PersistentVolume 20 | metadata: 21 | name: datadir-zoo-1 22 | namespace: kafka 23 | labels: 24 | app: zk 25 | petindex: "1" 26 | spec: 27 | accessModes: 28 | - ReadWriteOnce 29 | capacity: 30 | storage: 100Mi 31 | hostPath: 32 | path: /tmp/k8s-data/datadir-zoo-1 33 | --- 34 | apiVersion: v1 35 | kind: PersistentVolume 36 | metadata: 37 | name: datadir-zoo-2 38 | namespace: kafka 39 | labels: 40 | app: zk 41 | petindex: "2" 42 | spec: 43 | accessModes: 44 | - ReadWriteOnce 45 | capacity: 46 | storage: 100Mi 47 | hostPath: 48 | path: /tmp/k8s-data/datadir-zoo-2 49 | --- 50 | apiVersion: v1 51 | kind: PersistentVolume 52 | metadata: 53 | name: datadir-zoo-3 54 | namespace: kafka 55 | labels: 56 | app: zk 57 | petindex: "3" 58 | spec: 59 | accessModes: 60 | - ReadWriteOnce 61 | capacity: 62 | storage: 100Mi 63 | hostPath: 64 | path: /tmp/k8s-data/datadir-zoo-3 65 | --- 66 | apiVersion: v1 67 | kind: PersistentVolume 68 | metadata: 69 | name: datadir-zoo-4 70 | namespace: kafka 71 | labels: 72 | app: zk 73 | petindex: "4" 74 | spec: 75 | accessModes: 76 | - ReadWriteOnce 77 | capacity: 78 | storage: 100Mi 79 | hostPath: 80 | path: /tmp/k8s-data/datadir-zoo-4 81 | -------------------------------------------------------------------------------- /zookeeper/init/on-change.sh: -------------------------------------------------------------------------------- 1 | #! /bin/bash 2 | 3 | # Copyright 2016 The Kubernetes Authors All rights reserved. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | # This script configures zookeeper cluster member ship for version of zookeeper 18 | # < 3.5.0. It should not be used with the on-start.sh script in this example. 19 | # As of April-2016 is 3.4.8 is the latest stable. 20 | 21 | CFG=/opt/zookeeper/conf/zoo.cfg 22 | CFG_BAK=/opt/zookeeper/conf/zoo.cfg.bak 23 | MY_ID=/tmp/zookeeper/myid 24 | 25 | # write myid 26 | IFS='-' read -ra ADDR <<< "$(hostname)" 27 | echo $(expr "1" + "${ADDR[1]}") > "${MY_ID}" 28 | 29 | # TODO: This is a dumb way to reconfigure zookeeper because it allows dynamic 30 | # reconfig, but it's simple. 31 | i=0 32 | echo " 33 | tickTime=2000 34 | initLimit=10 35 | syncLimit=5 36 | dataDir=/tmp/zookeeper 37 | clientPort=2181 38 | " > "${CFG_BAK}" 39 | 40 | while read -ra LINE; do 41 | let i=i+1 42 | echo "server.${i}=${LINE}:2888:3888" >> "${CFG_BAK}" 43 | done 44 | cp ${CFG_BAK} ${CFG} 45 | 46 | # TODO: Typically one needs to first add a new member as an "observer" then 47 | # promote it to "participant", but that requirement is relaxed if we never 48 | # start > 1 at a time. 49 | /opt/zookeeper/bin/zkServer.sh restart 50 | -------------------------------------------------------------------------------- /test/test.sh: -------------------------------------------------------------------------------- 1 | 2 | # List topics 3 | kubectl exec testclient -- ./bin/kafka-topics.sh --zookeeper zookeeper:2181 --list 4 | 5 | # Create topic 6 | kubectl exec testclient -- ./bin/kafka-topics.sh --zookeeper zookeeper:2181 --topic test1 --create --partitions 1 --replication-factor 1 7 | 8 | # Set one of your terminals to listen to messages on the test topic 9 | kubectl exec -ti testclient -- ./bin/kafka-console-consumer.sh --zookeeper zookeeper:2181 --topic test1 --from-beginning 10 | 11 | # Go ahead and produce messages 12 | echo "Write a message followed by enter, exit using Ctrl+C" 13 | kubectl exec -ti testclient -- ./bin/kafka-console-producer.sh --broker-list kafka-0.broker.kafka.svc.cluster.local:9092 --topic test1 14 | 15 | # Bootstrap even if two nodes are down (shorter name requires same namespace) 16 | kubectl exec -ti testclient -- ./bin/kafka-console-producer.sh --broker-list kafka-0.broker:9092,kafka-1.broker:9092,kafka-2.broker:9092 --topic test1 17 | 18 | # The following commands run in the pod 19 | kubectl exec -ti testclient -- /bin/bash 20 | 21 | # Topic 2, replicated 22 | ./bin/kafka-topics.sh --zookeeper zookeeper:2181 --describe --topic test2 23 | 24 | ./bin/kafka-verifiable-consumer.sh \ 25 | --broker-list=kafka-0.broker.kafka.svc.cluster.local:9092,kafka-1.broker.kafka.svc.cluster.local:9092 \ 26 | --topic=test2 --group-id=A --verbose 27 | 28 | # If a topic isn't available this producer will tell you 29 | # WARN Error while fetching metadata with correlation id X : {topicname=LEADER_NOT_AVAILABLE} 30 | # ... but with current config Kafka will auto-create the topic 31 | ./bin/kafka-verifiable-producer.sh \ 32 | --broker-list=kafka-0.broker.kafka.svc.cluster.local:9092,kafka-1.broker.kafka.svc.cluster.local:9092 \ 33 | --value-prefix=1 --topic=test2 \ 34 | --acks=1 --throughput=1 --max-messages=10 35 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | > Thanks https://github.com/Yolean/kubernetes-kafka. 2 | 3 | 4 | ```sh 5 | 6 | # Pre-req 7 | oc cluster up # setup openshift on docker for Mac (Skip this if you already have a cluster) 8 | git clone https://github.com/debianmaster/openshift-kafka.git && cd openshift-kafka 9 | oc login -u system:admin 10 | 11 | 12 | 13 | # Create namespace 14 | oc new-project kafka 15 | 16 | # Unfortunately we need root priv for the containers that are used in this example 17 | oc adm policy add-scc-to-user anyuid -z default 18 | 19 | # Create Persistant Volumes No londer need these 20 | #./zookeeper/bootstrap/pv.sh 21 | #./bootstrap/pv.sh 22 | 23 | # Create Persistant volumes claims 24 | oc create -f ./bootstrap/pvc.yml 25 | oc create -f ./zookeeper/bootstrap/pvc.yml 26 | 27 | # Create Zookeeper & Kakfa 28 | oc create -f ./zookeeper/service.yml 29 | oc create -f ./zookeeper/zookeeper.yaml 30 | oc create -f ./ 31 | 32 | # Test 33 | oc create -f test/99testclient.yml 34 | oc rsh testclient #terminal 1 35 | oc rsh testclient #terminal 2 36 | 37 | ## Inside testclient container Terminal 1 38 | ./bin/kafka-topics.sh --zookeeper zookeeper:2181 --topic test1 --create --partitions 1 \ 39 | --replication-factor 1 #create topic 40 | ./bin/kafka-console-consumer.sh --zookeeper zookeeper:2181 --topic test1 \ 41 | --from-beginning #read topic 42 | 43 | ## Inside testclient container Terminal 2 44 | ./bin/kafka-console-producer.sh --broker-list kafka-0.broker.kafka.svc.cluster.local:9092,\ 45 | kafka-1.broker.kafka.svc.cluster.local:9092 --topic test1 # Type message and press ^Z when done, check terminal 1 46 | 47 | 48 | # Scale up Cluster 49 | oc edit petset kakfa #change replicas to desired number (make sure you have enough PV,PVC) 50 | 51 | # Cleanup 52 | oc delete all,petset --all 53 | oc delete pv,pvc --all 54 | ``` 55 | 56 | ![Demo Image](https://pbs.twimg.com/media/Cx5nXXQVIAEOvzL.jpg:large) 57 | -------------------------------------------------------------------------------- /zookeeper/README.md: -------------------------------------------------------------------------------- 1 | # Zookeeper 2 | 3 | This example runs zookeeper through a petset. 4 | 5 | ## Bootstrap 6 | 7 | Create the petset in this directory 8 | ``` 9 | $ kubetl create -f zookeeper.yaml 10 | ``` 11 | 12 | Once you have all 3 nodes in Running, you can run the "test.sh" script in this directory. 13 | 14 | ## Failover 15 | 16 | You can test failover by killing the leader. Insert a key: 17 | ```console 18 | $ kubectl exec zoo-0 -- /opt/zookeeper/bin/zkCli.sh create /foo bar; 19 | $ kubectl exec zoo-2 -- /opt/zookeeper/bin/zkCli.sh get /foo; 20 | 21 | Watch existing members: 22 | ```console 23 | $ kubectl run --attach bbox --image=busybox --restart=Never -- sh -c 'while true; do for i in 0 1 2; do echo zoo-$i $(echo stats | nc zoo-$i.zk:2181 | grep Mode); sleep 1; done; done'; 24 | zoo-2 Mode: follower 25 | zoo-0 Mode: follower 26 | zoo-1 Mode: leader 27 | zoo-2 Mode: follower 28 | ``` 29 | 30 | Delete pets and wait for the petset controller to bring the back up: 31 | ```console 32 | $ kubectl delete po -l app=zk 33 | $ kubectl get po --watch-only 34 | NAME READY STATUS RESTARTS AGE 35 | zoo-0 0/1 Init:0/2 0 16s 36 | zoo-0 0/1 Init:0/2 0 21s 37 | zoo-0 0/1 PodInitializing 0 23s 38 | zoo-0 1/1 Running 0 41s 39 | zoo-1 0/1 Pending 0 0s 40 | zoo-1 0/1 Init:0/2 0 0s 41 | zoo-1 0/1 Init:0/2 0 14s 42 | zoo-1 0/1 PodInitializing 0 17s 43 | zoo-1 0/1 Running 0 18s 44 | zoo-2 0/1 Pending 0 0s 45 | zoo-2 0/1 Init:0/2 0 0s 46 | zoo-2 0/1 Init:0/2 0 12s 47 | zoo-2 0/1 Init:0/2 0 28s 48 | zoo-2 0/1 PodInitializing 0 31s 49 | zoo-2 0/1 Running 0 32s 50 | ... 51 | 52 | zoo-0 Mode: follower 53 | zoo-1 Mode: leader 54 | zoo-2 Mode: follower 55 | ``` 56 | 57 | Check the previously inserted key: 58 | ```console 59 | $ kubectl exec zoo-1 -- /opt/zookeeper/bin/zkCli.sh get /foo 60 | ionid = 0x354887858e80035, negotiated timeout = 30000 61 | 62 | WATCHER:: 63 | 64 | WatchedEvent state:SyncConnected type:None path:null 65 | bar 66 | ``` 67 | 68 | ## Scaling 69 | 70 | You can scale up by modifying the number of replicas on the PetSet. 71 | 72 | ## Image Upgrade 73 | 74 | TODO: Add details 75 | 76 | ## Maintenance 77 | 78 | TODO: Add details 79 | 80 | ## Limitations 81 | * Both petset and init containers are in alpha 82 | * Look through the on-start and on-change scripts for TODOs 83 | * Doesn't support the addition of observers through the petset 84 | * Only supports storage options that have backends for persistent volume claims 85 | 86 | -------------------------------------------------------------------------------- /zookeeper/init/install.sh: -------------------------------------------------------------------------------- 1 | #! /bin/bash 2 | 3 | # Copyright 2016 The Kubernetes Authors All rights reserved. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | # This volume is assumed to exist and is shared with parent of the init 18 | # container. It contains the zookeeper installation. 19 | INSTALL_VOLUME="/opt" 20 | 21 | # This volume is assumed to exist and is shared with the peer-finder 22 | # init container. It contains on-start/change configuration scripts. 23 | WORKDIR_VOLUME="/work-dir" 24 | 25 | # As of April-2016 is 3.4.8 is the latest stable, but versions 3.5.0 onward 26 | # allow dynamic reconfiguration. 27 | VERSION="3.5.0-alpha" 28 | 29 | for i in "$@" 30 | do 31 | case $i in 32 | -v=*|--version=*) 33 | VERSION="${i#*=}" 34 | shift 35 | ;; 36 | -i=*|--install-into=*) 37 | INSTALL_VOLUME="${i#*=}" 38 | shift 39 | ;; 40 | -w=*|--work-dir=*) 41 | WORKDIR_VOLUME="${i#*=}" 42 | shift 43 | ;; 44 | *) 45 | # unknown option 46 | ;; 47 | esac 48 | done 49 | 50 | echo installing config scripts into "${WORKDIR_VOLUME}" 51 | mkdir -p "${WORKDIR_VOLUME}" 52 | cp /on-start.sh "${WORKDIR_VOLUME}"/ 53 | cp /on-change.sh "${WORKDIR_VOLUME}"/ 54 | cp /peer-finder "${WORKDIR_VOLUME}"/ 55 | 56 | echo installing zookeeper-"${VERSION}" into "${INSTALL_VOLUME}" 57 | mkdir -p "${INSTALL_VOLUME}" 58 | wget -q -O - http://apache.mirrors.pair.com/zookeeper/zookeeper-"${VERSION}"/zookeeper-"${VERSION}".tar.gz | tar -xzf - -C "${INSTALL_VOLUME}" 59 | mv "${INSTALL_VOLUME}"/zookeeper-"${VERSION}" "${INSTALL_VOLUME}"/zookeeper 60 | cp "${INSTALL_VOLUME}"/zookeeper/conf/zoo_sample.cfg "${INSTALL_VOLUME}"/zookeeper/conf/zoo.cfg 61 | 62 | # TODO: Should dynamic config be tied to the version? 63 | IFS="." read -ra RELEASE <<< "${VERSION}" 64 | if [ $(expr "${RELEASE[1]}") -gt 4 ]; then 65 | echo zookeeper-"${VERSION}" supports dynamic reconfiguration, enabling it 66 | echo "standaloneEnabled=false" >> "${INSTALL_VOLUME}"/zookeeper/conf/zoo.cfg 67 | echo "dynamicConfigFile="${INSTALL_VOLUME}"/zookeeper/conf/zoo.cfg.dynamic" >> "${INSTALL_VOLUME}"/zookeeper/conf/zoo.cfg 68 | fi 69 | 70 | # TODO: This is a hack, netcat is convenient to have in the zookeeper container 71 | # I want to avoid using a custom zookeeper image just for this. So copy it. 72 | NC=$(which nc) 73 | if [ "${NC}" != "" ]; then 74 | echo copying nc into "${INSTALL_VOLUME}" 75 | cp "${NC}" "${INSTALL_VOLUME}" 76 | fi 77 | -------------------------------------------------------------------------------- /zookeeper/init/on-start.sh: -------------------------------------------------------------------------------- 1 | #! /bin/bash 2 | 3 | # Copyright 2016 The Kubernetes Authors All rights reserved. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | # This script configures zookeeper cluster member ship for version of zookeeper 18 | # >= 3.5.0. It should not be used with the on-change.sh script in this example. 19 | # As of April-2016 is 3.4.8 is the latest stable. 20 | 21 | # Both /opt and /tmp/zookeeper are assumed to be volumes shared with the parent. 22 | CFG=/opt/zookeeper/conf/zoo.cfg.dynamic 23 | CFG_BAK=/opt/zookeeper/conf/zoo.cfg.bak 24 | MY_ID_FILE=/tmp/zookeeper/myid 25 | HOSTNAME=$(hostname) 26 | 27 | while read -ra LINE; do 28 | PEERS=("${PEERS[@]}" $LINE) 29 | done 30 | 31 | # Don't add the first member as an observer 32 | if [ ${#PEERS[@]} -eq 1 ]; then 33 | # We need to write our index in this list of servers into MY_ID_FILE. 34 | # Note that this may not always coincide with the hostname id. 35 | echo 1 > "${MY_ID_FILE}" 36 | echo "server.1=${PEERS[0]}:2888:3888;2181" > "${CFG}" 37 | # TODO: zkServer-initialize is the safe way to handle changes to datadir 38 | # because simply starting will create a new datadir, BUT if the user changed 39 | # pod template they might end up with 2 datadirs and brief split brain. 40 | exit 41 | fi 42 | 43 | # Every subsequent member is added as an observer and promoted to a participant 44 | echo "" > "${CFG_BAK}" 45 | i=0 46 | for peer in "${PEERS[@]}"; do 47 | let i=i+1 48 | if [[ "${peer}" == *"${HOSTNAME}"* ]]; then 49 | MY_ID=$i 50 | MY_NAME=${peer} 51 | echo $i > "${MY_ID_FILE}" 52 | echo "server.${i}=${peer}:2888:3888:observer;2181" >> "${CFG_BAK}" 53 | else 54 | echo "server.${i}=${peer}:2888:3888:participant;2181" >> "${CFG_BAK}" 55 | fi 56 | done 57 | 58 | # Once the dynamic config file is written it shouldn't be modified, so the final 59 | # reconfigure needs to happen through the "reconfig" command. 60 | cp ${CFG_BAK} ${CFG} 61 | 62 | # TODO: zkServer-initialize is the safe way to handle changes to datadir 63 | # because simply starting will create a new datadir, BUT if the user changed 64 | # pod template they might end up with 2 datadirs and brief split brain. 65 | /opt/zookeeper/bin/zkServer.sh start 66 | 67 | # TODO: We shouldn't need to specify the address of the master as long as 68 | # there's quorum. According to the docs the new server is just not allowed to 69 | # vote, it's still allowed to propose config changes, and it knows the 70 | # existing members of the ensemble from *its* config. This works as expected, 71 | # but we should correlate with more satisfying empirical evidence. 72 | /opt/zookeeper/bin/zkCli.sh reconfig -add "server.$MY_ID=$MY_NAME:2888:3888:participant;2181" 73 | /opt/zookeeper/bin/zkServer.sh stop 74 | -------------------------------------------------------------------------------- /zookeeper/zookeeper.yaml: -------------------------------------------------------------------------------- 1 | # A headless service to create DNS records 2 | apiVersion: v1 3 | kind: Service 4 | metadata: 5 | annotations: 6 | service.alpha.kubernetes.io/tolerate-unready-endpoints: "true" 7 | name: zk 8 | namespace: kafka 9 | labels: 10 | app: zk 11 | spec: 12 | ports: 13 | - port: 2888 14 | name: peer 15 | - port: 3888 16 | name: leader-election 17 | # *.zk.default.svc.cluster.local 18 | clusterIP: None 19 | selector: 20 | app: zk 21 | --- 22 | apiVersion: apps/v1beta1 23 | kind: StatefulSet 24 | metadata: 25 | name: zoo 26 | namespace: kafka 27 | spec: 28 | serviceName: "zk" 29 | replicas: 3 30 | template: 31 | metadata: 32 | labels: 33 | app: zk 34 | annotations: 35 | pod.alpha.kubernetes.io/initialized: "true" 36 | pod.alpha.kubernetes.io/init-containers: '[ 37 | { 38 | "name": "install", 39 | "image": "gcr.io/google_containers/zookeeper-install:0.1", 40 | "imagePullPolicy": "Always", 41 | "args": ["--version=3.5.2-alpha", "--install-into=/opt", "--work-dir=/work-dir"], 42 | "volumeMounts": [ 43 | { 44 | "name": "opt", 45 | "mountPath": "/opt/" 46 | }, 47 | { 48 | "name": "workdir", 49 | "mountPath": "/work-dir" 50 | } 51 | ] 52 | }, 53 | { 54 | "name": "bootstrap", 55 | "image": "java:openjdk-8-jre", 56 | "command": ["/work-dir/peer-finder"], 57 | "args": ["-on-start=\"/work-dir/on-start.sh\"", "-service=zk"], 58 | "env": [ 59 | { 60 | "name": "POD_NAMESPACE", 61 | "valueFrom": { 62 | "fieldRef": { 63 | "apiVersion": "v1", 64 | "fieldPath": "metadata.namespace" 65 | } 66 | } 67 | } 68 | ], 69 | "volumeMounts": [ 70 | { 71 | "name": "opt", 72 | "mountPath": "/opt/" 73 | }, 74 | { 75 | "name": "workdir", 76 | "mountPath": "/work-dir" 77 | }, 78 | { 79 | "name": "datadir", 80 | "mountPath": "/tmp/zookeeper" 81 | } 82 | ] 83 | } 84 | ]' 85 | spec: 86 | containers: 87 | - name: zk 88 | image: java:openjdk-8-jre 89 | ports: 90 | - containerPort: 2181 91 | name: client 92 | - containerPort: 2888 93 | name: peer 94 | - containerPort: 3888 95 | name: leader-election 96 | command: 97 | - /opt/zookeeper/bin/zkServer.sh 98 | args: 99 | - start-foreground 100 | readinessProbe: 101 | exec: 102 | command: 103 | - sh 104 | - -c 105 | - "/opt/zookeeper/bin/zkCli.sh ls /" 106 | initialDelaySeconds: 15 107 | timeoutSeconds: 5 108 | volumeMounts: 109 | - name: datadir 110 | mountPath: /tmp/zookeeper 111 | - name: opt 112 | mountPath: /opt/ 113 | volumes: 114 | - name: opt 115 | emptyDir: {} 116 | - name: workdir 117 | emptyDir: {} 118 | volumeClaimTemplates: 119 | - metadata: 120 | name: datadir 121 | namespace: kafka 122 | annotations: 123 | volume.alpha.kubernetes.io/storage-class: anything 124 | spec: 125 | accessModes: [ "ReadWriteOnce" ] 126 | resources: 127 | requests: 128 | storage: 20Gi 129 | --------------------------------------------------------------------------------