├── continuse ├── mpi │ └── .shared_directory_for_mpi └── kube │ ├── bin │ ├── kubectl.sh │ └── chk_master.sh │ └── service │ ├── kube-proxy.service │ ├── kube-scheduler.service │ ├── kube-kubelet.service │ ├── kube-controller-manager.service │ └── kube-apiserver.service ├── examples ├── mpich │ ├── images │ │ ├── bash_profile │ │ ├── build │ │ ├── auto_ssh.sh │ │ ├── known_hosts.py │ │ ├── Dockerfile │ │ ├── entrypoint.sh │ │ ├── pod_ip.py │ │ └── README.md │ ├── mpich-service.yaml │ ├── mpich-controller.yaml │ ├── mpich-master.yaml │ └── README.md ├── redis-cluster │ ├── images │ │ ├── bash_profile │ │ ├── build │ │ ├── Dockerfile │ │ ├── entrypoint.sh │ │ ├── README.md │ │ ├── auto_config_redis_cluster.py │ │ └── redis.conf │ ├── redis-cluster-worker-service.yaml │ ├── redis-cluster-worker.yaml │ ├── redis-cluster-manager.yaml │ └── README.md └── kube-ui │ ├── kube-system.yaml │ ├── run.sh │ ├── kube-ui-svc.yaml │ └── kube-ui-rc.yaml ├── kube ├── user-data ├── config.rb └── Vagrantfile ├── controller ├── user-data ├── config.rb └── Vagrantfile └── README.md /continuse/mpi/.shared_directory_for_mpi: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /examples/mpich/images/bash_profile: -------------------------------------------------------------------------------- 1 | export PATH=$PATH:/tmp/mpich/bin 2 | -------------------------------------------------------------------------------- /examples/mpich/images/build: -------------------------------------------------------------------------------- 1 | sudo docker build --tag continuse/mpich:v3 . 2 | -------------------------------------------------------------------------------- /examples/redis-cluster/images/bash_profile: -------------------------------------------------------------------------------- 1 | export PATH=$PATH:/tmp/mpich/bin 2 | -------------------------------------------------------------------------------- /examples/redis-cluster/images/build: -------------------------------------------------------------------------------- 1 | sudo docker build --tag continuse/redis:v3 . 2 | -------------------------------------------------------------------------------- /examples/kube-ui/kube-system.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Namespace 3 | metadata: 4 | name: kube-system 5 | labels: 6 | ENV: Monitor 7 | 8 | -------------------------------------------------------------------------------- /examples/kube-ui/run.sh: -------------------------------------------------------------------------------- 1 | kubectl create -f kube-system.yaml 2 | kubectl create -f kube-ui-rc.yaml --namespace=kube-system 3 | kubectl create -f kube-ui-svc.yaml --namespace=kube-system 4 | -------------------------------------------------------------------------------- /continuse/kube/bin/kubectl.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | master=`/usr/bin/etcdctl get /continuse/kube/master` 4 | echo "Master Node IP Address : $master" 5 | 6 | kubectl --server="$master:8080" $* 7 | -------------------------------------------------------------------------------- /examples/mpich/images/auto_ssh.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/expect 2 | 3 | set server [lindex $argv 0] 4 | 5 | spawn ssh -o "StrictHostKeyChecking no" root@$server 6 | expect -re "#" 7 | send "exit" 8 | -------------------------------------------------------------------------------- /continuse/kube/service/kube-proxy.service: -------------------------------------------------------------------------------- 1 | [Unit] 2 | Description=Kubernetes Proxy 3 | Documentation=https://github.com/GoogleCloudPlatform/kubernetes 4 | 5 | [Service] 6 | ExecStart=/continuse/kube/bin/chk_master.sh proxy 7 | Restart=always 8 | RestartSec=10 9 | 10 | [X-Fleet] 11 | Global=true 12 | MachineMetadata=role=kube 13 | -------------------------------------------------------------------------------- /examples/kube-ui/kube-ui-svc.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Service 3 | metadata: 4 | name: kube-ui 5 | namespace: kube-system 6 | labels: 7 | k8s-app: kube-ui 8 | kubernetes.io/cluster-service: "true" 9 | kubernetes.io/name: "KubeUI" 10 | spec: 11 | selector: 12 | k8s-app: kube-ui 13 | ports: 14 | - port: 80 15 | targetPort: 8080 16 | -------------------------------------------------------------------------------- /examples/mpich/mpich-service.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Service 3 | metadata: 4 | name: mpich-master 5 | # namespace: mpich-system 6 | labels: 7 | name: mpich-master 8 | spec: 9 | type: NodePort 10 | ports: 11 | # the port that this service should serve on 12 | - port: 22 13 | targetPort: 22 14 | nodePort: 32000 15 | selector: 16 | name: mpich-master 17 | -------------------------------------------------------------------------------- /continuse/kube/service/kube-scheduler.service: -------------------------------------------------------------------------------- 1 | [Unit] 2 | Description=Kubernetes Scheduler 3 | Documentation=https://github.com/GoogleCloudPlatform/kubernetes 4 | Requires=kube-apiserver.service 5 | After=kube-apiserver.service 6 | 7 | [Service] 8 | ExecStart=/continuse/kube/bin/kube-scheduler --master=127.0.0.1:8080 9 | Restart=always 10 | RestartSec=10 11 | 12 | [X-Fleet] 13 | MachineOf=kube-apiserver.service 14 | -------------------------------------------------------------------------------- /examples/redis-cluster/redis-cluster-worker-service.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Service 3 | metadata: 4 | name: redis-cluster-worker 5 | # namespace: redis-cluster-system 6 | labels: 7 | name: redis-worker 8 | spec: 9 | type: NodePort 10 | ports: 11 | # the port that this service should serve on 12 | - port: 6379 13 | targetPort: 6379 14 | nodePort: 32100 15 | selector: 16 | name: redis-cluster-worker 17 | -------------------------------------------------------------------------------- /continuse/kube/service/kube-kubelet.service: -------------------------------------------------------------------------------- 1 | [Unit] 2 | Description=Kubernetes Kubelet 3 | Documentation=https://github.com/GoogleCloudPlatform/kubernetes 4 | Requires=flanneld.service 5 | After=flanneld.service 6 | 7 | [Service] 8 | EnvironmentFile=/etc/environment 9 | ExecStart=/continuse/kube/bin/chk_master.sh kubelet ${COREOS_PUBLIC_IPV4} 10 | Restart=always 11 | RestartSec=10 12 | 13 | [X-Fleet] 14 | Global=true 15 | MachineMetadata=role=kube 16 | -------------------------------------------------------------------------------- /examples/mpich/images/known_hosts.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | 3 | import sys 4 | 5 | new_hosts = "%s" % sys.argv[1] 6 | old_hosts = "%s.old" % sys.argv[1] 7 | 8 | with open(new_hosts) as f: 9 | new = f.read().splitlines() 10 | 11 | with open(old_hosts) as f: 12 | old = f.read().splitlines() 13 | 14 | new1 = set(new) 15 | old1 = set(old) 16 | 17 | if old1.issubset(new1) : 18 | hosts = set.difference(new1, old1) 19 | 20 | for i in hosts: 21 | print i 22 | -------------------------------------------------------------------------------- /continuse/kube/service/kube-controller-manager.service: -------------------------------------------------------------------------------- 1 | [Unit] 2 | Description=Kubernetes Controller Manager 3 | Documentation=https://github.com/GoogleCloudPlatform/kubernetes 4 | Requires=kube-apiserver.service 5 | After=kube-apiserver.service 6 | 7 | [Service] 8 | ExecStart=/continuse/kube/bin/kube-controller-manager \ 9 | --service_account_private_key_file=/continuse/kube/bin/kube-serviceaccount.key \ 10 | --master=127.0.0.1:8080 \ 11 | --logtostderr=true 12 | Restart=always 13 | RestartSec=10 14 | 15 | [X-Fleet] 16 | MachineOf=kube-apiserver.service 17 | -------------------------------------------------------------------------------- /examples/redis-cluster/redis-cluster-worker.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: ReplicationController 3 | metadata: 4 | name: redis-cluster-worker 5 | # namespace: mpich-system 6 | spec: 7 | replicas: 6 8 | selector: 9 | name: redis-cluster-worker 10 | template: 11 | metadata: 12 | labels: 13 | name: redis-cluster-worker 14 | spec: 15 | containers: 16 | - name: redis-cluster-worker 17 | image: continuse/redis:v3 18 | env: 19 | - name: WORKER 20 | value: "true" 21 | ports: 22 | - containerPort: 6379 23 | resources: 24 | limits: 25 | cpu: "0.1" 26 | 27 | -------------------------------------------------------------------------------- /examples/mpich/mpich-controller.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: ReplicationController 3 | metadata: 4 | name: mpich-worker 5 | # namespace: mpich-system 6 | spec: 7 | replicas: 2 8 | selector: 9 | name: mpich-worker 10 | template: 11 | metadata: 12 | labels: 13 | name: mpich-worker 14 | spec: 15 | containers: 16 | - name: mpich-worker 17 | image: continuse/mpich:v3 18 | env: 19 | - name: WORKER 20 | value: "true" 21 | ports: 22 | - containerPort: 22 23 | resources: 24 | limits: 25 | cpu: "0.1" 26 | volumeMounts: 27 | - mountPath: /share 28 | name: share 29 | volumes: 30 | - name: share 31 | hostPath: 32 | path: /continuse/mpi 33 | 34 | -------------------------------------------------------------------------------- /examples/mpich/mpich-master.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Pod 3 | metadata: 4 | name: mpich-master 5 | # namespace: mpich-system 6 | labels: 7 | name: mpich-master 8 | role: master 9 | spec: 10 | containers: 11 | - name: mpich-master 12 | image: continuse/mpich:v3 13 | env: 14 | - name: MASTER 15 | value: "true" 16 | - name: ETCDCTL_PEERS 17 | value: "192.168.10.11:4001,192.168.10.12:4001,192.168.10.13:4001" 18 | - name: KUBE_NAMESPACE 19 | value: "default" 20 | - name: KUBE_METADATA_NAME_KEY 21 | value: "mpich-" 22 | ports: 23 | - containerPort: 22 24 | resources: 25 | limits: 26 | cpu: "0.1" 27 | volumeMounts: 28 | - mountPath: /share 29 | name: share 30 | volumes: 31 | - name: share 32 | hostPath: 33 | path : /continuse/mpi 34 | -------------------------------------------------------------------------------- /examples/kube-ui/kube-ui-rc.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: ReplicationController 3 | metadata: 4 | name: kube-ui-v3 5 | namespace: kube-system 6 | labels: 7 | k8s-app: kube-ui 8 | version: v3 9 | kubernetes.io/cluster-service: "true" 10 | spec: 11 | replicas: 1 12 | selector: 13 | k8s-app: kube-ui 14 | version: v3 15 | template: 16 | metadata: 17 | labels: 18 | k8s-app: kube-ui 19 | version: v3 20 | kubernetes.io/cluster-service: "true" 21 | spec: 22 | containers: 23 | - name: kube-ui 24 | image: gcr.io/google_containers/kube-ui:v3 25 | resources: 26 | limits: 27 | cpu: 100m 28 | memory: 50Mi 29 | ports: 30 | - containerPort: 8080 31 | livenessProbe: 32 | httpGet: 33 | path: / 34 | port: 8080 35 | initialDelaySeconds: 30 36 | timeoutSeconds: 5 37 | -------------------------------------------------------------------------------- /examples/redis-cluster/redis-cluster-manager.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: ReplicationController 3 | metadata: 4 | name: redis-cluster-manager 5 | # namespace: redis-cluster-system 6 | spec: 7 | replicas: 1 8 | selector: 9 | name: redis-cluster-manager 10 | template: 11 | metadata: 12 | labels: 13 | name: redis-cluster-manager 14 | role: cluster-cluster-manager 15 | spec: 16 | containers: 17 | - name: redis-cluster-manager 18 | image: continuse/redis:v3 19 | env: 20 | - name: ETCDCTL_PEERS 21 | value: "192.168.10.11:4001,192.168.10.12:4001,192.168.10.13:4001" 22 | - name: KUBE_NAMESPACE 23 | value: "default" 24 | - name: KUBE_LABEL 25 | value: "redis-cluster-worker" 26 | - name: REPLICAS 27 | value: "1" 28 | ports: 29 | - containerPort: 6379 30 | resources: 31 | limits: 32 | cpu: "0.1" 33 | -------------------------------------------------------------------------------- /examples/redis-cluster/images/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM ubuntu:14.04 2 | MAINTAINER Jaewoo Lee 3 | 4 | RUN apt-get update 5 | 6 | RUN apt-get install -y gcc make ruby expect telnet curl wget python-pip 7 | 8 | RUN wget http://download.redis.io/releases/redis-3.0.3.tar.gz 9 | RUN tar xvfz redis-3.0.3.tar.gz && cd redis-3.0.3 && make && make install 10 | 11 | # ETCD for python 12 | RUN cd /tmp && wget https://github.com/jplana/python-etcd/archive/0.4.1.tar.gz && \ 13 | tar xvfz 0.4.1.tar.gz && cd python-etcd-0.4.1 && pip install . 14 | 15 | # Python for Redis Cluster 16 | RUN wget https://github.com/ContinUSE/redis-py-cluster/archive/1.0.0.tar.gz \ 17 | && tar xvfz 1.0.0.tar.gz \ 18 | && cd redis-py-cluster-1.0.0 \ 19 | && pip install . 20 | 21 | RUN gem install redis 22 | 23 | COPY redis.conf /redis.conf 24 | COPY auto_config_redis_cluster.py /auto_config_redis_cluster.py 25 | COPY entrypoint.sh /entrypoint.sh 26 | 27 | EXPOSE 6379 28 | 29 | #CMD ["/usr/local/bin/redis-server", "/redis.conf"] 30 | CMD ["/entrypoint.sh"] 31 | -------------------------------------------------------------------------------- /continuse/kube/service/kube-apiserver.service: -------------------------------------------------------------------------------- 1 | [Unit] 2 | Description=Kubernetes API Server 3 | Documentation=https://github.com/GoogleCloudPlatform/kubernetes 4 | Requires=etcd2.service 5 | After=etcd2.service 6 | 7 | [Service] 8 | EnvironmentFile=/etc/environment 9 | ExecStartPre=/usr/bin/etcdctl set /continuse/kube/master ${COREOS_PUBLIC_IPV4} 10 | ExecStart=/continuse/kube/bin/kube-apiserver \ 11 | --service_account_key_file=/continuse/kube/bin/kube-serviceaccount.key \ 12 | --service_account_lookup=false \ 13 | --admission_control=NamespaceLifecycle,NamespaceAutoProvision,LimitRanger,SecurityContextDeny,ServiceAccount,ResourceQuota \ 14 | --runtime_config=api/v1 \ 15 | --allow_privileged=true \ 16 | --insecure_bind_address=0.0.0.0 \ 17 | --insecure_port=8080 \ 18 | --kubelet_https=true \ 19 | --secure_port=6443 \ 20 | --service-cluster-ip-range=10.100.0.0/16 \ 21 | --etcd_servers=http://127.0.0.1:2379 \ 22 | --public_address_override=${COREOS_PUBLIC_IPV4} \ 23 | --logtostderr=true 24 | Restart=always 25 | RestartSec=10 26 | 27 | [X-Fleet] 28 | MachineMetadata=role=controller 29 | -------------------------------------------------------------------------------- /examples/redis-cluster/images/entrypoint.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | telnet_status() 4 | { 5 | echo "b" | telnet -e "b" $1 $2 > /dev/null 2>&1 6 | if [ $? == 0 ] 7 | then 8 | return 0 9 | else 10 | return 1 11 | fi 12 | } 13 | 14 | wait_pods_update() 15 | { 16 | if [ "$ETCDCTL_PEERS" ]; then 17 | peers=`echo $ETCDCTL_PEERS | sed -e "s/,/ /g"` 18 | 19 | for i in $peers 20 | do 21 | read host port <<< $(echo $i | awk 'BEGIN {FS=":"; OFS =" "} {print $1,$2}') 22 | telnet_status $host $port 23 | if [ $? == 0 ] 24 | then 25 | curl -L http://$host:$port/v2/keys/registry/pods/$KUBE_NAMESPACE?wait=true\&recursive=true 26 | if [ $? == 0 ]; then 27 | break 28 | fi 29 | fi 30 | done 31 | fi 32 | } 33 | 34 | if [ "$WORKER" ]; then 35 | /usr/local/bin/redis-server /redis.conf 36 | fi 37 | 38 | while true; 39 | do 40 | #sleep 10 41 | 42 | /auto_config_redis_cluster.py 43 | 44 | wait_pods_update 45 | 46 | sleep 10 47 | done 48 | -------------------------------------------------------------------------------- /examples/mpich/images/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM ubuntu:14.04 2 | MAINTAINER Jaewoo Lee 3 | 4 | RUN apt-get update 5 | 6 | RUN apt-get install -y openssh-server 7 | RUN mkdir /var/run/sshd 8 | 9 | RUN echo 'root:root' |chpasswd 10 | 11 | RUN sed -ri 's/^PermitRootLogin\s+.*/PermitRootLogin yes/' /etc/ssh/sshd_config 12 | RUN sed -ri 's/UsePAM yes/#UsePAM yes/g' /etc/ssh/sshd_config 13 | 14 | RUN ssh-keygen -b 2048 -t rsa -f /root/.ssh/id_rsa -q -N "" 15 | RUN cd /root/.ssh && cp id_rsa.pub authorized_keys 16 | 17 | RUN apt-get install -y python-pip gfortran 18 | 19 | # ETCD for python 20 | RUN cd /tmp && wget https://github.com/jplana/python-etcd/archive/0.4.1.tar.gz && \ 21 | tar xvfz 0.4.1.tar.gz && cd python-etcd-0.4.1 && pip install . 22 | 23 | # MPICH3 Install 24 | RUN cd /tmp && wget http://www.mpich.org/static/downloads/3.1.4/mpich-3.1.4.tar.gz && \ 25 | tar xvfz mpich-3.1.4.tar.gz && cd mpich-3.1.4 && ./configure --prefix=/tmp/mpich && \ 26 | make && make VERBOSE=1 && make install 27 | 28 | RUN apt-get install -y expect telnet curl 29 | 30 | COPY entrypoint.sh /entrypoint.sh 31 | COPY pod_ip.py /pod_ip.py 32 | COPY known_hosts.py /known_hosts.py 33 | COPY auto_ssh.sh /auto_ssh.sh 34 | COPY bash_profile /root/.bash_profile 35 | 36 | ENV PATH $PATH:/tmp/mpich/bin 37 | 38 | EXPOSE 22 39 | 40 | CMD ["/entrypoint.sh"] 41 | -------------------------------------------------------------------------------- /examples/mpich/images/entrypoint.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | telnet_status() 4 | { 5 | echo "b" | telnet -e "b" $1 $2 > /dev/null 2>&1 6 | if [ $? == 0 ] 7 | then 8 | return 0 9 | else 10 | return 1 11 | fi 12 | } 13 | 14 | wait_pods_update() 15 | { 16 | if [ "$ETCDCTL_PEERS" ]; then 17 | peers=`echo $ETCDCTL_PEERS | sed -e "s/,/ /g"` 18 | 19 | for i in $peers 20 | do 21 | read host port <<< $(echo $i | awk 'BEGIN {FS=":"; OFS =" "} {print $1,$2}') 22 | telnet_status $host $port 23 | if [ $? == 0 ] 24 | then 25 | curl -L http://$host:$port/v2/keys/registry/pods/$KUBE_NAMESPACE?wait=true\&recursive=true 26 | if [ $? == 0 ]; then 27 | break 28 | fi 29 | fi 30 | done 31 | fi 32 | } 33 | 34 | if [ "$WORKER" ]; then 35 | /usr/sbin/sshd -D 36 | fi 37 | 38 | /usr/sbin/sshd -D & 39 | 40 | MACHINE_FILE=/root/host_file 41 | FIRST_HOSTS=/root/first_hosts 42 | 43 | touch $MACHINE_FILE 44 | touch $MACHINE_FILE.old 45 | 46 | while true; 47 | do 48 | /pod_ip.py > $MACHINE_FILE 49 | /known_hosts.py $MACHINE_FILE > $FIRST_HOSTS 50 | list=`cat $FIRST_HOSTS` 51 | for i in $list 52 | do 53 | /auto_ssh.sh $i 54 | done 55 | 56 | wait_pods_update 57 | 58 | cp $MACHINE_FILE $MACHINE_FILE.old 59 | 60 | done 61 | -------------------------------------------------------------------------------- /examples/mpich/images/pod_ip.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | 3 | import os, sys, json, etcd 4 | 5 | class JsonObject: 6 | def __init__(self, d): 7 | self.__dict__ = d 8 | 9 | class EtcdObject: 10 | def __init__(self, ip, port, namespace, key): 11 | self.ip = ip 12 | self.port = int(port) 13 | self.namespace = namespace 14 | self.key = key 15 | 16 | def getHosts(self): 17 | client = etcd.Client(host=self.ip, port=self.port) 18 | 19 | etcd_key = "/registry/pods/%s" % self.namespace 20 | 21 | try: 22 | r = client.read(etcd_key, recursive=True, sorted=True, timeout=1) 23 | for child in r.children: 24 | data = json.loads(child.value, object_hook=JsonObject) 25 | if self.key == None : 26 | if data.status.phase == "Running": 27 | print data.status.podIP 28 | else : 29 | if data.metadata.name[0:len(self.key)] == self.key and \ 30 | data.status.phase == "Running": 31 | print data.status.podIP 32 | 33 | return 0 34 | except Exception: 35 | return -1 36 | 37 | etcdctl_peers = os.getenv('ETCDCTL_PEERS') 38 | kube_namespace = os.getenv('KUBE_NAMESPACE') 39 | kube_metadata_name_key = os.getenv('KUBE_METADATA_NAME_KEY') 40 | 41 | if isinstance(etcdctl_peers, str) : 42 | etcd_ip_port = etcdctl_peers.split(',') 43 | 44 | for ctrl_ip_port in etcd_ip_port : 45 | ip, port = ctrl_ip_port.split(':') 46 | a = EtcdObject(ip, port, kube_namespace, kube_metadata_name_key) 47 | if a.getHosts() == 0 : 48 | exit(0) 49 | -------------------------------------------------------------------------------- /examples/mpich/README.md: -------------------------------------------------------------------------------- 1 | # MPICH Multinode Cluster 2 | Describe how to make docker image and config Kubernetes cluster for MPI cluster. It is easy to node add/remove using ReplicationController function of Kubernetes. 3 | This is show you a basic MPICH cluster configuration and MPI hello world application on Kubernetes cluster using MPICH3 (specifically 3.1.4). 4 | 5 | ## Reliable, Scalable MPICH cluster on Kubernetes 6 | The followings are multi-node MPICH cluster configuration on Kubernetes. It deploys a master with replicated workers. 7 | 8 | ### Master Pods 9 | Create this master as follows: 10 | ``` 11 | kubectl create -f mpich-master.yaml 12 | ``` 13 | 14 | ### Replicated mpich servers 15 | Create this worker as follows: 16 | ``` 17 | kubectl create -f mpich-controller.yaml 18 | ``` 19 | 20 | ### Scale our replicated pods 21 | we will add more replicas for worker: 22 | ``` 23 | kubectl scale rc mpich-worker --replicas=3 24 | ``` 25 | 26 | ### mpich master service 27 | Create the service by running: Using NodePort as 32000 tcp port 28 | ``` 29 | kubectl create -f mpich-service.yaml 30 | ``` 31 | 32 | ### Connect to master & Basic test 33 | ``` 34 | $ ssh root@192.168.10.71 -p 32000 (Any one of kube cluster nodes) 35 | root@192.168.10.71's password: (root apsswrod as 'root') 36 | Last login: Thu Aug 6 04:11:46 2015 from mpich-master 37 | root@mpich-master:~# mpirun -f host_file -n 3 hostname 38 | mpich-master 39 | mpich-worker-1p1ko 40 | mpich-worker-r4j6z 41 | root@mpich-master:~# 42 | ``` 43 | 44 | ### Conclusion 45 | Now We have a reliable, scalable MPICH cluster installation. By scaling the replication controller for mpich worker instances, we can increase or decrease the number of mpich-worker instances and auto update for host file on master. If you want to test for sample MPI program, visit to http://mpitutorial.com/tutorials/mpi-hello-world/. (MPICH package installed /tmp/mpich) 46 | 47 | -------------------------------------------------------------------------------- /examples/mpich/images/README.md: -------------------------------------------------------------------------------- 1 | # MPICH Multinode Cluster 2 | Describe how to make docker image and config Kubernetes cluster for MPI cluster. It is easy to node add/remove using ReplicationController function of Kubernetes. 3 | This is show you a basic MPICH cluster configuration and MPI hello world application on Kubernetes cluster using MPICH3 (specifically 3.1.4). 4 | 5 | ## Reliable, Scalable MPICH cluster on Kubernetes 6 | The followings are multi-node MPICH cluster configuration on Kubernetes. It deploys a master with replicated workers. 7 | 8 | ### Master Pods 9 | Create this master as follows: 10 | ``` 11 | kubectl create -f mpich-master.yaml 12 | ``` 13 | 14 | ### Replicated mpich servers 15 | Create this worker as follows: 16 | ``` 17 | kubectl create -f mpich-controller.yaml 18 | ``` 19 | 20 | ### Scale our replicated pods 21 | we will add more replicas for worker: 22 | ``` 23 | kubectl scale rc mpich-worker --replicas=3 24 | ``` 25 | 26 | ### mpich master service 27 | Create the service by running: Using NodePort as 32000 tcp port 28 | ``` 29 | kubectl create -f mpich-service.yaml 30 | ``` 31 | 32 | ### Connect to master & Basic test 33 | ``` 34 | $ ssh root@192.168.10.71 -p 32000 (Any one of kube cluster nodes) 35 | root@192.168.10.71's password: (root apsswrod as 'root') 36 | Last login: Thu Aug 6 04:11:46 2015 from mpich-master 37 | root@mpich-master:~# mpirun -f host_file -n 3 hostname 38 | mpich-master 39 | mpich-worker-1p1ko 40 | mpich-worker-r4j6z 41 | root@mpich-master:~# 42 | ``` 43 | 44 | ### Conclusion 45 | Now We have a reliable, scalable MPICH cluster installation. By scaling the replication controller for mpich worker instances, we can increase or decrease the number of mpich-worker instances and auto update for host file on master. If you want to test for sample MPI program, visit to http://mpitutorial.com/tutorials/mpi-hello-world/. (MPICH package installed /tmp/mpich) 46 | 47 | -------------------------------------------------------------------------------- /kube/user-data: -------------------------------------------------------------------------------- 1 | #cloud-config 2 | 3 | coreos: 4 | update: 5 | reboot-strategy: off 6 | etcd2: 7 | initial-cluster: controller-01=http://192.168.10.11:2380,controller-02=http://192.168.10.12:2380,controller-03=http://192.168.10.13:2380 8 | listen-client-urls: http://0.0.0.0:2379,http://0.0.0.0:4001 9 | #listen-peer-urls: http://$private_ipv4:2380,http://$private_ipv4:7001 10 | #advertise-client-urls: http://$public_ipv4:2379 11 | #initial-advertise-peer-urls: http://$private_ipv4:2380 12 | proxy: on 13 | fleet: 14 | public-ip: $public_ipv4 15 | metadata: role=kube 16 | flannel: 17 | interface: $public_ipv4 18 | units: 19 | - name: flanneld.service 20 | command: start 21 | drop-ins: 22 | - name: 50-network-config.conf 23 | content: | 24 | [Unit] 25 | Requires=etcd2.service 26 | [Service] 27 | ExecStartPre=/usr/bin/etcdctl set /coreos.com/network/config '{"Network":"10.244.0.0/16", "Backend": {"Type": "vxlan"}}' 28 | - name: fleet.service 29 | command: start 30 | - name: docker.service 31 | command: start 32 | drop-ins: 33 | - name: 50-insecure-registry.conf 34 | content: | 35 | [Unit] 36 | Requires=flanneld.service 37 | After=flanneld.service 38 | [Service] 39 | Environment=DOCKER_OPTS='--insecure-registry="0.0.0.0/0"' 40 | 41 | - name: settimezone.service 42 | command: start 43 | content: | 44 | [Unit] 45 | Description=Set the timezone 46 | 47 | [Service] 48 | ExecStart=/usr/bin/timedatectl set-timezone Asia/Seoul 49 | RemainAfterExit=yes 50 | Type=oneshot 51 | 52 | write_files: 53 | - path: /home/core/.bash_profile 54 | permissions: '0644' 55 | content: | 56 | #!/bin/bash 57 | export PATH=$PATH:/continuse/kube/bin 58 | -------------------------------------------------------------------------------- /controller/user-data: -------------------------------------------------------------------------------- 1 | #cloud-config 2 | 3 | coreos: 4 | update: 5 | reboot-strategy: off 6 | etcd2: 7 | #generate a new token for each unique cluster from https://discovery.etcd.io/new 8 | discovery: 9 | # multi-region and multi-cloud deployments need to use $public_ipv4 10 | advertise-client-urls: http://$public_ipv4:2379 11 | initial-advertise-peer-urls: http://$private_ipv4:2380 12 | listen-client-urls: http://0.0.0.0:2379,http://0.0.0.0:4001 13 | listen-peer-urls: http://$private_ipv4:2380,http://$private_ipv4:7001 14 | fleet: 15 | public-ip: $public_ipv4 16 | metadata: role=controller 17 | flannel: 18 | interface: $public_ipv4 19 | units: 20 | - name: etcd2.service 21 | command: start 22 | - name: fleet.service 23 | command: start 24 | - name: docker-tcp.socket 25 | command: start 26 | enable: true 27 | content: | 28 | [Unit] 29 | Description=Docker Socket for the API 30 | 31 | [Socket] 32 | ListenStream=2375 33 | Service=docker.service 34 | BindIPv6Only=both 35 | 36 | [Install] 37 | WantedBy=sockets.target 38 | - name: flanneld.service 39 | command: start 40 | drop-ins: 41 | - name: 50-network-config.conf 42 | content: | 43 | [Unit] 44 | Requires=etcd2.service 45 | [Service] 46 | ExecStartPre=/usr/bin/etcdctl set /coreos.com/network/config '{"Network": "10.244.0.0/16", "Backend": {"Type": "vxlan"}}' 47 | - name: fleet.service 48 | command: start 49 | - name: docker.service 50 | command: start 51 | drop-ins: 52 | - name: 50-insecure-registry.conf 53 | content: | 54 | [Unit] 55 | Requires=flanneld.service 56 | After=flanneld.service 57 | [Service] 58 | Environment=DOCKER_OPTS='--insecure-registry="0.0.0.0/0"' 59 | - name: settimezone.service 60 | command: start 61 | content: | 62 | [Unit] 63 | Description=Set the timezone 64 | 65 | [Service] 66 | ExecStart=/usr/bin/timedatectl set-timezone Asia/Seoul 67 | RemainAfterExit=yes 68 | Type=oneshot 69 | 70 | write_files: 71 | - path: /home/core/.bash_profile 72 | permissions: '0644' 73 | content: | 74 | #!/bin/bash 75 | export PATH=$PATH:/continuse/kube/bin 76 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # kubernetes-coreos-cluster-hands-on 2 | [Kubernetes](https://github.com/GoogleCloudPlatform/kubernetes) 3 | cluster setup with [Vagrant](https://www.vagrantup.com) and 4 | [CoreOS](https://coreos.com). 5 | 6 | ## Pre-requisites 7 | 8 | * [Vagrant](https://www.vagrantup.com) 9 | * [Virtualbox](https://www.virtualbox.org) 10 | * For more information about virtualbox setup, see the test environments section in https://github.com/ContinUSE/openstack-on-coreos 11 | 12 | ## Controller and Kube Cluster Setup 13 | 14 | ### Download Scripts 15 | Controller Cluster has 3 nodes as controller-01/controller-02/controller-03, and Kube Cluster has 3 nodes as kube-01-01, kube-02, kube-03. 16 | ``` 17 | $ ls 18 | README.md* continuse/ controller/ examples/ kube/ 19 | ``` 20 | 21 | Modify in controller/Vagrantfile and kube/Vagrantfile for NFS directory (ex: Local directory path is ~/kubernetes-cluster/continuse) 22 | ``` 23 | config.vm.synced_folder "~/kubernetes-cluster/continuse", "/continuse", id: "root", :nfs => true, :mount_options => ["nolock,vers=3,udp"], :map_uid => 0, :map_gid => 0 24 | ``` 25 | 26 | ### Controller / Kube Cluster Building 27 | ``` 28 | $ cd controller 29 | $ vagrant up 30 | ............ 31 | ........... 32 | $ cd kube 33 | $ vagrant up 34 | ``` 35 | 36 | ### Controller Cluster login & Kubernetes Binary file copy 37 | ``` 38 | $ cd controller 39 | $ vagrant ssh controller-01 40 | 41 | On controller-01 42 | $ cd /continuse/kube/bin 43 | $ wget https://github.com/ContinUSE/kubernetes-coreos-cluster/releases/download/v1.1.2/kube_1.1.2.tgz 44 | $ tar xvfz kube_1.1.2.tgz 45 | ``` 46 | 47 | ### Make Service Account Key file 48 | ``` 49 | On controller-01 50 | $ cd /continuse/kube/bin 51 | $ openssl genrsa -out kube-serviceaccount.key 2048 52 | ``` 53 | 54 | ### Kubernetes Service Start 55 | ``` 56 | $ cd /continuse/kube/service 57 | $ fleetctl start kube-apiserver.service 58 | $ fleetctl start kube-controller-manager.service 59 | $ fleetctl start kube-scheduler.service 60 | $ fleetctl start kube-kubelet.service 61 | $ fleetctl start kube-proxy.service 62 | ``` 63 | 64 | ### Kubernetes Service for GUI Interface 65 | ``` 66 | $ cd /continuse/kube/examples/kube-ui 67 | $ kubectl create -f kube-system.yaml 68 | $ kubectl create -f kube-ui-rc.yaml --namespace=kube-system 69 | $ kubectl create -f kube-ui-svc.yaml --namespace=kube-system 70 | ``` 71 | 72 | ### Kubernetes UI -- Web-based 73 | ``` 74 | http://192.168.10.12:8080/ui/ IP address of Kubernetes API Server Running 75 | ``` 76 | 77 | #### cAdvisor -- Web-based monitoring 78 | ``` 79 | http://192.168.10.71:4194/containers/ Each Kube Cluster node 80 | ``` 81 | 82 | **KUBERNETES_VERSION** v1..0.1 83 | 84 | ### Usage 85 | 86 | If you want to test something examples, start with [Kubernetes examples] 87 | (https://github.com/GoogleCloudPlatform/kubernetes/blob/master/examples/). 88 | And I am trying yo make examples. See with [My Examples] (https://github.com/ContinUSE/kubernetes-coreos-cluster/tree/master/examples). 89 | 90 | -------------------------------------------------------------------------------- /kube/config.rb: -------------------------------------------------------------------------------- 1 | # Size of the CoreOS cluster created by Vagrant 2 | $num_instances=3 3 | 4 | # Used to fetch a new discovery token for a cluster of size $num_instances 5 | $new_discovery_url="https://discovery.etcd.io/new?size=#{$num_instances}" 6 | 7 | # To automatically replace the discovery token on 'vagrant up', uncomment 8 | # the lines below: 9 | # 10 | #if File.exists?('user-data') && ARGV[0].eql?('up') 11 | # require 'open-uri' 12 | # require 'yaml' 13 | # 14 | # token = open($new_discovery_url).read 15 | # 16 | # data = YAML.load(IO.readlines('user-data')[1..-1].join) 17 | # if data['coreos'].key? 'etcd' 18 | # data['coreos']['etcd']['discovery'] = token 19 | # end 20 | # if data['coreos'].key? 'etcd2' 21 | # data['coreos']['etcd2']['discovery'] = token 22 | # end 23 | # 24 | # yaml = YAML.dump(data) 25 | # File.open('user-data', 'w') { |file| file.write("#cloud-config\n\n#{yaml}") } 26 | #end 27 | # 28 | 29 | # 30 | # coreos-vagrant is configured through a series of configuration 31 | # options (global ruby variables) which are detailed below. To modify 32 | # these options, first copy this file to "config.rb". Then simply 33 | # uncomment the necessary lines, leaving the $, and replace everything 34 | # after the equals sign.. 35 | 36 | # Change basename of the VM 37 | # The default value is "core", which results in VMs named starting with 38 | # "core-01" through to "core-${num_instances}". 39 | $instance_name_prefix="kube" 40 | 41 | # Official CoreOS channel from which updates should be downloaded 42 | $update_channel='stable' 43 | 44 | # Log the serial consoles of CoreOS VMs to log/ 45 | # Enable by setting value to true, disable with false 46 | # WARNING: Serial logging is known to result in extremely high CPU usage with 47 | # VirtualBox, so should only be used in debugging situations 48 | #$enable_serial_logging=false 49 | 50 | # Enable port forwarding of Docker TCP socket 51 | # Set to the TCP port you want exposed on the *host* machine, default is 2375 52 | # If 2375 is used, Vagrant will auto-increment (e.g. in the case of $num_instances > 1) 53 | # You can then use the docker tool locally by setting the following env var: 54 | # export DOCKER_HOST='tcp://127.0.0.1:2375' 55 | #$expose_docker_tcp=2375 56 | 57 | # Enable NFS sharing of your home directory ($HOME) to CoreOS 58 | # It will be mounted at the same path in the VM as on the host. 59 | # Example: /Users/foobar -> /Users/foobar 60 | #$share_home=false 61 | 62 | # Customize VMs 63 | #$vm_gui = false 64 | $vm_memory = 2048 65 | $vm_cpus = 1 66 | 67 | # Share additional folders to the CoreOS VMs 68 | # For example, 69 | # $shared_folders = {'/path/on/host' => '/path/on/guest', '/home/foo/app' => '/app'} 70 | # or, to map host folders to guest folders of the same name, 71 | # $shared_folders = Hash[*['/home/foo/app1', '/home/foo/app2'].map{|d| [d, d]}.flatten] 72 | #$shared_folders = {} 73 | 74 | # Enable port forwarding from guest(s) to host machine, syntax is: { 80 => 8080 }, auto correction is enabled by default. 75 | #$forwarded_ports = {} 76 | -------------------------------------------------------------------------------- /controller/config.rb: -------------------------------------------------------------------------------- 1 | # Size of the CoreOS cluster created by Vagrant 2 | $num_instances=3 3 | 4 | # Used to fetch a new discovery token for a cluster of size $num_instances 5 | $new_discovery_url="https://discovery.etcd.io/new?size=#{$num_instances}" 6 | 7 | # To automatically replace the discovery token on 'vagrant up', uncomment 8 | # the lines below: 9 | # 10 | #if File.exists?('user-data') && ARGV[0].eql?('up') 11 | # require 'open-uri' 12 | # require 'yaml' 13 | # 14 | # token = open($new_discovery_url).read 15 | # 16 | # data = YAML.load(IO.readlines('user-data')[1..-1].join) 17 | # if data['coreos'].key? 'etcd' 18 | # data['coreos']['etcd']['discovery'] = token 19 | # end 20 | # if data['coreos'].key? 'etcd2' 21 | # data['coreos']['etcd2']['discovery'] = token 22 | # end 23 | # 24 | # yaml = YAML.dump(data) 25 | # File.open('user-data', 'w') { |file| file.write("#cloud-config\n\n#{yaml}") } 26 | #end 27 | # 28 | 29 | # 30 | # coreos-vagrant is configured through a series of configuration 31 | # options (global ruby variables) which are detailed below. To modify 32 | # these options, first copy this file to "config.rb". Then simply 33 | # uncomment the necessary lines, leaving the $, and replace everything 34 | # after the equals sign.. 35 | 36 | # Change basename of the VM 37 | # The default value is "core", which results in VMs named starting with 38 | # "core-01" through to "core-${num_instances}". 39 | $instance_name_prefix="controller" 40 | 41 | # Official CoreOS channel from which updates should be downloaded 42 | $update_channel='stable' 43 | 44 | # Log the serial consoles of CoreOS VMs to log/ 45 | # Enable by setting value to true, disable with false 46 | # WARNING: Serial logging is known to result in extremely high CPU usage with 47 | # VirtualBox, so should only be used in debugging situations 48 | #$enable_serial_logging=false 49 | 50 | # Enable port forwarding of Docker TCP socket 51 | # Set to the TCP port you want exposed on the *host* machine, default is 2375 52 | # If 2375 is used, Vagrant will auto-increment (e.g. in the case of $num_instances > 1) 53 | # You can then use the docker tool locally by setting the following env var: 54 | # export DOCKER_HOST='tcp://127.0.0.1:2375' 55 | #$expose_docker_tcp=2375 56 | 57 | # Enable NFS sharing of your home directory ($HOME) to CoreOS 58 | # It will be mounted at the same path in the VM as on the host. 59 | # Example: /Users/foobar -> /Users/foobar 60 | #$share_home=false 61 | 62 | # Customize VMs 63 | #$vm_gui = false 64 | $vm_memory = 3072 65 | #$vm_cpus = 1 66 | 67 | # Share additional folders to the CoreOS VMs 68 | # For example, 69 | # $shared_folders = {'/path/on/host' => '/path/on/guest', '/home/foo/app' => '/app'} 70 | # or, to map host folders to guest folders of the same name, 71 | # $shared_folders = Hash[*['/home/foo/app1', '/home/foo/app2'].map{|d| [d, d]}.flatten] 72 | #$shared_folders = {} 73 | 74 | # Enable port forwarding from guest(s) to host machine, syntax is: { 80 => 8080 }, auto correction is enabled by default. 75 | #$forwarded_ports = {} 76 | -------------------------------------------------------------------------------- /examples/redis-cluster/README.md: -------------------------------------------------------------------------------- 1 | # Redis Cluster 2 | This example provides the ability to automatically process such as initial redis cluster configuration, node expansion and node failover operation on Kubernetes environment. 3 | 4 | ## Reliable, Scalable Redis Cluster on Kubernetes 5 | The followings are multi-node Redis Cluster Usage. This example that I tried to apply on a trial basis. 6 | 7 | ### Redis Cluster Manager Pods 8 | Create redis cluster manager as follows: 9 | ``` 10 | $ kubectl create -f redis-cluster-manager.yaml 11 | replicationcontrollers/redis-cluster-manager 12 | ``` 13 | 14 | Log view for status check 15 | ``` 16 | $ kubectl get pods 17 | NAME READY STATUS RESTARTS AGE 18 | redis-cluster-manager-8eo9n 1/1 Running 0 1m 19 | $ kubectl logs -f redis-cluster-manager-8eo9n 20 | Getting information for redis cluster........... 21 | ......... 22 | ........ 23 | ........ 24 | The redis cluster need at least 6 nodes. 25 | % Total % Received % Xferd Average Speed Time Time Time Current 26 | Dload Upload Total Spent Left Speed 27 | ``` 28 | 29 | ### Replicated redis cluster workers 30 | Create redis cluster worker as follows: 31 | ``` 32 | $ kubectl create -f redis-cluster-worker.yaml 33 | replicationcontrollers/redis-cluster-worker 34 | ``` 35 | You can view status check for redis-cluster-manager above "kubectl logs...." command. 36 | 37 | ### Service creation for redis-cluster-worker pods 38 | ``` 39 | $ kubectl create -f redis-cluster-worker-service.yaml 40 | You have exposed your service on an external port on all nodes in your 41 | cluster. If you want to expose this service to the external internet, you may 42 | need to set up firewall rules for the service port(s) (tcp:32100) to serve traffic. 43 | 44 | See http://releases.k8s.io/HEAD/docs/services-firewalls.md for more details. 45 | services/redis-cluster-worker 46 | ``` 47 | And redis-cli command Usage: 48 | The host IP address is one of node of kube, and port is 32100. (This port number defined in redis-cluster-worker-service.yaml) 49 | ``` 50 | $ sudo docker run --rm -it redis redis-cli -c -h 192.168.10.71 -p 32100 51 | 192.168.10.71:32100> cluster info 52 | cluster_state:ok 53 | cluster_slots_assigned:16384 54 | cluster_slots_ok:16384 55 | cluster_slots_pfail:0 56 | cluster_slots_fail:0 57 | cluster_known_nodes:6 58 | cluster_size:3 59 | cluster_current_epoch:6 60 | cluster_my_epoch:3 61 | cluster_stats_messages_sent:2121 62 | cluster_stats_messages_received:2121 63 | 192.168.10.71:32100> 64 | ``` 65 | 66 | ### Node Add (1 master / 1 slave) 67 | ``` 68 | $ kubectl scale rc redis-cluster-worker --replicas=8 69 | scaled 70 | ``` 71 | See log view for ALL procedure such as node add and data reshard. 72 | 73 | # Node Failure Test 74 | kube-02 fail test 75 | ``` 76 | vagrant halt kube-02 77 | ``` 78 | It takes about five minutes for the automatic recovery. Checking the log view of redis-cluster-manager. 79 | 80 | ### Issue 81 | The system is still unstable after recovery of failure. 82 | -------------------------------------------------------------------------------- /examples/redis-cluster/images/README.md: -------------------------------------------------------------------------------- 1 | # Redis Cluster 2 | This example provides the ability to automatically process such as initial redis cluster configuration, node expansion and node failover operation on Kubernetes environment. 3 | 4 | ## Reliable, Scalable Redis Cluster on Kubernetes 5 | The followings are multi-node Redis Cluster Usage. This example that I tried to apply on a trial basis. 6 | 7 | ### Redis Cluster Manager Pods 8 | Create redis cluster manager as follows: 9 | ``` 10 | $ kubectl create -f redis-cluster-manager.yaml 11 | replicationcontrollers/redis-cluster-manager 12 | ``` 13 | 14 | Log view for status check 15 | ``` 16 | $ kubectl get pods 17 | NAME READY STATUS RESTARTS AGE 18 | redis-cluster-manager-8eo9n 1/1 Running 0 1m 19 | $ kubectl logs -f redis-cluster-manager-8eo9n 20 | Getting information for redis cluster........... 21 | ......... 22 | ........ 23 | ........ 24 | The redis cluster need at least 6 nodes. 25 | % Total % Received % Xferd Average Speed Time Time Time Current 26 | Dload Upload Total Spent Left Speed 27 | ``` 28 | 29 | ### Replicated redis cluster workers 30 | Create redis cluster worker as follows: 31 | ``` 32 | $ kubectl create -f redis-cluster-worker.yaml 33 | replicationcontrollers/redis-cluster-worker 34 | ``` 35 | You can view status check for redis-cluster-manager above "kubectl logs...." command. 36 | 37 | ### Service creation for redis-cluster-worker pods 38 | ``` 39 | $ kubectl create -f redis-cluster-worker-service.yaml 40 | You have exposed your service on an external port on all nodes in your 41 | cluster. If you want to expose this service to the external internet, you may 42 | need to set up firewall rules for the service port(s) (tcp:32100) to serve traffic. 43 | 44 | See http://releases.k8s.io/HEAD/docs/services-firewalls.md for more details. 45 | services/redis-cluster-worker 46 | ``` 47 | And redis-cli command Usage: 48 | The host IP address is one of node of kube, and port is 32100. (This port number defined in redis-cluster-worker-service.yaml) 49 | ``` 50 | $ sudo docker run --rm -it redis redis-cli -c -h 192.168.10.71 -p 32100 51 | 192.168.10.71:32100> cluster info 52 | cluster_state:ok 53 | cluster_slots_assigned:16384 54 | cluster_slots_ok:16384 55 | cluster_slots_pfail:0 56 | cluster_slots_fail:0 57 | cluster_known_nodes:6 58 | cluster_size:3 59 | cluster_current_epoch:6 60 | cluster_my_epoch:3 61 | cluster_stats_messages_sent:2121 62 | cluster_stats_messages_received:2121 63 | 192.168.10.71:32100> 64 | ``` 65 | 66 | ### Node Add (1 master / 1 slave) 67 | ``` 68 | $ kubectl scale rc redis-cluster-worker --replicas=8 69 | scaled 70 | ``` 71 | See log view for ALL procedure such as node add and data reshard. 72 | 73 | # Node Failure Test 74 | kube-02 fail test 75 | ``` 76 | vagrant halt kube-02 77 | ``` 78 | It takes about five minutes for the automatic recovery. Checking the log view of redis-cluster-manager. 79 | 80 | ### Issue 81 | The system is still unstable after recovery of failure. 82 | -------------------------------------------------------------------------------- /continuse/kube/bin/chk_master.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | ETCDCTL_PEERS=192.168.10.11:4001,192.168.10.12:4001,192.168.10.13:4001 4 | ping_status() 5 | { 6 | ping -q -c2 $1 > /dev/null 2>&1 7 | if [ $? == 0 ] 8 | then 9 | return 0 10 | else 11 | return 1 12 | fi 13 | } 14 | 15 | correct_ip_chk() 16 | { 17 | re='[0-9]+([.][0-9]+)?$' 18 | if ! [[ $master_ip =~ $re ]] 19 | then 20 | return 1 21 | else 22 | return 0 23 | fi 24 | } 25 | 26 | get_master_info() 27 | { 28 | master_ip='' 29 | if [ "$ETCDCTL_PEERS" ]; then 30 | peers=`echo $ETCDCTL_PEERS | sed -e "s/,/ /g"` 31 | 32 | for i in $peers 33 | do 34 | read host port <<< $(echo $i | awk 'BEGIN {FS=":"; OFS =" "} {print $1,$2}') 35 | ping_status $host 36 | if [ $? == 0 ] 37 | then 38 | values=`curl -L http://$host:$port/v2/keys/continuse/kube/master 2> /dev/null` 39 | master_ip=`echo $values | awk -F ',' '{print $3}' | awk -F "\"" '{print $4}'` 40 | break 41 | fi 42 | done 43 | fi 44 | } 45 | 46 | wait_master_info() 47 | { 48 | master_ip='' 49 | if [ "$ETCDCTL_PEERS" ]; then 50 | peers=`echo $ETCDCTL_PEERS | sed -e "s/,/ /g"` 51 | 52 | for i in $peers 53 | do 54 | read host port <<< $(echo $i | awk 'BEGIN {FS=":"; OFS =" "} {print $1,$2}') 55 | ping_status $host 56 | if [ $? == 0 ] 57 | then 58 | values=`curl -L http://$host:$port/v2/keys/continuse/kube/master?wait=true 2> /dev/null` 59 | if [ $? == 0 ]; then 60 | master_ip=`echo $values | awk -F ',' '{print $3}' | awk -F "\"" '{print $4}'` 61 | break 62 | fi 63 | fi 64 | done 65 | fi 66 | } 67 | 68 | exec_command() 69 | { 70 | if [ $1 == 'kubelet' ];then 71 | /usr/bin/pkill kubelet 72 | /continuse/kube/bin/kubelet \ 73 | --address=0.0.0.0 \ 74 | --port=10250 \ 75 | --hostname_override=$2 \ 76 | --api_servers=$master_ip:8080 \ 77 | --allow_privileged=true \ 78 | --logtostderr=true \ 79 | --cadvisor_port=4194 \ 80 | --healthz_bind_address=0.0.0.0 \ 81 | --healthz_port=10248 & 82 | fi 83 | 84 | if [ $1 == 'proxy' ]; then 85 | /usr/bin/pkill -f kube-proxy 86 | /continuse/kube/bin/kube-proxy \ 87 | --master=$master_ip:8080 \ 88 | --logtostderr=true & 89 | fi 90 | } 91 | 92 | # Usage : chk_master.sh {kubelet or proxy} $COREOS_PUBLIC_IPV4) 93 | while true; do 94 | echo 'Getting master ip address from etcd....' 95 | get_master_info 96 | 97 | # check for the master ip address 98 | correct_ip_chk 99 | if [ $? == 0 ] 100 | then 101 | break 102 | fi 103 | 104 | # If do not getting correct ip address, retry after sleep time. 105 | sleep 10 106 | done 107 | 108 | while true; do 109 | exec_command $1 $2 110 | 111 | wait_master_info 112 | done 113 | 114 | -------------------------------------------------------------------------------- /kube/Vagrantfile: -------------------------------------------------------------------------------- 1 | # -*- mode: ruby -*- 2 | # # vi: set ft=ruby : 3 | 4 | require 'fileutils' 5 | 6 | Vagrant.require_version ">= 1.6.0" 7 | 8 | CLOUD_CONFIG_PATH = File.join(File.dirname(__FILE__), "user-data") 9 | CONFIG = File.join(File.dirname(__FILE__), "config.rb") 10 | 11 | # Defaults for config options defined in CONFIG 12 | $num_instances = 1 13 | $instance_name_prefix = "core" 14 | $update_channel = "alpha" 15 | $enable_serial_logging = false 16 | $share_home = false 17 | $vm_gui = false 18 | $vm_memory = 1024 19 | $vm_cpus = 1 20 | $shared_folders = {} 21 | $forwarded_ports = {} 22 | 23 | # Attempt to apply the deprecated environment variable NUM_INSTANCES to 24 | # $num_instances while allowing config.rb to override it 25 | if ENV["NUM_INSTANCES"].to_i > 0 && ENV["NUM_INSTANCES"] 26 | $num_instances = ENV["NUM_INSTANCES"].to_i 27 | end 28 | 29 | if File.exist?(CONFIG) 30 | require CONFIG 31 | end 32 | 33 | # Use old vb_xxx config variables when set 34 | def vm_gui 35 | $vb_gui.nil? ? $vm_gui : $vb_gui 36 | end 37 | 38 | def vm_memory 39 | $vb_memory.nil? ? $vm_memory : $vb_memory 40 | end 41 | 42 | def vm_cpus 43 | $vb_cpus.nil? ? $vm_cpus : $vb_cpus 44 | end 45 | 46 | Vagrant.configure("2") do |config| 47 | # always use Vagrants insecure key 48 | config.ssh.insert_key = false 49 | 50 | config.vm.box = "coreos-%s" % $update_channel 51 | config.vm.box_version = ">= 308.0.1" 52 | config.vm.box_url = "http://%s.release.core-os.net/amd64-usr/current/coreos_production_vagrant.json" % $update_channel 53 | 54 | ["vmware_fusion", "vmware_workstation"].each do |vmware| 55 | config.vm.provider vmware do |v, override| 56 | override.vm.box_url = "http://%s.release.core-os.net/amd64-usr/current/coreos_production_vagrant_vmware_fusion.json" % $update_channel 57 | end 58 | end 59 | 60 | config.vm.provider :virtualbox do |v| 61 | # On VirtualBox, we don't have guest additions or a functional vboxsf 62 | # in CoreOS, so tell Vagrant that so it can be smarter. 63 | v.check_guest_additions = false 64 | v.functional_vboxsf = false 65 | end 66 | 67 | # plugin conflict 68 | if Vagrant.has_plugin?("vagrant-vbguest") then 69 | config.vbguest.auto_update = false 70 | end 71 | 72 | (1..$num_instances).each do |i| 73 | config.vm.define vm_name = "%s-%02d" % [$instance_name_prefix, i] do |config| 74 | config.vm.hostname = vm_name 75 | 76 | if $enable_serial_logging 77 | logdir = File.join(File.dirname(__FILE__), "log") 78 | FileUtils.mkdir_p(logdir) 79 | 80 | serialFile = File.join(logdir, "%s-serial.txt" % vm_name) 81 | FileUtils.touch(serialFile) 82 | 83 | ["vmware_fusion", "vmware_workstation"].each do |vmware| 84 | config.vm.provider vmware do |v, override| 85 | v.vmx["serial0.present"] = "TRUE" 86 | v.vmx["serial0.fileType"] = "file" 87 | v.vmx["serial0.fileName"] = serialFile 88 | v.vmx["serial0.tryNoRxLoss"] = "FALSE" 89 | end 90 | end 91 | 92 | config.vm.provider :virtualbox do |vb, override| 93 | vb.customize ["modifyvm", :id, "--uart1", "0x3F8", "4"] 94 | vb.customize ["modifyvm", :id, "--uartmode1", serialFile] 95 | end 96 | end 97 | 98 | if $expose_docker_tcp 99 | config.vm.network "forwarded_port", guest: 2375, host: ($expose_docker_tcp + i - 1), auto_correct: true 100 | end 101 | 102 | $forwarded_ports.each do |guest, host| 103 | config.vm.network "forwarded_port", guest: guest, host: host, auto_correct: true 104 | end 105 | 106 | ["vmware_fusion", "vmware_workstation"].each do |vmware| 107 | config.vm.provider vmware do |v| 108 | v.gui = vm_gui 109 | v.vmx['memsize'] = vm_memory 110 | v.vmx['numvcpus'] = vm_cpus 111 | end 112 | end 113 | 114 | config.vm.provider :virtualbox do |vb| 115 | vb.gui = vm_gui 116 | vb.memory = vm_memory 117 | vb.cpus = vm_cpus 118 | end 119 | 120 | config.vm.network :private_network, ip: "10.0.5.#{i+70}", :adapter => 2 121 | config.vm.network :private_network, ip: "172.16.0.#{i+70}", :netmask => "255.255.0.0", :adapter => 3 122 | config.vm.network :private_network, ip: "192.168.10.#{i+70}", :netmask => "255.255.255.0", :adapter => 4 123 | 124 | # Comment below to disble NFS for sharing the host machine into the coreos-vagrant VM. 125 | config.vm.synced_folder "~/test/continuse", "/continuse", id: "root", :nfs => true, :mount_options => ["nolock,vers=3,udp"], :map_uid => 0, :map_gid => 0 126 | 127 | config.vm.provider :virtualbox do |vb| 128 | vb.customize ["modifyvm", :id, "--nic2", "NatNetwork"] 129 | vb.customize ["modifyvm", :id, "--nicpromisc1", "allow-all"] 130 | vb.customize ["modifyvm", :id, "--nicpromisc2", "allow-all"] 131 | vb.customize ["modifyvm", :id, "--nicpromisc3", "allow-all"] 132 | vb.customize ["modifyvm", :id, "--nicpromisc4", "allow-all"] 133 | end 134 | 135 | if File.exist?(CLOUD_CONFIG_PATH) 136 | config.vm.provision :file, :source => "#{CLOUD_CONFIG_PATH}", :destination => "/tmp/vagrantfile-user-data" 137 | config.vm.provision :shell, :inline => "mv /tmp/vagrantfile-user-data /var/lib/coreos-vagrant/", :privileged => true 138 | end 139 | 140 | end 141 | end 142 | end 143 | -------------------------------------------------------------------------------- /controller/Vagrantfile: -------------------------------------------------------------------------------- 1 | # -*- mode: ruby -*- 2 | # # vi: set ft=ruby : 3 | 4 | require 'fileutils' 5 | 6 | Vagrant.require_version ">= 1.6.0" 7 | 8 | CLOUD_CONFIG_PATH = File.join(File.dirname(__FILE__), "user-data") 9 | CONFIG = File.join(File.dirname(__FILE__), "config.rb") 10 | 11 | # Defaults for config options defined in CONFIG 12 | $num_instances = 1 13 | $instance_name_prefix = "core" 14 | $update_channel = "alpha" 15 | $enable_serial_logging = false 16 | $share_home = false 17 | $vm_gui = false 18 | $vm_memory = 1024 19 | $vm_cpus = 1 20 | $shared_folders = {} 21 | $forwarded_ports = {} 22 | 23 | # Attempt to apply the deprecated environment variable NUM_INSTANCES to 24 | # $num_instances while allowing config.rb to override it 25 | if ENV["NUM_INSTANCES"].to_i > 0 && ENV["NUM_INSTANCES"] 26 | $num_instances = ENV["NUM_INSTANCES"].to_i 27 | end 28 | 29 | if File.exist?(CONFIG) 30 | require CONFIG 31 | end 32 | 33 | # Use old vb_xxx config variables when set 34 | def vm_gui 35 | $vb_gui.nil? ? $vm_gui : $vb_gui 36 | end 37 | 38 | def vm_memory 39 | $vb_memory.nil? ? $vm_memory : $vb_memory 40 | end 41 | 42 | def vm_cpus 43 | $vb_cpus.nil? ? $vm_cpus : $vb_cpus 44 | end 45 | 46 | Vagrant.configure("2") do |config| 47 | # always use Vagrants insecure key 48 | config.ssh.insert_key = false 49 | 50 | config.vm.box = "coreos-%s" % $update_channel 51 | config.vm.box_version = ">= 308.0.1" 52 | config.vm.box_url = "http://%s.release.core-os.net/amd64-usr/current/coreos_production_vagrant.json" % $update_channel 53 | 54 | ["vmware_fusion", "vmware_workstation"].each do |vmware| 55 | config.vm.provider vmware do |v, override| 56 | override.vm.box_url = "http://%s.release.core-os.net/amd64-usr/current/coreos_production_vagrant_vmware_fusion.json" % $update_channel 57 | end 58 | end 59 | 60 | config.vm.provider :virtualbox do |v| 61 | # On VirtualBox, we don't have guest additions or a functional vboxsf 62 | # in CoreOS, so tell Vagrant that so it can be smarter. 63 | v.check_guest_additions = false 64 | v.functional_vboxsf = false 65 | end 66 | 67 | # plugin conflict 68 | if Vagrant.has_plugin?("vagrant-vbguest") then 69 | config.vbguest.auto_update = false 70 | end 71 | 72 | (1..$num_instances).each do |i| 73 | config.vm.define vm_name = "%s-%02d" % [$instance_name_prefix, i] do |config| 74 | config.vm.hostname = vm_name 75 | 76 | if $enable_serial_logging 77 | logdir = File.join(File.dirname(__FILE__), "log") 78 | FileUtils.mkdir_p(logdir) 79 | 80 | serialFile = File.join(logdir, "%s-serial.txt" % vm_name) 81 | FileUtils.touch(serialFile) 82 | 83 | ["vmware_fusion", "vmware_workstation"].each do |vmware| 84 | config.vm.provider vmware do |v, override| 85 | v.vmx["serial0.present"] = "TRUE" 86 | v.vmx["serial0.fileType"] = "file" 87 | v.vmx["serial0.fileName"] = serialFile 88 | v.vmx["serial0.tryNoRxLoss"] = "FALSE" 89 | end 90 | end 91 | 92 | config.vm.provider :virtualbox do |vb, override| 93 | vb.customize ["modifyvm", :id, "--uart1", "0x3F8", "4"] 94 | vb.customize ["modifyvm", :id, "--uartmode1", serialFile] 95 | end 96 | end 97 | 98 | if $expose_docker_tcp 99 | config.vm.network "forwarded_port", guest: 2375, host: ($expose_docker_tcp + i - 1), auto_correct: true 100 | end 101 | 102 | $forwarded_ports.each do |guest, host| 103 | config.vm.network "forwarded_port", guest: guest, host: host, auto_correct: true 104 | end 105 | 106 | ["vmware_fusion", "vmware_workstation"].each do |vmware| 107 | config.vm.provider vmware do |v| 108 | v.gui = vm_gui 109 | v.vmx['memsize'] = vm_memory 110 | v.vmx['numvcpus'] = vm_cpus 111 | end 112 | end 113 | 114 | config.vm.provider :virtualbox do |vb| 115 | vb.gui = vm_gui 116 | vb.memory = vm_memory 117 | vb.cpus = vm_cpus 118 | end 119 | 120 | config.vm.network :private_network, ip: "10.0.5.#{i+10}", :adapter => 2 121 | config.vm.network :private_network, ip: "172.16.0.#{i+10}", :netmask => "255.255.0.0", :adapter => 3 122 | config.vm.network :private_network, ip: "192.168.10.#{i+10}", :netmask => "255.255.255.0", :adapter => 4 123 | 124 | # Comment below to disble NFS for sharing the host machine into the coreos-vagrant VM. 125 | config.vm.synced_folder "~/test/continuse", "/continuse", id: "root", :nfs => true, :mount_options => ["nolock,vers=3,udp"], :map_uid => 0, :map_gid => 0 126 | 127 | config.vm.provider :virtualbox do |vb| 128 | vb.customize ["modifyvm", :id, "--nic2", "NatNetwork"] 129 | vb.customize ["modifyvm", :id, "--nicpromisc1", "allow-all"] 130 | vb.customize ["modifyvm", :id, "--nicpromisc2", "allow-all"] 131 | vb.customize ["modifyvm", :id, "--nicpromisc3", "allow-all"] 132 | vb.customize ["modifyvm", :id, "--nicpromisc4", "allow-all"] 133 | end 134 | 135 | if File.exist?(CLOUD_CONFIG_PATH) 136 | config.vm.provision :file, :source => "#{CLOUD_CONFIG_PATH}", :destination => "/tmp/vagrantfile-user-data" 137 | config.vm.provision :shell, :inline => "mv /tmp/vagrantfile-user-data /var/lib/coreos-vagrant/", :privileged => true 138 | end 139 | 140 | end 141 | end 142 | end 143 | -------------------------------------------------------------------------------- /examples/redis-cluster/images/auto_config_redis_cluster.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | 3 | import os, sys, json, etcd, time 4 | from rediscluster.cluster_mgt import RedisClusterMgt 5 | from subprocess import call 6 | 7 | class JsonObject: 8 | def __init__(self, d): 9 | self.__dict__ = d 10 | 11 | class EtcdObject: 12 | def __init__(self, ip, port, namespace, key): 13 | self.ip = ip 14 | self.port = int(port) 15 | self.namespace = namespace 16 | self.key = key 17 | 18 | def getHosts(self): 19 | hostIP = set() 20 | podIP = [] 21 | hostIP_podIP = {} 22 | 23 | client = etcd.Client(host=self.ip, port=self.port) 24 | 25 | etcd_key = "/registry/pods/%s" % self.namespace 26 | 27 | try: 28 | r = client.read(etcd_key, recursive=True, sorted=True, timeout=1) 29 | for child in r.children: 30 | data = json.loads(child.value, object_hook=JsonObject) 31 | 32 | if data.metadata.labels.name[0:len(self.key)] == self.key and \ 33 | data.status.phase == "Running": 34 | hostIP.add(data.status.hostIP) 35 | podIP.append('%s:6379' % data.status.podIP) 36 | if not data.status.hostIP in hostIP_podIP.keys() : 37 | hostIP_podIP[data.status.hostIP] = [] 38 | hostIP_podIP[data.status.hostIP].append('%s:6379' % data.status.podIP) 39 | 40 | return hostIP, podIP, hostIP_podIP 41 | except Exception: 42 | return -1, -1, -1 43 | 44 | def redis_cluster(hosts): 45 | startup_nodes = [] 46 | 47 | for i in hosts: 48 | hostIP = i.split(':') 49 | x = {"host":hostIP[0], "port":hostIP[1]} 50 | startup_nodes.append(x) 51 | 52 | try: 53 | r = RedisClusterMgt(startup_nodes) 54 | 55 | return r.slots(), r.nodes() 56 | except Exception: 57 | return -1, -1 58 | 59 | def distrib_host(hosts, rep) : 60 | master = [] 61 | slave = [] 62 | 63 | i = 0 64 | while True: 65 | if len(hosts) == 0: 66 | break 67 | #for key in hosts.keys(): 68 | for k in sorted(hosts, key=lambda k: len(hosts[k]), reverse=True): 69 | if len(hosts[k]) != 0: 70 | pod = hosts[k].pop() 71 | if (i % (rep+1)) == 0: 72 | master.append(pod) 73 | else : 74 | slave.append(pod) 75 | i += 1 76 | 77 | else : 78 | del hosts[k] 79 | 80 | return master, slave 81 | 82 | # Redis Cluster Init Setup 83 | def redis_init_setup(hostIP, podIP, host_podIP, replicas): 84 | if len(podIP) > 0 and len(podIP) % (replicas + 1) == 0 : 85 | print "###### The redis cluster create at the first time." 86 | m, s = distrib_host(hostIP_podIP, replicas) 87 | x = len(m) 88 | y = len(s) 89 | if (x + y) % (replicas + 1) == 0 and (x + y) >= 3 * (replicas + 1) : 90 | command1 = "spawn /redis-3.0.3/src/redis-trib.rb create --replicas %d" % replicas 91 | command2 = ' '.join(m) 92 | command3 = ' '.join(s) 93 | command = ' '.join([command1, command2, command3]) 94 | with open('expect.script', 'w') as f: 95 | f.write("set timeout 30\n") 96 | f.write("%s\n" % command) 97 | f.write("expect \"*yes*\"\n") 98 | f.write("send \"yes\\r\"\n") 99 | f.write("expect \"All 16384 slots covered\"\n") 100 | f.write("expect eof\n") 101 | f.close() 102 | 103 | # expect excute 104 | command_list = ["/usr/bin/expect", "-f", "expect.script"] 105 | xx = call(command_list) 106 | else : 107 | print 'The redis cluster need at least %d nodes.' % (3 * (replicas + 1) ) 108 | 109 | def redis_add_setup(hostIP, podIP, hostIP_podIP, replicas, real_master, real_slave): 110 | # Getting new node info 111 | for key in hostIP_podIP.keys(): 112 | diff = set(hostIP_podIP[key]) - (set(real_master) | set(real_slave)) 113 | hostIP_podIP[key] = list(diff) 114 | 115 | new_master, new_slave = distrib_host(hostIP_podIP, replicas) 116 | 117 | f = open('expect.script', 'w') 118 | f.write("set timeout 30\n") 119 | for i in new_master: 120 | command1 = "spawn /redis-3.0.3/src/redis-trib.rb add-node %s %s" % \ 121 | (i, real_master[0]) 122 | f.write("%s\n" % command1) 123 | f.write("expect \"New node added correctly\"\n") 124 | 125 | f.write("expect eof\n") 126 | f.close() 127 | 128 | # expect excute 129 | command_list = ["/usr/bin/expect", "-f", "expect.script"] 130 | call(command_list) 131 | 132 | time.sleep(3) 133 | 134 | # get node id 135 | ret, nodes = redis_cluster(podIP) 136 | 137 | cluster_length = len(real_master) 138 | command1 = "spawn /redis-3.0.3/src/redis-trib.rb reshard %s" % (real_master[0]) 139 | for i in new_master: 140 | # redis cluster fix processing 141 | for key in nodes.keys(): 142 | if nodes[key]['role'] == 'master': 143 | command_list = [] 144 | command_list.append("/redis-3.0.3/src/redis-trib.rb") 145 | command_list.append("fix") 146 | command_list.append(key) 147 | call(command_list) 148 | 149 | # reshard for a new node 150 | cluster_length += 1 151 | f = open('expect.script', 'w') 152 | f.write("set timeout 30\n") 153 | f.write("%s\n"% command1) 154 | f.write("expect \"How many slots do you want to move\"\n") 155 | f.write("send \"%d\\r\"\n" % (16384 / cluster_length)) 156 | f.write("expect \"What is the receiving node ID\"\n") 157 | f.write("send \"%s\\r\"\n" % nodes[i]['node_id']) 158 | f.write("expect \"Source node #1\"\n") 159 | f.write("send \"all\\r\"\n") 160 | f.write("expect \"Do you want to proceed with the proposed reshard plan\"\n") 161 | f.write("send \"yes\\r\"\n") 162 | f.write("expect eof\n") 163 | f.close() 164 | 165 | # expect excute 166 | command_list = ["/usr/bin/expect", "-f", "expect.script"] 167 | if call(command_list) != 0: 168 | command1 = "spawn /redis-3.0.3/src/redis-trib.rb del-node %s %s" % (real_master[0], nodes[i]['node_id']) 169 | f = open('expect.script', 'w') 170 | f.write("set timeout 30\n") 171 | f.write("%s\n"% command1) 172 | f.write("expect eof\n") 173 | f.close() 174 | xx = call(command_list) 175 | print 'DEL-NODE', xx 176 | return 177 | 178 | # make slave 179 | i = 0 180 | for m in new_master: 181 | while replicas : 182 | f = open('expect.script', 'w') 183 | f.write("set timeout 30\n") 184 | command2 = "spawn /redis-3.0.3/src/redis-trib.rb add-node --slave %s %s" % \ 185 | (new_slave[i], m) 186 | f.write("%s\n"% command2) 187 | f.write("expect eof\n") 188 | f.close() 189 | # expect excute 190 | command_list = ["/usr/bin/expect", "-f", "expect.script"] 191 | call(command_list) 192 | 193 | i += 1 194 | if i % replicas == 0: 195 | break 196 | 197 | def redis_recovery_setup(ret, real_master, real_slave, podIP, hostIP_podIP, replicas): 198 | m = {} # master from redis cluster info 199 | s = {} # slave from redis cluster info 200 | t = {} # count of slave for master 201 | 202 | # split master / slave from ret dict 203 | for key in ret.keys(): 204 | if key == 'master': 205 | for ip in ret[key].keys(): 206 | m[ip] = ret[key][ip] 207 | if key == 'slave': 208 | for ip in ret[key].keys(): 209 | s[ip] = ret[key][ip] 210 | 211 | # init count of slave no, of master 212 | for key in m.keys(): 213 | t[key] = 0 214 | 215 | for v in s.values(): 216 | for k in m.keys(): 217 | if m[k] == v: 218 | i = t[k] 219 | t[k] = i + 1 220 | 221 | be_slave = {} 222 | diff = set(podIP) - (set(real_master) | set(real_slave)) 223 | for x in diff: 224 | for y in hostIP_podIP.keys(): 225 | if x in hostIP_podIP[y]: 226 | be_slave[x] = y 227 | 228 | for x in t.keys(): 229 | for y in hostIP_podIP.keys(): 230 | if x in hostIP_podIP[y]: 231 | while t[x] < replicas: 232 | t[x] = t[x] + 1 233 | for z in be_slave.keys(): 234 | if be_slave[z] != y: 235 | command1 = "spawn /redis-3.0.3/src/redis-trib.rb add-node --slave %s %s" % (z, x) 236 | f = open('expect.script', 'w') 237 | f.write("set timeout 30\n") 238 | f.write("%s\n"% command1) 239 | f.write("expect eof\n") 240 | f.close() 241 | # expect excute 242 | command_list = ["/usr/bin/expect", "-f", "expect.script"] 243 | call(command_list) 244 | 245 | etcdctl_peers = os.getenv('ETCDCTL_PEERS') 246 | kube_namespace = os.getenv('KUBE_NAMESPACE') 247 | kube_label = os.getenv('KUBE_LABEL') 248 | replicas = int(os.getenv("REPLICAS")) 249 | 250 | # Check for etcdctl_peers is str type 251 | if not isinstance(etcdctl_peers, str) : 252 | exit(1) 253 | 254 | etcd_ip_port = etcdctl_peers.split(',') 255 | 256 | # pod info from etcd 257 | flag = 0 258 | for ctrl_ip_port in etcd_ip_port : 259 | ip, port = ctrl_ip_port.split(':') 260 | a = EtcdObject(ip, port, kube_namespace, kube_label) 261 | hostIP, podIP, hostIP_podIP = a.getHosts() 262 | 263 | if isinstance(hostIP_podIP, dict) and isinstance(hostIP, set) and isinstance(podIP, list): 264 | flag = 1 265 | break 266 | 267 | # redis cluster info 268 | if flag == 1: 269 | print "Getting information for redis cluster..........." 270 | time.sleep(5) 271 | ret, nodes = redis_cluster(podIP) 272 | 273 | if ret == -1: 274 | ### REDIS INITIAL SETUP ### 275 | print "Initial setup for the redis cluster.........." 276 | redis_init_setup(hostIP, podIP, hostIP_podIP, replicas) 277 | 278 | else : 279 | real_master = ret['master'].keys() 280 | real_slave = ret['slave'].keys() 281 | 282 | ### Nomal Case (NO CHANGE OR ADD NODE) ### 283 | if len(real_master) * replicas == len(real_slave) : 284 | ### THERE IS NO CHANGE THE CLUSTER ### 285 | if set(real_master) | set(real_slave) == set(podIP): # Normal 286 | print "There is no change in the redis cluster." 287 | ### THE REDIS CLUSTER NODE ADD ### 288 | else : 289 | print "The redis cluster node add." 290 | redis_add_setup(hostIP, podIP, hostIP_podIP, replicas, real_master, real_slave) 291 | 292 | ### Abnormal : one or more nodes failure ### 293 | else : 294 | print "node failure....." 295 | redis_recovery_setup(ret, real_master, real_slave, podIP, hostIP_podIP, replicas) 296 | 297 | ### ETCD CONNECTION ERROR 298 | else : 299 | print "Error : etcd connection error" 300 | 301 | -------------------------------------------------------------------------------- /examples/redis-cluster/images/redis.conf: -------------------------------------------------------------------------------- 1 | # Redis configuration file example 2 | 3 | # Note on units: when memory size is needed, it is possible to specify 4 | # it in the usual form of 1k 5GB 4M and so forth: 5 | # 6 | # 1k => 1000 bytes 7 | # 1kb => 1024 bytes 8 | # 1m => 1000000 bytes 9 | # 1mb => 1024*1024 bytes 10 | # 1g => 1000000000 bytes 11 | # 1gb => 1024*1024*1024 bytes 12 | # 13 | # units are case insensitive so 1GB 1Gb 1gB are all the same. 14 | 15 | ################################## INCLUDES ################################### 16 | 17 | # Include one or more other config files here. This is useful if you 18 | # have a standard template that goes to all Redis servers but also need 19 | # to customize a few per-server settings. Include files can include 20 | # other files, so use this wisely. 21 | # 22 | # Notice option "include" won't be rewritten by command "CONFIG REWRITE" 23 | # from admin or Redis Sentinel. Since Redis always uses the last processed 24 | # line as value of a configuration directive, you'd better put includes 25 | # at the beginning of this file to avoid overwriting config change at runtime. 26 | # 27 | # If instead you are interested in using includes to override configuration 28 | # options, it is better to use include as the last line. 29 | # 30 | # include /path/to/local.conf 31 | # include /path/to/other.conf 32 | 33 | ################################ GENERAL ##################################### 34 | 35 | # By default Redis does not run as a daemon. Use 'yes' if you need it. 36 | # Note that Redis will write a pid file in /var/run/redis.pid when daemonized. 37 | daemonize no 38 | 39 | # When running daemonized, Redis writes a pid file in /var/run/redis.pid by 40 | # default. You can specify a custom pid file location here. 41 | pidfile /var/run/redis.pid 42 | 43 | # Accept connections on the specified port, default is 6379. 44 | # If port 0 is specified Redis will not listen on a TCP socket. 45 | port 6379 46 | 47 | # TCP listen() backlog. 48 | # 49 | # In high requests-per-second environments you need an high backlog in order 50 | # to avoid slow clients connections issues. Note that the Linux kernel 51 | # will silently truncate it to the value of /proc/sys/net/core/somaxconn so 52 | # make sure to raise both the value of somaxconn and tcp_max_syn_backlog 53 | # in order to get the desired effect. 54 | tcp-backlog 511 55 | 56 | # By default Redis listens for connections from all the network interfaces 57 | # available on the server. It is possible to listen to just one or multiple 58 | # interfaces using the "bind" configuration directive, followed by one or 59 | # more IP addresses. 60 | # 61 | # Examples: 62 | # 63 | # bind 192.168.1.100 10.0.0.1 64 | # bind 127.0.0.1 65 | 66 | # Specify the path for the Unix socket that will be used to listen for 67 | # incoming connections. There is no default, so Redis will not listen 68 | # on a unix socket when not specified. 69 | # 70 | # unixsocket /tmp/redis.sock 71 | # unixsocketperm 700 72 | 73 | # Close the connection after a client is idle for N seconds (0 to disable) 74 | timeout 0 75 | 76 | # TCP keepalive. 77 | # 78 | # If non-zero, use SO_KEEPALIVE to send TCP ACKs to clients in absence 79 | # of communication. This is useful for two reasons: 80 | # 81 | # 1) Detect dead peers. 82 | # 2) Take the connection alive from the point of view of network 83 | # equipment in the middle. 84 | # 85 | # On Linux, the specified value (in seconds) is the period used to send ACKs. 86 | # Note that to close the connection the double of the time is needed. 87 | # On other kernels the period depends on the kernel configuration. 88 | # 89 | # A reasonable value for this option is 60 seconds. 90 | tcp-keepalive 0 91 | 92 | # Specify the server verbosity level. 93 | # This can be one of: 94 | # debug (a lot of information, useful for development/testing) 95 | # verbose (many rarely useful info, but not a mess like the debug level) 96 | # notice (moderately verbose, what you want in production probably) 97 | # warning (only very important / critical messages are logged) 98 | loglevel notice 99 | 100 | # Specify the log file name. Also the empty string can be used to force 101 | # Redis to log on the standard output. Note that if you use standard 102 | # output for logging but daemonize, logs will be sent to /dev/null 103 | logfile "" 104 | 105 | # To enable logging to the system logger, just set 'syslog-enabled' to yes, 106 | # and optionally update the other syslog parameters to suit your needs. 107 | # syslog-enabled no 108 | 109 | # Specify the syslog identity. 110 | # syslog-ident redis 111 | 112 | # Specify the syslog facility. Must be USER or between LOCAL0-LOCAL7. 113 | # syslog-facility local0 114 | 115 | # Set the number of databases. The default database is DB 0, you can select 116 | # a different one on a per-connection basis using SELECT where 117 | # dbid is a number between 0 and 'databases'-1 118 | databases 16 119 | 120 | ################################ SNAPSHOTTING ################################ 121 | # 122 | # Save the DB on disk: 123 | # 124 | # save 125 | # 126 | # Will save the DB if both the given number of seconds and the given 127 | # number of write operations against the DB occurred. 128 | # 129 | # In the example below the behaviour will be to save: 130 | # after 900 sec (15 min) if at least 1 key changed 131 | # after 300 sec (5 min) if at least 10 keys changed 132 | # after 60 sec if at least 10000 keys changed 133 | # 134 | # Note: you can disable saving completely by commenting out all "save" lines. 135 | # 136 | # It is also possible to remove all the previously configured save 137 | # points by adding a save directive with a single empty string argument 138 | # like in the following example: 139 | # 140 | # save "" 141 | 142 | save 900 1 143 | save 300 10 144 | save 60 10000 145 | 146 | # By default Redis will stop accepting writes if RDB snapshots are enabled 147 | # (at least one save point) and the latest background save failed. 148 | # This will make the user aware (in a hard way) that data is not persisting 149 | # on disk properly, otherwise chances are that no one will notice and some 150 | # disaster will happen. 151 | # 152 | # If the background saving process will start working again Redis will 153 | # automatically allow writes again. 154 | # 155 | # However if you have setup your proper monitoring of the Redis server 156 | # and persistence, you may want to disable this feature so that Redis will 157 | # continue to work as usual even if there are problems with disk, 158 | # permissions, and so forth. 159 | stop-writes-on-bgsave-error yes 160 | 161 | # Compress string objects using LZF when dump .rdb databases? 162 | # For default that's set to 'yes' as it's almost always a win. 163 | # If you want to save some CPU in the saving child set it to 'no' but 164 | # the dataset will likely be bigger if you have compressible values or keys. 165 | rdbcompression yes 166 | 167 | # Since version 5 of RDB a CRC64 checksum is placed at the end of the file. 168 | # This makes the format more resistant to corruption but there is a performance 169 | # hit to pay (around 10%) when saving and loading RDB files, so you can disable it 170 | # for maximum performances. 171 | # 172 | # RDB files created with checksum disabled have a checksum of zero that will 173 | # tell the loading code to skip the check. 174 | rdbchecksum yes 175 | 176 | # The filename where to dump the DB 177 | dbfilename dump.rdb 178 | 179 | # The working directory. 180 | # 181 | # The DB will be written inside this directory, with the filename specified 182 | # above using the 'dbfilename' configuration directive. 183 | # 184 | # The Append Only File will also be created inside this directory. 185 | # 186 | # Note that you must specify a directory here, not a file name. 187 | dir ./ 188 | 189 | ################################# REPLICATION ################################# 190 | 191 | # Master-Slave replication. Use slaveof to make a Redis instance a copy of 192 | # another Redis server. A few things to understand ASAP about Redis replication. 193 | # 194 | # 1) Redis replication is asynchronous, but you can configure a master to 195 | # stop accepting writes if it appears to be not connected with at least 196 | # a given number of slaves. 197 | # 2) Redis slaves are able to perform a partial resynchronization with the 198 | # master if the replication link is lost for a relatively small amount of 199 | # time. You may want to configure the replication backlog size (see the next 200 | # sections of this file) with a sensible value depending on your needs. 201 | # 3) Replication is automatic and does not need user intervention. After a 202 | # network partition slaves automatically try to reconnect to masters 203 | # and resynchronize with them. 204 | # 205 | # slaveof 206 | 207 | # If the master is password protected (using the "requirepass" configuration 208 | # directive below) it is possible to tell the slave to authenticate before 209 | # starting the replication synchronization process, otherwise the master will 210 | # refuse the slave request. 211 | # 212 | # masterauth 213 | 214 | # When a slave loses its connection with the master, or when the replication 215 | # is still in progress, the slave can act in two different ways: 216 | # 217 | # 1) if slave-serve-stale-data is set to 'yes' (the default) the slave will 218 | # still reply to client requests, possibly with out of date data, or the 219 | # data set may just be empty if this is the first synchronization. 220 | # 221 | # 2) if slave-serve-stale-data is set to 'no' the slave will reply with 222 | # an error "SYNC with master in progress" to all the kind of commands 223 | # but to INFO and SLAVEOF. 224 | # 225 | slave-serve-stale-data yes 226 | 227 | # You can configure a slave instance to accept writes or not. Writing against 228 | # a slave instance may be useful to store some ephemeral data (because data 229 | # written on a slave will be easily deleted after resync with the master) but 230 | # may also cause problems if clients are writing to it because of a 231 | # misconfiguration. 232 | # 233 | # Since Redis 2.6 by default slaves are read-only. 234 | # 235 | # Note: read only slaves are not designed to be exposed to untrusted clients 236 | # on the internet. It's just a protection layer against misuse of the instance. 237 | # Still a read only slave exports by default all the administrative commands 238 | # such as CONFIG, DEBUG, and so forth. To a limited extent you can improve 239 | # security of read only slaves using 'rename-command' to shadow all the 240 | # administrative / dangerous commands. 241 | slave-read-only yes 242 | 243 | # Replication SYNC strategy: disk or socket. 244 | # 245 | # ------------------------------------------------------- 246 | # WARNING: DISKLESS REPLICATION IS EXPERIMENTAL CURRENTLY 247 | # ------------------------------------------------------- 248 | # 249 | # New slaves and reconnecting slaves that are not able to continue the replication 250 | # process just receiving differences, need to do what is called a "full 251 | # synchronization". An RDB file is transmitted from the master to the slaves. 252 | # The transmission can happen in two different ways: 253 | # 254 | # 1) Disk-backed: The Redis master creates a new process that writes the RDB 255 | # file on disk. Later the file is transferred by the parent 256 | # process to the slaves incrementally. 257 | # 2) Diskless: The Redis master creates a new process that directly writes the 258 | # RDB file to slave sockets, without touching the disk at all. 259 | # 260 | # With disk-backed replication, while the RDB file is generated, more slaves 261 | # can be queued and served with the RDB file as soon as the current child producing 262 | # the RDB file finishes its work. With diskless replication instead once 263 | # the transfer starts, new slaves arriving will be queued and a new transfer 264 | # will start when the current one terminates. 265 | # 266 | # When diskless replication is used, the master waits a configurable amount of 267 | # time (in seconds) before starting the transfer in the hope that multiple slaves 268 | # will arrive and the transfer can be parallelized. 269 | # 270 | # With slow disks and fast (large bandwidth) networks, diskless replication 271 | # works better. 272 | repl-diskless-sync no 273 | 274 | # When diskless replication is enabled, it is possible to configure the delay 275 | # the server waits in order to spawn the child that transfers the RDB via socket 276 | # to the slaves. 277 | # 278 | # This is important since once the transfer starts, it is not possible to serve 279 | # new slaves arriving, that will be queued for the next RDB transfer, so the server 280 | # waits a delay in order to let more slaves arrive. 281 | # 282 | # The delay is specified in seconds, and by default is 5 seconds. To disable 283 | # it entirely just set it to 0 seconds and the transfer will start ASAP. 284 | repl-diskless-sync-delay 5 285 | 286 | # Slaves send PINGs to server in a predefined interval. It's possible to change 287 | # this interval with the repl_ping_slave_period option. The default value is 10 288 | # seconds. 289 | # 290 | # repl-ping-slave-period 10 291 | 292 | # The following option sets the replication timeout for: 293 | # 294 | # 1) Bulk transfer I/O during SYNC, from the point of view of slave. 295 | # 2) Master timeout from the point of view of slaves (data, pings). 296 | # 3) Slave timeout from the point of view of masters (REPLCONF ACK pings). 297 | # 298 | # It is important to make sure that this value is greater than the value 299 | # specified for repl-ping-slave-period otherwise a timeout will be detected 300 | # every time there is low traffic between the master and the slave. 301 | # 302 | # repl-timeout 60 303 | 304 | # Disable TCP_NODELAY on the slave socket after SYNC? 305 | # 306 | # If you select "yes" Redis will use a smaller number of TCP packets and 307 | # less bandwidth to send data to slaves. But this can add a delay for 308 | # the data to appear on the slave side, up to 40 milliseconds with 309 | # Linux kernels using a default configuration. 310 | # 311 | # If you select "no" the delay for data to appear on the slave side will 312 | # be reduced but more bandwidth will be used for replication. 313 | # 314 | # By default we optimize for low latency, but in very high traffic conditions 315 | # or when the master and slaves are many hops away, turning this to "yes" may 316 | # be a good idea. 317 | repl-disable-tcp-nodelay no 318 | 319 | # Set the replication backlog size. The backlog is a buffer that accumulates 320 | # slave data when slaves are disconnected for some time, so that when a slave 321 | # wants to reconnect again, often a full resync is not needed, but a partial 322 | # resync is enough, just passing the portion of data the slave missed while 323 | # disconnected. 324 | # 325 | # The bigger the replication backlog, the longer the time the slave can be 326 | # disconnected and later be able to perform a partial resynchronization. 327 | # 328 | # The backlog is only allocated once there is at least a slave connected. 329 | # 330 | # repl-backlog-size 1mb 331 | 332 | # After a master has no longer connected slaves for some time, the backlog 333 | # will be freed. The following option configures the amount of seconds that 334 | # need to elapse, starting from the time the last slave disconnected, for 335 | # the backlog buffer to be freed. 336 | # 337 | # A value of 0 means to never release the backlog. 338 | # 339 | # repl-backlog-ttl 3600 340 | 341 | # The slave priority is an integer number published by Redis in the INFO output. 342 | # It is used by Redis Sentinel in order to select a slave to promote into a 343 | # master if the master is no longer working correctly. 344 | # 345 | # A slave with a low priority number is considered better for promotion, so 346 | # for instance if there are three slaves with priority 10, 100, 25 Sentinel will 347 | # pick the one with priority 10, that is the lowest. 348 | # 349 | # However a special priority of 0 marks the slave as not able to perform the 350 | # role of master, so a slave with priority of 0 will never be selected by 351 | # Redis Sentinel for promotion. 352 | # 353 | # By default the priority is 100. 354 | slave-priority 100 355 | 356 | # It is possible for a master to stop accepting writes if there are less than 357 | # N slaves connected, having a lag less or equal than M seconds. 358 | # 359 | # The N slaves need to be in "online" state. 360 | # 361 | # The lag in seconds, that must be <= the specified value, is calculated from 362 | # the last ping received from the slave, that is usually sent every second. 363 | # 364 | # This option does not GUARANTEE that N replicas will accept the write, but 365 | # will limit the window of exposure for lost writes in case not enough slaves 366 | # are available, to the specified number of seconds. 367 | # 368 | # For example to require at least 3 slaves with a lag <= 10 seconds use: 369 | # 370 | # min-slaves-to-write 3 371 | # min-slaves-max-lag 10 372 | # 373 | # Setting one or the other to 0 disables the feature. 374 | # 375 | # By default min-slaves-to-write is set to 0 (feature disabled) and 376 | # min-slaves-max-lag is set to 10. 377 | 378 | ################################## SECURITY ################################### 379 | 380 | # Require clients to issue AUTH before processing any other 381 | # commands. This might be useful in environments in which you do not trust 382 | # others with access to the host running redis-server. 383 | # 384 | # This should stay commented out for backward compatibility and because most 385 | # people do not need auth (e.g. they run their own servers). 386 | # 387 | # Warning: since Redis is pretty fast an outside user can try up to 388 | # 150k passwords per second against a good box. This means that you should 389 | # use a very strong password otherwise it will be very easy to break. 390 | # 391 | # requirepass foobared 392 | 393 | # Command renaming. 394 | # 395 | # It is possible to change the name of dangerous commands in a shared 396 | # environment. For instance the CONFIG command may be renamed into something 397 | # hard to guess so that it will still be available for internal-use tools 398 | # but not available for general clients. 399 | # 400 | # Example: 401 | # 402 | # rename-command CONFIG b840fc02d524045429941cc15f59e41cb7be6c52 403 | # 404 | # It is also possible to completely kill a command by renaming it into 405 | # an empty string: 406 | # 407 | # rename-command CONFIG "" 408 | # 409 | # Please note that changing the name of commands that are logged into the 410 | # AOF file or transmitted to slaves may cause problems. 411 | 412 | ################################### LIMITS #################################### 413 | 414 | # Set the max number of connected clients at the same time. By default 415 | # this limit is set to 10000 clients, however if the Redis server is not 416 | # able to configure the process file limit to allow for the specified limit 417 | # the max number of allowed clients is set to the current file limit 418 | # minus 32 (as Redis reserves a few file descriptors for internal uses). 419 | # 420 | # Once the limit is reached Redis will close all the new connections sending 421 | # an error 'max number of clients reached'. 422 | # 423 | # maxclients 10000 424 | 425 | # Don't use more memory than the specified amount of bytes. 426 | # When the memory limit is reached Redis will try to remove keys 427 | # according to the eviction policy selected (see maxmemory-policy). 428 | # 429 | # If Redis can't remove keys according to the policy, or if the policy is 430 | # set to 'noeviction', Redis will start to reply with errors to commands 431 | # that would use more memory, like SET, LPUSH, and so on, and will continue 432 | # to reply to read-only commands like GET. 433 | # 434 | # This option is usually useful when using Redis as an LRU cache, or to set 435 | # a hard memory limit for an instance (using the 'noeviction' policy). 436 | # 437 | # WARNING: If you have slaves attached to an instance with maxmemory on, 438 | # the size of the output buffers needed to feed the slaves are subtracted 439 | # from the used memory count, so that network problems / resyncs will 440 | # not trigger a loop where keys are evicted, and in turn the output 441 | # buffer of slaves is full with DELs of keys evicted triggering the deletion 442 | # of more keys, and so forth until the database is completely emptied. 443 | # 444 | # In short... if you have slaves attached it is suggested that you set a lower 445 | # limit for maxmemory so that there is some free RAM on the system for slave 446 | # output buffers (but this is not needed if the policy is 'noeviction'). 447 | # 448 | # maxmemory 449 | 450 | # MAXMEMORY POLICY: how Redis will select what to remove when maxmemory 451 | # is reached. You can select among five behaviors: 452 | # 453 | # volatile-lru -> remove the key with an expire set using an LRU algorithm 454 | # allkeys-lru -> remove any key according to the LRU algorithm 455 | # volatile-random -> remove a random key with an expire set 456 | # allkeys-random -> remove a random key, any key 457 | # volatile-ttl -> remove the key with the nearest expire time (minor TTL) 458 | # noeviction -> don't expire at all, just return an error on write operations 459 | # 460 | # Note: with any of the above policies, Redis will return an error on write 461 | # operations, when there are no suitable keys for eviction. 462 | # 463 | # At the date of writing these commands are: set setnx setex append 464 | # incr decr rpush lpush rpushx lpushx linsert lset rpoplpush sadd 465 | # sinter sinterstore sunion sunionstore sdiff sdiffstore zadd zincrby 466 | # zunionstore zinterstore hset hsetnx hmset hincrby incrby decrby 467 | # getset mset msetnx exec sort 468 | # 469 | # The default is: 470 | # 471 | # maxmemory-policy noeviction 472 | 473 | # LRU and minimal TTL algorithms are not precise algorithms but approximated 474 | # algorithms (in order to save memory), so you can tune it for speed or 475 | # accuracy. For default Redis will check five keys and pick the one that was 476 | # used less recently, you can change the sample size using the following 477 | # configuration directive. 478 | # 479 | # The default of 5 produces good enough results. 10 Approximates very closely 480 | # true LRU but costs a bit more CPU. 3 is very fast but not very accurate. 481 | # 482 | # maxmemory-samples 5 483 | 484 | ############################## APPEND ONLY MODE ############################### 485 | 486 | # By default Redis asynchronously dumps the dataset on disk. This mode is 487 | # good enough in many applications, but an issue with the Redis process or 488 | # a power outage may result into a few minutes of writes lost (depending on 489 | # the configured save points). 490 | # 491 | # The Append Only File is an alternative persistence mode that provides 492 | # much better durability. For instance using the default data fsync policy 493 | # (see later in the config file) Redis can lose just one second of writes in a 494 | # dramatic event like a server power outage, or a single write if something 495 | # wrong with the Redis process itself happens, but the operating system is 496 | # still running correctly. 497 | # 498 | # AOF and RDB persistence can be enabled at the same time without problems. 499 | # If the AOF is enabled on startup Redis will load the AOF, that is the file 500 | # with the better durability guarantees. 501 | # 502 | # Please check http://redis.io/topics/persistence for more information. 503 | 504 | # appendonly no 505 | appendonly yes 506 | 507 | # The name of the append only file (default: "appendonly.aof") 508 | 509 | appendfilename "appendonly.aof" 510 | 511 | # The fsync() call tells the Operating System to actually write data on disk 512 | # instead of waiting for more data in the output buffer. Some OS will really flush 513 | # data on disk, some other OS will just try to do it ASAP. 514 | # 515 | # Redis supports three different modes: 516 | # 517 | # no: don't fsync, just let the OS flush the data when it wants. Faster. 518 | # always: fsync after every write to the append only log. Slow, Safest. 519 | # everysec: fsync only one time every second. Compromise. 520 | # 521 | # The default is "everysec", as that's usually the right compromise between 522 | # speed and data safety. It's up to you to understand if you can relax this to 523 | # "no" that will let the operating system flush the output buffer when 524 | # it wants, for better performances (but if you can live with the idea of 525 | # some data loss consider the default persistence mode that's snapshotting), 526 | # or on the contrary, use "always" that's very slow but a bit safer than 527 | # everysec. 528 | # 529 | # More details please check the following article: 530 | # http://antirez.com/post/redis-persistence-demystified.html 531 | # 532 | # If unsure, use "everysec". 533 | 534 | # appendfsync always 535 | appendfsync everysec 536 | # appendfsync no 537 | 538 | # When the AOF fsync policy is set to always or everysec, and a background 539 | # saving process (a background save or AOF log background rewriting) is 540 | # performing a lot of I/O against the disk, in some Linux configurations 541 | # Redis may block too long on the fsync() call. Note that there is no fix for 542 | # this currently, as even performing fsync in a different thread will block 543 | # our synchronous write(2) call. 544 | # 545 | # In order to mitigate this problem it's possible to use the following option 546 | # that will prevent fsync() from being called in the main process while a 547 | # BGSAVE or BGREWRITEAOF is in progress. 548 | # 549 | # This means that while another child is saving, the durability of Redis is 550 | # the same as "appendfsync none". In practical terms, this means that it is 551 | # possible to lose up to 30 seconds of log in the worst scenario (with the 552 | # default Linux settings). 553 | # 554 | # If you have latency problems turn this to "yes". Otherwise leave it as 555 | # "no" that is the safest pick from the point of view of durability. 556 | 557 | no-appendfsync-on-rewrite no 558 | 559 | # Automatic rewrite of the append only file. 560 | # Redis is able to automatically rewrite the log file implicitly calling 561 | # BGREWRITEAOF when the AOF log size grows by the specified percentage. 562 | # 563 | # This is how it works: Redis remembers the size of the AOF file after the 564 | # latest rewrite (if no rewrite has happened since the restart, the size of 565 | # the AOF at startup is used). 566 | # 567 | # This base size is compared to the current size. If the current size is 568 | # bigger than the specified percentage, the rewrite is triggered. Also 569 | # you need to specify a minimal size for the AOF file to be rewritten, this 570 | # is useful to avoid rewriting the AOF file even if the percentage increase 571 | # is reached but it is still pretty small. 572 | # 573 | # Specify a percentage of zero in order to disable the automatic AOF 574 | # rewrite feature. 575 | 576 | auto-aof-rewrite-percentage 100 577 | auto-aof-rewrite-min-size 64mb 578 | 579 | # An AOF file may be found to be truncated at the end during the Redis 580 | # startup process, when the AOF data gets loaded back into memory. 581 | # This may happen when the system where Redis is running 582 | # crashes, especially when an ext4 filesystem is mounted without the 583 | # data=ordered option (however this can't happen when Redis itself 584 | # crashes or aborts but the operating system still works correctly). 585 | # 586 | # Redis can either exit with an error when this happens, or load as much 587 | # data as possible (the default now) and start if the AOF file is found 588 | # to be truncated at the end. The following option controls this behavior. 589 | # 590 | # If aof-load-truncated is set to yes, a truncated AOF file is loaded and 591 | # the Redis server starts emitting a log to inform the user of the event. 592 | # Otherwise if the option is set to no, the server aborts with an error 593 | # and refuses to start. When the option is set to no, the user requires 594 | # to fix the AOF file using the "redis-check-aof" utility before to restart 595 | # the server. 596 | # 597 | # Note that if the AOF file will be found to be corrupted in the middle 598 | # the server will still exit with an error. This option only applies when 599 | # Redis will try to read more data from the AOF file but not enough bytes 600 | # will be found. 601 | aof-load-truncated yes 602 | 603 | ################################ LUA SCRIPTING ############################### 604 | 605 | # Max execution time of a Lua script in milliseconds. 606 | # 607 | # If the maximum execution time is reached Redis will log that a script is 608 | # still in execution after the maximum allowed time and will start to 609 | # reply to queries with an error. 610 | # 611 | # When a long running script exceeds the maximum execution time only the 612 | # SCRIPT KILL and SHUTDOWN NOSAVE commands are available. The first can be 613 | # used to stop a script that did not yet called write commands. The second 614 | # is the only way to shut down the server in the case a write command was 615 | # already issued by the script but the user doesn't want to wait for the natural 616 | # termination of the script. 617 | # 618 | # Set it to 0 or a negative value for unlimited execution without warnings. 619 | lua-time-limit 5000 620 | 621 | ################################ REDIS CLUSTER ############################### 622 | # 623 | # ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 624 | # WARNING EXPERIMENTAL: Redis Cluster is considered to be stable code, however 625 | # in order to mark it as "mature" we need to wait for a non trivial percentage 626 | # of users to deploy it in production. 627 | # ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 628 | # 629 | # Normal Redis instances can't be part of a Redis Cluster; only nodes that are 630 | # started as cluster nodes can. In order to start a Redis instance as a 631 | # cluster node enable the cluster support uncommenting the following: 632 | # 633 | cluster-enabled yes 634 | 635 | # Every cluster node has a cluster configuration file. This file is not 636 | # intended to be edited by hand. It is created and updated by Redis nodes. 637 | # Every Redis Cluster node requires a different cluster configuration file. 638 | # Make sure that instances running in the same system do not have 639 | # overlapping cluster configuration file names. 640 | # 641 | cluster-config-file nodes-6379.conf 642 | 643 | # Cluster node timeout is the amount of milliseconds a node must be unreachable 644 | # for it to be considered in failure state. 645 | # Most other internal time limits are multiple of the node timeout. 646 | # 647 | # cluster-node-timeout 15000 648 | cluster-node-timeout 3000 649 | 650 | # A slave of a failing master will avoid to start a failover if its data 651 | # looks too old. 652 | # 653 | # There is no simple way for a slave to actually have a exact measure of 654 | # its "data age", so the following two checks are performed: 655 | # 656 | # 1) If there are multiple slaves able to failover, they exchange messages 657 | # in order to try to give an advantage to the slave with the best 658 | # replication offset (more data from the master processed). 659 | # Slaves will try to get their rank by offset, and apply to the start 660 | # of the failover a delay proportional to their rank. 661 | # 662 | # 2) Every single slave computes the time of the last interaction with 663 | # its master. This can be the last ping or command received (if the master 664 | # is still in the "connected" state), or the time that elapsed since the 665 | # disconnection with the master (if the replication link is currently down). 666 | # If the last interaction is too old, the slave will not try to failover 667 | # at all. 668 | # 669 | # The point "2" can be tuned by user. Specifically a slave will not perform 670 | # the failover if, since the last interaction with the master, the time 671 | # elapsed is greater than: 672 | # 673 | # (node-timeout * slave-validity-factor) + repl-ping-slave-period 674 | # 675 | # So for example if node-timeout is 30 seconds, and the slave-validity-factor 676 | # is 10, and assuming a default repl-ping-slave-period of 10 seconds, the 677 | # slave will not try to failover if it was not able to talk with the master 678 | # for longer than 310 seconds. 679 | # 680 | # A large slave-validity-factor may allow slaves with too old data to failover 681 | # a master, while a too small value may prevent the cluster from being able to 682 | # elect a slave at all. 683 | # 684 | # For maximum availability, it is possible to set the slave-validity-factor 685 | # to a value of 0, which means, that slaves will always try to failover the 686 | # master regardless of the last time they interacted with the master. 687 | # (However they'll always try to apply a delay proportional to their 688 | # offset rank). 689 | # 690 | # Zero is the only value able to guarantee that when all the partitions heal 691 | # the cluster will always be able to continue. 692 | # 693 | # cluster-slave-validity-factor 10 694 | 695 | # Cluster slaves are able to migrate to orphaned masters, that are masters 696 | # that are left without working slaves. This improves the cluster ability 697 | # to resist to failures as otherwise an orphaned master can't be failed over 698 | # in case of failure if it has no working slaves. 699 | # 700 | # Slaves migrate to orphaned masters only if there are still at least a 701 | # given number of other working slaves for their old master. This number 702 | # is the "migration barrier". A migration barrier of 1 means that a slave 703 | # will migrate only if there is at least 1 other working slave for its master 704 | # and so forth. It usually reflects the number of slaves you want for every 705 | # master in your cluster. 706 | # 707 | # Default is 1 (slaves migrate only if their masters remain with at least 708 | # one slave). To disable migration just set it to a very large value. 709 | # A value of 0 can be set but is useful only for debugging and dangerous 710 | # in production. 711 | # 712 | # cluster-migration-barrier 1 713 | 714 | # By default Redis Cluster nodes stop accepting queries if they detect there 715 | # is at least an hash slot uncovered (no available node is serving it). 716 | # This way if the cluster is partially down (for example a range of hash slots 717 | # are no longer covered) all the cluster becomes, eventually, unavailable. 718 | # It automatically returns available as soon as all the slots are covered again. 719 | # 720 | # However sometimes you want the subset of the cluster which is working, 721 | # to continue to accept queries for the part of the key space that is still 722 | # covered. In order to do so, just set the cluster-require-full-coverage 723 | # option to no. 724 | # 725 | # cluster-require-full-coverage yes 726 | 727 | # In order to setup your cluster make sure to read the documentation 728 | # available at http://redis.io web site. 729 | 730 | ################################## SLOW LOG ################################### 731 | 732 | # The Redis Slow Log is a system to log queries that exceeded a specified 733 | # execution time. The execution time does not include the I/O operations 734 | # like talking with the client, sending the reply and so forth, 735 | # but just the time needed to actually execute the command (this is the only 736 | # stage of command execution where the thread is blocked and can not serve 737 | # other requests in the meantime). 738 | # 739 | # You can configure the slow log with two parameters: one tells Redis 740 | # what is the execution time, in microseconds, to exceed in order for the 741 | # command to get logged, and the other parameter is the length of the 742 | # slow log. When a new command is logged the oldest one is removed from the 743 | # queue of logged commands. 744 | 745 | # The following time is expressed in microseconds, so 1000000 is equivalent 746 | # to one second. Note that a negative number disables the slow log, while 747 | # a value of zero forces the logging of every command. 748 | slowlog-log-slower-than 10000 749 | 750 | # There is no limit to this length. Just be aware that it will consume memory. 751 | # You can reclaim memory used by the slow log with SLOWLOG RESET. 752 | slowlog-max-len 128 753 | 754 | ################################ LATENCY MONITOR ############################## 755 | 756 | # The Redis latency monitoring subsystem samples different operations 757 | # at runtime in order to collect data related to possible sources of 758 | # latency of a Redis instance. 759 | # 760 | # Via the LATENCY command this information is available to the user that can 761 | # print graphs and obtain reports. 762 | # 763 | # The system only logs operations that were performed in a time equal or 764 | # greater than the amount of milliseconds specified via the 765 | # latency-monitor-threshold configuration directive. When its value is set 766 | # to zero, the latency monitor is turned off. 767 | # 768 | # By default latency monitoring is disabled since it is mostly not needed 769 | # if you don't have latency issues, and collecting data has a performance 770 | # impact, that while very small, can be measured under big load. Latency 771 | # monitoring can easily be enabled at runtime using the command 772 | # "CONFIG SET latency-monitor-threshold " if needed. 773 | latency-monitor-threshold 0 774 | 775 | ############################# EVENT NOTIFICATION ############################## 776 | 777 | # Redis can notify Pub/Sub clients about events happening in the key space. 778 | # This feature is documented at http://redis.io/topics/notifications 779 | # 780 | # For instance if keyspace events notification is enabled, and a client 781 | # performs a DEL operation on key "foo" stored in the Database 0, two 782 | # messages will be published via Pub/Sub: 783 | # 784 | # PUBLISH __keyspace@0__:foo del 785 | # PUBLISH __keyevent@0__:del foo 786 | # 787 | # It is possible to select the events that Redis will notify among a set 788 | # of classes. Every class is identified by a single character: 789 | # 790 | # K Keyspace events, published with __keyspace@__ prefix. 791 | # E Keyevent events, published with __keyevent@__ prefix. 792 | # g Generic commands (non-type specific) like DEL, EXPIRE, RENAME, ... 793 | # $ String commands 794 | # l List commands 795 | # s Set commands 796 | # h Hash commands 797 | # z Sorted set commands 798 | # x Expired events (events generated every time a key expires) 799 | # e Evicted events (events generated when a key is evicted for maxmemory) 800 | # A Alias for g$lshzxe, so that the "AKE" string means all the events. 801 | # 802 | # The "notify-keyspace-events" takes as argument a string that is composed 803 | # of zero or multiple characters. The empty string means that notifications 804 | # are disabled. 805 | # 806 | # Example: to enable list and generic events, from the point of view of the 807 | # event name, use: 808 | # 809 | # notify-keyspace-events Elg 810 | # 811 | # Example 2: to get the stream of the expired keys subscribing to channel 812 | # name __keyevent@0__:expired use: 813 | # 814 | # notify-keyspace-events Ex 815 | # 816 | # By default all notifications are disabled because most users don't need 817 | # this feature and the feature has some overhead. Note that if you don't 818 | # specify at least one of K or E, no events will be delivered. 819 | notify-keyspace-events "" 820 | 821 | ############################### ADVANCED CONFIG ############################### 822 | 823 | # Hashes are encoded using a memory efficient data structure when they have a 824 | # small number of entries, and the biggest entry does not exceed a given 825 | # threshold. These thresholds can be configured using the following directives. 826 | hash-max-ziplist-entries 512 827 | hash-max-ziplist-value 64 828 | 829 | # Similarly to hashes, small lists are also encoded in a special way in order 830 | # to save a lot of space. The special representation is only used when 831 | # you are under the following limits: 832 | list-max-ziplist-entries 512 833 | list-max-ziplist-value 64 834 | 835 | # Sets have a special encoding in just one case: when a set is composed 836 | # of just strings that happen to be integers in radix 10 in the range 837 | # of 64 bit signed integers. 838 | # The following configuration setting sets the limit in the size of the 839 | # set in order to use this special memory saving encoding. 840 | set-max-intset-entries 512 841 | 842 | # Similarly to hashes and lists, sorted sets are also specially encoded in 843 | # order to save a lot of space. This encoding is only used when the length and 844 | # elements of a sorted set are below the following limits: 845 | zset-max-ziplist-entries 128 846 | zset-max-ziplist-value 64 847 | 848 | # HyperLogLog sparse representation bytes limit. The limit includes the 849 | # 16 bytes header. When an HyperLogLog using the sparse representation crosses 850 | # this limit, it is converted into the dense representation. 851 | # 852 | # A value greater than 16000 is totally useless, since at that point the 853 | # dense representation is more memory efficient. 854 | # 855 | # The suggested value is ~ 3000 in order to have the benefits of 856 | # the space efficient encoding without slowing down too much PFADD, 857 | # which is O(N) with the sparse encoding. The value can be raised to 858 | # ~ 10000 when CPU is not a concern, but space is, and the data set is 859 | # composed of many HyperLogLogs with cardinality in the 0 - 15000 range. 860 | hll-sparse-max-bytes 3000 861 | 862 | # Active rehashing uses 1 millisecond every 100 milliseconds of CPU time in 863 | # order to help rehashing the main Redis hash table (the one mapping top-level 864 | # keys to values). The hash table implementation Redis uses (see dict.c) 865 | # performs a lazy rehashing: the more operation you run into a hash table 866 | # that is rehashing, the more rehashing "steps" are performed, so if the 867 | # server is idle the rehashing is never complete and some more memory is used 868 | # by the hash table. 869 | # 870 | # The default is to use this millisecond 10 times every second in order to 871 | # actively rehash the main dictionaries, freeing memory when possible. 872 | # 873 | # If unsure: 874 | # use "activerehashing no" if you have hard latency requirements and it is 875 | # not a good thing in your environment that Redis can reply from time to time 876 | # to queries with 2 milliseconds delay. 877 | # 878 | # use "activerehashing yes" if you don't have such hard requirements but 879 | # want to free memory asap when possible. 880 | activerehashing yes 881 | 882 | # The client output buffer limits can be used to force disconnection of clients 883 | # that are not reading data from the server fast enough for some reason (a 884 | # common reason is that a Pub/Sub client can't consume messages as fast as the 885 | # publisher can produce them). 886 | # 887 | # The limit can be set differently for the three different classes of clients: 888 | # 889 | # normal -> normal clients including MONITOR clients 890 | # slave -> slave clients 891 | # pubsub -> clients subscribed to at least one pubsub channel or pattern 892 | # 893 | # The syntax of every client-output-buffer-limit directive is the following: 894 | # 895 | # client-output-buffer-limit 896 | # 897 | # A client is immediately disconnected once the hard limit is reached, or if 898 | # the soft limit is reached and remains reached for the specified number of 899 | # seconds (continuously). 900 | # So for instance if the hard limit is 32 megabytes and the soft limit is 901 | # 16 megabytes / 10 seconds, the client will get disconnected immediately 902 | # if the size of the output buffers reach 32 megabytes, but will also get 903 | # disconnected if the client reaches 16 megabytes and continuously overcomes 904 | # the limit for 10 seconds. 905 | # 906 | # By default normal clients are not limited because they don't receive data 907 | # without asking (in a push way), but just after a request, so only 908 | # asynchronous clients may create a scenario where data is requested faster 909 | # than it can read. 910 | # 911 | # Instead there is a default limit for pubsub and slave clients, since 912 | # subscribers and slaves receive data in a push fashion. 913 | # 914 | # Both the hard or the soft limit can be disabled by setting them to zero. 915 | client-output-buffer-limit normal 0 0 0 916 | client-output-buffer-limit slave 256mb 64mb 60 917 | client-output-buffer-limit pubsub 32mb 8mb 60 918 | 919 | # Redis calls an internal function to perform many background tasks, like 920 | # closing connections of clients in timeout, purging expired keys that are 921 | # never requested, and so forth. 922 | # 923 | # Not all tasks are performed with the same frequency, but Redis checks for 924 | # tasks to perform according to the specified "hz" value. 925 | # 926 | # By default "hz" is set to 10. Raising the value will use more CPU when 927 | # Redis is idle, but at the same time will make Redis more responsive when 928 | # there are many keys expiring at the same time, and timeouts may be 929 | # handled with more precision. 930 | # 931 | # The range is between 1 and 500, however a value over 100 is usually not 932 | # a good idea. Most users should use the default of 10 and raise this up to 933 | # 100 only in environments where very low latency is required. 934 | hz 10 935 | 936 | # When a child rewrites the AOF file, if the following option is enabled 937 | # the file will be fsync-ed every 32 MB of data generated. This is useful 938 | # in order to commit the file to the disk more incrementally and avoid 939 | # big latency spikes. 940 | aof-rewrite-incremental-fsync yes 941 | --------------------------------------------------------------------------------