├── .dockerignore ├── .gitignore ├── .travis.yml ├── Dockerfile ├── LICENSE ├── README.md ├── build-and-push.sh ├── deploy.sh ├── k8s ├── redis-config-map.yaml ├── redis-node-statefulset.yaml ├── redis-nodes-service.yaml ├── redis-readonly-service.yaml └── redis-service.yaml └── sidecar.sh /.dockerignore: -------------------------------------------------------------------------------- 1 | * 2 | !sidecar.sh 3 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .idea -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | sudo: required 2 | language: bash 3 | services: 4 | - docker 5 | env: 6 | global: 7 | - CLOUDSDK_CONTAINER_USE_CLIENT_CERTIFICATE=True 8 | before_deploy: 9 | # Authorize with gcloud 10 | - echo $GCLOUD_ENCODED_CREDS | base64 -d > /tmp/gcloud.json 11 | - gcloud auth activate-service-account $(jq -r ".client_email" /tmp/gcloud.json) --key-file=/tmp/gcloud.json 12 | - > 13 | if [[ -n "$TRAVIS_TAG" ]] ; then 14 | gcloud container clusters get-credentials production --project=commercial-tribe --zone=us-east1-c; 15 | else 16 | gcloud container clusters get-credentials staging --project=commercial-tribe-staging --zone=us-central1-a; 17 | fi 18 | 19 | # Install and configure kubectl 20 | - CLOUDSDK_CORE_DISABLE_PROMPTS=true sudo gcloud components update kubectl --version 142.0.0 21 | - export PATH=/usr/lib/google-cloud-sdk/bin:$PATH 22 | - sudo chown -R $USER /home/travis/.config/gcloud 23 | deploy: 24 | - provider: script 25 | script: ./build-and-push.sh 26 | on: 27 | all_branches: true 28 | - provider: script 29 | script: ./deploy.sh gke_commercial-tribe-staging_us-central1-a_staging develop 30 | on: 31 | branch: develop 32 | - provider: script 33 | script: ./deploy.sh gke_commercial-tribe-staging_us-central1-a_staging staging 34 | on: 35 | branch: master 36 | - provider: script 37 | script: ./deploy.sh gke_commercial-tribe_us-east1-c_production production 38 | on: 39 | tags: true 40 | condition: $TRAVIS_TAG =~ ^v20[0-9]{2}\.[1-4]\.[1-6]\.[0-9]+$ 41 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM redis:3.2 2 | MAINTAINER Jason Waldrip 3 | 4 | ADD https://storage.googleapis.com/kubernetes-release/release/v1.6.0/bin/linux/amd64/kubectl /usr/local/bin/kubectl 5 | RUN chmod +x /usr/local/bin/kubectl 6 | 7 | WORKDIR /app 8 | ADD . /app 9 | RUN chmod +x /app/sidecar.sh 10 | CMD /app/sidecar.sh 11 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2017 CommercialTribe, Inc. 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Redis on Kubernetes as StatefulSet 2 | 3 | The following document describes the deployment of a self-bootstrapping, reliable, 4 | multi-node Redis on Kubernetes. It deploys a master with replicated slaves, as 5 | well as replicated redis sentinels which are use for health checking and failover. 6 | 7 | ## Prerequisites 8 | 9 | This example assumes that you have a Kubernetes cluster installed and running, 10 | and that you have installed the kubectl command line tool somewhere in your path. 11 | Please see the getting started for installation instructions for your platform. 12 | 13 | ### Storage Class 14 | 15 | This makes use of a StorageClass, either create a storage class with the name of 16 | "ssd" or update the StatefulSet to point to to the correct StorageClass. 17 | 18 | ## Running 19 | 20 | To get your cluster up and running simple run: 21 | 22 | `kubectl apply -Rf k8s` 23 | 24 | The cluster will automatically bootstrap itself. 25 | 26 | ### Caveats 27 | 28 | Your pods may not show up in the dashboard. This is because we automatically add 29 | additional labels to the pods to recognize the master. To see the pods within the 30 | dashboard you should look at the redis-nodes service instead. 31 | -------------------------------------------------------------------------------- /build-and-push.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | image="commercialtribe/redis-sentinel-sidecar:v20170425.0" 3 | 4 | login(){ 5 | docker login -u $DOCKER_USERNAME -p $DOCKER_PASSWORD 6 | } 7 | 8 | push(){ 9 | docker push $image 10 | } 11 | 12 | build() { 13 | docker build -t $image . 14 | } 15 | 16 | build && (push || (login && push)) 17 | -------------------------------------------------------------------------------- /deploy.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | set -e 3 | kubectl --context="$1" --namespace="$2" apply -Rf=./k8s --force 4 | kubectl --context="$1" --namespace="$2" delete pods -l name=redis-node 5 | -------------------------------------------------------------------------------- /k8s/redis-config-map.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: ConfigMap 3 | metadata: 4 | name: redis-sentinel 5 | data: 6 | # Use the following file for reference http://download.redis.io/redis-stable/redis.conf 7 | node.conf: | 8 | protected-mode no 9 | port 6379 10 | tcp-backlog 511 11 | loglevel notice 12 | logfile "" 13 | dir /data 14 | -------------------------------------------------------------------------------- /k8s/redis-node-statefulset.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: apps/v1beta1 2 | kind: StatefulSet 3 | metadata: 4 | name: redis 5 | annotations: 6 | service.alpha.kubernetes.io/tolerate-unready-endpoints: "true" 7 | spec: 8 | serviceName: redis-nodes 9 | replicas: 3 10 | template: 11 | metadata: 12 | labels: 13 | name: redis-node 14 | spec: 15 | terminationGracePeriodSeconds: 10 16 | containers: 17 | # Redis 18 | - name: redis-node 19 | image: redis:3.2 20 | command: 21 | - redis-server 22 | args: 23 | - /config/node.conf 24 | ports: 25 | - name: redis 26 | containerPort: 6379 27 | volumeMounts: 28 | - name: data 29 | mountPath: /data 30 | - name: redis-config 31 | mountPath: /config 32 | resources: 33 | requests: 34 | cpu: 100m 35 | memory: 1Gi 36 | livenessProbe: &healthcheck 37 | exec: 38 | command: [ "redis-cli", "ping" ] 39 | readinessProbe: 40 | <<: *healthcheck 41 | 42 | # Sentinel 43 | - name: redis-sentinel 44 | image: redis:3.2 45 | command: [ "bash", "-c", "touch sentinel.conf && redis-sentinel sentinel.conf" ] 46 | ports: 47 | - name: sentinel 48 | containerPort: 26379 49 | resources: 50 | requests: 51 | cpu: 25m 52 | memory: 50Mi 53 | livenessProbe: &healthcheck 54 | exec: 55 | command: [ "redis-cli", "-p", "26379", "ping" ] 56 | readinessProbe: 57 | <<: *healthcheck 58 | 59 | # Sidecar 60 | - name: redis-sidecar 61 | image: commercialtribe/redis-sentinel-sidecar:v20170425.0 62 | imagePullPolicy: Always 63 | env: 64 | - name: POD_NAMESPACE 65 | valueFrom: 66 | fieldRef: 67 | fieldPath: metadata.namespace 68 | volumeMounts: 69 | - name: pod-info 70 | mountPath: /etc/pod-info 71 | readinessProbe: 72 | exec: 73 | command: [ "cat", "booted" ] 74 | resources: 75 | requests: 76 | cpu: 25m 77 | memory: 50Mi 78 | 79 | volumes: 80 | - name: pod-info 81 | downwardAPI: 82 | items: 83 | - path: labels 84 | fieldRef: 85 | fieldPath: metadata.labels 86 | - name: redis-config 87 | configMap: 88 | name: redis-sentinel 89 | volumeClaimTemplates: 90 | - metadata: 91 | name: data 92 | annotations: 93 | volume.beta.kubernetes.io/storage-class: "ssd" 94 | gcp-auto-backup: "yes" 95 | spec: 96 | accessModes: [ "ReadWriteOnce" ] 97 | resources: 98 | requests: 99 | storage: 20Gi 100 | -------------------------------------------------------------------------------- /k8s/redis-nodes-service.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Service 3 | metadata: 4 | name: redis-nodes 5 | labels: 6 | service: redis 7 | spec: 8 | clusterIP: None 9 | ports: 10 | - port: 6379 11 | name: redis 12 | selector: 13 | name: redis-node 14 | -------------------------------------------------------------------------------- /k8s/redis-readonly-service.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Service 3 | metadata: 4 | name: redis-readonly 5 | labels: 6 | service: redis 7 | spec: 8 | ports: 9 | - port: 6379 10 | name: redis 11 | selector: 12 | name: redis-node 13 | role: slave 14 | -------------------------------------------------------------------------------- /k8s/redis-service.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Service 3 | metadata: 4 | name: redis 5 | labels: 6 | service: redis 7 | spec: 8 | ports: 9 | - port: 6379 10 | name: redis 11 | selector: 12 | name: redis-node 13 | role: master 14 | -------------------------------------------------------------------------------- /sidecar.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | set -e 3 | [[ $DEBUG == "true" ]] && set -x 4 | 5 | # Set vars 6 | ip=${POD_IP-`hostname -i`} # ip address of pod 7 | redis_port=${NODE_PORT_NUMBER-6379} # redis port 8 | sentinel_port=${SENTINEL_PORT_NUMBER-26379} # sentinel port 9 | group_name="$POD_NAMESPACE-$(hostname | sed 's/-[0-9]$//')" # master group name 10 | quorum="${SENTINEL_QUORUM-2}" # quorum needed 11 | 12 | # Sentinel options 13 | down_after_milliseconds=${DOWN_AFTER_MILLESECONDS-1000} 14 | failover_timeout=${FAILOVER_TIMEOUT-$(($down_after_milliseconds * 10))} 15 | parallel_syncs=${PARALEL_SYNCS-1} 16 | 17 | # Get all the kubernetes pods 18 | labels=`echo $(cat /etc/pod-info/labels) | tr -d '"' | tr " " ","` 19 | 20 | try_step_interval=${TRY_STEP_INTERVAL-"1"} 21 | max_tries=${MAX_TRIES-"3"} 22 | retry() { 23 | local tries=0 24 | until $@ ; do 25 | status=$? 26 | tries=$(($tries + 1)) 27 | if [ $tries -gt $max_tries ] ; then 28 | echoerr "Failed to run \`$@\` after $max_tries tries..." 29 | return $status 30 | fi 31 | sleepsec=$(($tries * $try_step_interval)) 32 | echoerr "Failed: \`$@\`, retyring in $sleepsec seconds..." 33 | sleep $sleepsec 34 | done 35 | return $? 36 | } 37 | 38 | cli(){ 39 | retry timeout 5 redis-cli -p $redis_port $@ 40 | } 41 | 42 | sentinel-cli(){ 43 | retry timeout 5 redis-cli -p $sentinel_port $@ 44 | } 45 | 46 | ping() { 47 | cli ping > /dev/null 48 | } 49 | 50 | ping-sentinel() { 51 | sentinel-cli ping > /dev/null 52 | } 53 | 54 | ping-both(){ 55 | ping && ping-sentinel 56 | } 57 | 58 | role() { 59 | host=${1-"127.0.0.1"} 60 | (cli -h $host info || echo -n "role:none") | grep "role:" | sed "s/role://" | tr -d "\n" | tr -d "\r" 61 | } 62 | 63 | become-slave-of() { 64 | host=$1 65 | cli slaveof $host $redis_port 66 | sentinel-monitor $1 67 | } 68 | 69 | sentinel-monitor() { 70 | host=$1 71 | sentinel-cli sentinel monitor $group_name $host $redis_port $quorum 72 | sentinel-cli sentinel set $group_name down-after-milliseconds $down_after_milliseconds 73 | sentinel-cli sentinel set $group_name failover-timeout $failover_timeout 74 | sentinel-cli sentinel set $group_name parallel-syncs $parallel_syncs 75 | } 76 | 77 | active-master(){ 78 | master="" 79 | for host in `hosts` ; do 80 | if [[ `role $host` = "master" ]] ; then 81 | master=$host 82 | break 83 | fi 84 | done 85 | echo -n $master 86 | } 87 | 88 | hosts(){ 89 | echo "" 90 | kubectl get pods -l=$labels \ 91 | --template="{{range \$i, \$e :=.items}}{{\$e.status.podIP}} {{end}}" \ 92 | | sed "s/ $//" | tr " " "\n" | grep -E "^[0-9]" | grep --invert-match $ip 93 | } 94 | 95 | # Boot the sidecar 96 | boot(){ 97 | # set roll label to "none" 98 | set-role-label "none" 99 | 100 | # wait, as things may still be failing over 101 | sleep $(($failover_timeout / 1000)) 102 | 103 | # Check to ensure both the sentinel and redis are up, 104 | # if not, exit with an error 105 | ping-both || panic "redis and/or sentinel is not up" 106 | 107 | # Store the current active-master to a variable 108 | master=$(active-master) 109 | 110 | if [[ -n "$master" ]] ; then 111 | # There is a master, become a slave 112 | become-slave-of $master 113 | else 114 | # There is not active master, so become the master 115 | sentinel-monitor $ip 116 | fi 117 | echo "Ready!" 118 | touch booted 119 | } 120 | 121 | # Set the role label on the pod to the specified value 122 | set-role-label(){ 123 | kubectl label --overwrite pods `hostname` role=$1 124 | } 125 | 126 | # Print a message to stderr 127 | echoerr () { 128 | >&2 echo $1 129 | } 130 | 131 | # Exit, printing an error message 132 | panic () { 133 | echoerr $1 134 | exit 1 135 | } 136 | 137 | monitor-label(){ 138 | last_role=none 139 | while true ; do 140 | # Check to ensure both the sentinel and redis are up, 141 | # if not, exit with an error 142 | ping-both || panic "redis and/or sentinel is not up" 143 | 144 | # Store the current role to a variable 145 | current_role=`role` 146 | 147 | # Monitor the role, if it changes, set the label accordingly 148 | if [[ "$last_role" != "$current_role" ]] ; then 149 | set-role-label $current_role 150 | last_role=$current_role 151 | fi 152 | 153 | # Don't ever allow multiple masters 154 | if [ "$current_role" = "master" ] ; then 155 | if [ `active-master` != $ip ] ; then 156 | # If I am a master and not the active one, then just become a slave 157 | become-slave-of $master 158 | fi 159 | fi 160 | sleep 1 161 | done 162 | } 163 | 164 | boot 165 | monitor-label 166 | --------------------------------------------------------------------------------