├── .gitignore ├── hosts.yml ├── hosts2.yml ├── config.yml ├── README.md └── fabfile.py /.gitignore: -------------------------------------------------------------------------------- 1 | fabfile.pyc 2 | -------------------------------------------------------------------------------- /hosts.yml: -------------------------------------------------------------------------------- 1 | --- 2 | sentinels: 3 | 172.22.41.15 4 | 172.22.42.15 5 | 172.22.43.15 6 | redis: 7 | 172.22.41.27 8 | 172.22.41.133 9 | 172.22.42.122 10 | 172.22.42.239 11 | 172.22.43.81 12 | 172.22.43.48 13 | -------------------------------------------------------------------------------- /hosts2.yml: -------------------------------------------------------------------------------- 1 | --- 2 | sentinels: 3 | 172.22.41.15 4 | 172.22.42.15 5 | 172.22.43.15 6 | redis: 7 | 172.22.41.48 8 | 172.22.41.49 9 | 172.22.42.55 10 | 172.22.42.56 11 | 172.22.43.119 12 | 172.22.43.120 13 | -------------------------------------------------------------------------------- /config.yml: -------------------------------------------------------------------------------- 1 | --- 2 | 3 | # SSH user. 'ec2-user' is the default user for AMI based releases. 4 | # Switch to 'ubuntu' or other if necessary. 5 | ssh_user: ec2-user 6 | 7 | ssh_key: ~/.ssh/orchid-prd-master-key.pem 8 | 9 | # Fetch host list. 'static' mode uses hosts.yml file and 10 | # 'dynamic' uses AWS API. 11 | fetch_mode: static 12 | 13 | # When using 'fetch_mode: dynamic', this fabfile will look into your AWS Account 14 | # and seek for a Key:Value tag like: 'Role: redis-sentinel' 15 | 16 | sentinel: 17 | key: Role 18 | value: redis-sentinel 19 | 20 | redis: 21 | key: Role 22 | value: redis-server 23 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # POD-MANAGER 2 | 3 | This simple tool uses Fabric to automate the creation of new Redis Pods running on Docker containers. It can run using a pre-defined hosts list or fetch them dynamically from AWS. 4 | 5 | ## Requirements 6 | 7 | * Python 2.7 8 | * Fabric 9 | 10 | ## Install 11 | 12 | pip install fabric 13 | 14 | ## Configure 15 | 16 | ### config.yml 17 | 18 | * `ssh_user`: the username you are going to use to login in redis instances 19 | * `ssh_key`: path to ssh key 20 | * `fetch_mode`: can be `static` or `dynamic`, depending if you have a static set of hosts (sentinels and redis), or if you are going to fetch them though AWS API. 21 | * `sentinel.key`: the AWS tag key you have defined for sentinels 22 | * `sentinel.value`: the AWS tag value you defined for sentinels 23 | * `redis.key`: same as above but for redis hosts 24 | * `redis.value`: same as above but for redis hosts 25 | 26 | ### hosts.yml 27 | 28 | Use this file if you have set `fetch_mode: static` on `config.yml`. 29 | 30 | Something like: 31 | 32 | --- 33 | sentinels: 34 | 172.22.41.15 35 | 172.22.42.15 36 | 172.22.43.15 37 | redis: 38 | 172.22.41.27 39 | 172.22.41.133 40 | 172.22.42.122 41 | 172.22.42.239 42 | 172.22.43.81 43 | 172.22.43.48 44 | 45 | ## Usage 46 | 47 | ### Task list 48 | 49 | fab -l 50 | 51 | ### New Pod 52 | 53 | fab podName:,, 54 | 55 | _ie:_ 56 | 57 | fab podName:colossus,7190,redis-cluster-2 58 | 59 | ## TODO 60 | 61 | * Discovery using AWS API / boto 62 | * Validations about container name and port 63 | * define more than one cluster on `hosts.yml` 64 | -------------------------------------------------------------------------------- /fabfile.py: -------------------------------------------------------------------------------- 1 | # -*- coding: UTF-8 -*- 2 | 3 | from __future__ import print_function 4 | from pprint import pprint as pp 5 | 6 | import os 7 | import sys 8 | import yaml 9 | import random 10 | 11 | from fabric.api import * 12 | from fabric.state import * 13 | from fabric.utils import * 14 | from fabric.colors import * 15 | from fabric.contrib import * 16 | from fabric.decorators import * 17 | 18 | ''' read config file 19 | ''' 20 | with open('config.yml', 'r') as cf: 21 | conf = yaml.safe_load(cf) 22 | 23 | ''' set environment variables 24 | ''' 25 | env.timeout = 1 26 | env.parallel = False 27 | env.warn_only = True 28 | env.forward_agent = True 29 | env.skip_bad_hosts = True 30 | 31 | ''' set environment variables and other setting based on loaded config.yml 32 | ''' 33 | env.user = conf['ssh_user'] 34 | env.key_filename = conf['ssh_key'] 35 | 36 | fetch_mode = conf['fetch_mode'] 37 | 38 | sentinel_key = conf['sentinel']['key'] 39 | sentinel_value = conf['sentinel']['value'] 40 | 41 | redis_key = conf['redis']['key'] 42 | redis_value = conf['redis']['value'] 43 | 44 | def selectHostFile(cluster): 45 | global env 46 | global master 47 | global slaves 48 | global sentinel_hosts 49 | global redis_hosts 50 | 51 | if cluster == 'redis-cluster': 52 | host_file = "hosts.yml" 53 | elif cluster == 'redis-cluster-2': 54 | host_file = "hosts2.yml" 55 | else: 56 | print(red("ERROR: Missing cluster!")) 57 | sys.exit(1) 58 | 59 | ''' read hosts.yml file and set roles accordingly 60 | ''' 61 | with open(host_file, 'r') as hf: 62 | hosts = yaml.safe_load(hf) 63 | 64 | if not hosts.has_key('sentinels'): 65 | print(red("ERROR: no sentinels section!")) 66 | sys.exit(1) 67 | else: 68 | sentinel_hosts = hosts['sentinels'].split(' ') 69 | print("Sentinel list:", cyan(sentinel_hosts)) 70 | 71 | if not hosts.has_key('redis'): 72 | print(red("ERROR: no redis section!")) 73 | sys.exit(1) 74 | else: 75 | redis_hosts = hosts['redis'].split(' ') 76 | print("Redis list:", cyan(redis_hosts)) 77 | 78 | master = selectMaster(redis_hosts) 79 | slaves = selectSlaves(redis_hosts) 80 | 81 | 82 | # only set roles after electing master and define slave list 83 | env.roledefs = { 84 | 'sentinels': sentinel_hosts, 85 | 'redis': redis_hosts, 86 | 'master': master, 87 | 'slaves': slaves 88 | } 89 | 90 | def selectMaster(redis_hosts): 91 | #pick up a random server to be our pod master 92 | 93 | selectMaster.master = random.choice(redis_hosts) 94 | global master 95 | return selectMaster.master 96 | 97 | def selectSlaves(redis_hosts): 98 | # from redis host list, remove elected master 99 | 100 | slaves = redis_hosts[:] 101 | slaves.remove(master) 102 | return slaves 103 | 104 | @roles('redis') 105 | def runRedis(name, port): 106 | ''' docker run --name redis- -p : -t -d -i redis redis-server --port 107 | ''' 108 | # FIXME: Test before starting container FOO on port INT if there is other container 109 | # already running with the same name and/or using same port 110 | cmd = "sudo docker run --name {name} -p {port}:{port} -t -d -i redis redis-server --port {port}".format(name=name, port=port) 111 | run(cmd) 112 | 113 | def myslaves(port): 114 | ''' 115 | redis-cli -h -p slaveof 116 | redis-cli -h -p config set slave-read-only no 117 | ''' 118 | for slave in slaves: 119 | print(blue("Running slaveof's")) 120 | print(blue(slave)) 121 | slaveofCmd = "redis-cli -h {slave} -p {port} slaveof {master} {port}".format(slave=slave, port=port, master=master) 122 | local(slaveofCmd) 123 | 124 | print(green("Setting slave RW")) 125 | print(green(slave)) 126 | roCmd = "redis-cli -h {slave} -p {port} config set slave-read-only no".format(slave=slave, port=port) 127 | local(roCmd) 128 | 129 | def updateSentinels(name, port): 130 | ''' 131 | redis-cli -h -p 26379 sentinel monitor 2 132 | redis-cli -h -p 26379 sentinel set down-after-milliseconds 1000 133 | redis-cli -h -p 26379 sentinel set failover-timeout 1000 134 | redis-cli -h -p 26379 sentinel set parallel-syncs 1 135 | ''' 136 | print(cyan("Updating sentinels")) 137 | for sentinel in sentinel_hosts: 138 | print(cyan(sentinel)) 139 | sentinelMonitorCmd = "redis-cli -h {sentinel} -p 26379 sentinel monitor {name} {master} {port} 2".format(sentinel=sentinel, name=name, master=master, port=port) 140 | local(sentinelMonitorCmd) 141 | 142 | sentinelSetDownCmd = "redis-cli -h {sentinel} -p 26379 sentinel set {name} down-after-milliseconds 1000".format(sentinel=sentinel, name=name) 143 | local(sentinelSetDownCmd) 144 | 145 | sentinelSetFailoverCmd = "redis-cli -h {sentinel} -p 26379 sentinel set {name} failover-timeout 1000".format(sentinel=sentinel, name=name) 146 | local(sentinelSetFailoverCmd) 147 | 148 | sentinelSetParallelCmd = "redis-cli -h {sentinel} -p 26379 sentinel set {name} parallel-syncs 1".format(sentinel=sentinel, name=name) 149 | local(sentinelSetParallelCmd) 150 | 151 | @task 152 | def podName(name, port, cluster): 153 | ''' pick pod name to use like 'podName:redis-shoemaker,port,redis-cluster-2' 154 | ''' 155 | execute(selectHostFile, cluster); 156 | print("Cluster Name:", red(cluster)) 157 | print("Pod Name:", green(name)) 158 | print("Elected Master:", blue(master)) 159 | print("Slave list:", cyan(slaves)) 160 | 161 | # start redis container on all hosts 162 | execute(runRedis, name, port, hosts=redis_hosts) 163 | 164 | # start slaves 165 | execute(myslaves, port) 166 | 167 | # update sentinels 168 | execute(updateSentinels, name, port) 169 | 170 | @roles('sentinels') 171 | @task 172 | def pingSentinels(): 173 | ''' ping sentinels 174 | ''' 175 | run('hostname') 176 | 177 | @roles('redis') 178 | @task 179 | def pingRedis(): 180 | ''' ping redis 181 | ''' 182 | run('hostname') 183 | 184 | 185 | # DONE: define application name and port 186 | # DONE: run redis- on every node 187 | # DONE: select one node to be the master of this pod 188 | # DONE: point all-1 instances to master (slaveof) 189 | # DONE: set slaves to read-only 190 | # DONE: add pod to sentinels (use master ip) 191 | 192 | # TODO: use boto to fetch host dinamicaly 193 | --------------------------------------------------------------------------------