├── README.md ├── redis-migrate.py └── requirements.txt /README.md: -------------------------------------------------------------------------------- 1 | Small utility for interactively migrating a bunch of redis servers to another bunch of redis servers. 2 | Run with --help for help. 3 | 4 | [![githalytics.com alpha](https://cruel-carlota.pagodabox.com/e907837e7c5a0bb078bbb3a75e381a3f "githalytics.com")](http://githalytics.com/RedisLabs/redis-migrate) 5 | -------------------------------------------------------------------------------- /redis-migrate.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | import sys 3 | import redis 4 | import argparse 5 | import urlparse 6 | import time 7 | import os 8 | import curses 9 | import signal 10 | 11 | def fail(msg): 12 | print >> sys.stderr, msg 13 | exit(1) 14 | 15 | def redisHost(r): 16 | return r.connection_pool.connection_kwargs['host'] 17 | 18 | def redisPort(r): 19 | return r.connection_pool.connection_kwargs['port'] 20 | 21 | def redisPassword(r): 22 | return r.connection_pool.connection_kwargs['password'] 23 | 24 | def getRedisList(urls): 25 | res = [] 26 | for srcUrl in urls: 27 | url = urlparse.urlparse(srcUrl) 28 | if not url.scheme: 29 | srcUrl = 'redis://' + srcUrl 30 | url = urlparse.urlparse(srcUrl) 31 | if url.scheme != 'redis': 32 | fail('Invalid scheme %s for %s, aborting'%(url.scheme,srcUrl)) 33 | r = redis.Redis(host=url.hostname, port=(url.port if url.port else 6379), password=url.password) 34 | try: 35 | ver = r.info()['redis_version'] 36 | r.ver = ver 37 | except redis.ConnectionError as e: 38 | fail('Failed connecting (%s) to %s, aborting'%(e,srcUrl)) 39 | res.append(r) 40 | return res 41 | 42 | 43 | def writeLn(y, x, txt, attr=0): 44 | stdscr.move(y,0) 45 | stdscr.clrtoeol() 46 | stdscr.move(y,x) 47 | stdscr.addstr(txt, attr) 48 | stdscr.refresh() 49 | 50 | def checkInput(): 51 | c = stdscr.getch() 52 | if c < 0 or c > 255: 53 | return None 54 | c = chr(c).lower() 55 | if c == 'q': 56 | sys.exit() 57 | return c 58 | 59 | def compareVersion(va, vb): 60 | for vaPart,vbPart in zip([int(x) for x in va.split('.')], [int(x) for x in vb.split('.')]): 61 | if vaPart > vbPart: 62 | return 1 63 | elif vaPart < vbPart: 64 | return -1 65 | return 0 66 | 67 | 68 | def signalWinch(signum, frame): 69 | pass 70 | 71 | def valOrNA(x): 72 | return x if x != None else 'N/A' 73 | 74 | def bytesToStr(bytes): 75 | if bytes < 1024: 76 | return '%dB'%bytes 77 | if bytes < 1024*1024: 78 | return '%dKB'%(bytes/1024) 79 | if bytes < 1024*1024*1024: 80 | return '%dMB'%(bytes/(1024*1024)) 81 | return '%dGB'%(bytes/(1024*1024*1024)) 82 | 83 | if __name__ == '__main__': 84 | parser = argparse.ArgumentParser(description='Interactively migrate a bunch of redis servers to another bunch of redis servers.') 85 | parser.add_argument('--src', metavar='src_url', nargs='+', required=True, help='list of source redises to sync from') 86 | parser.add_argument('--dst', metavar='dst_url', nargs='+', required=True, help='list of destination redises to sync to') 87 | 88 | args = parser.parse_args() 89 | 90 | if len(args.src) != len(args.dst): 91 | fail('Number of sources must match number of destinations') 92 | 93 | srcs = getRedisList(args.src) 94 | dsts = getRedisList(args.dst) 95 | 96 | stdscr = curses.initscr() 97 | curses.halfdelay(10) 98 | curses.noecho() 99 | curses.curs_set(0) 100 | 101 | signal.signal(signal.SIGWINCH, signalWinch) 102 | 103 | try: 104 | # Get aggregate sizes from sources 105 | keys = None 106 | mem = 0 107 | for r in srcs: 108 | info = r.info() 109 | mem += info['used_memory'] 110 | if compareVersion(r.ver, '2.6') >= 0: 111 | ks = r.info('keyspace') 112 | if keys == None: 113 | keys = 0 114 | for db in ks: 115 | keys += ks[db]['keys'] 116 | 117 | 118 | writeLn(0, 0, 'Syncing %.2fMB and %s keys from %d redises'%(float(mem)/(1024*1024), valOrNA(keys), len(srcs))) 119 | writeLn(1, 0, 'q - Quit, s - Start', curses.A_BOLD) 120 | while checkInput() != 's': 121 | pass 122 | writeLn(1, 0, 'q - Quit', curses.A_BOLD) 123 | 124 | # Start replication from all slaves 125 | for sr,dr in zip(srcs,dsts): 126 | if compareVersion(dr.ver, '2.6') >= 0: 127 | dr.config_set('slave-read-only', 'yes') 128 | drAuth = dr.config_get('masterauth')['masterauth'] 129 | if redisPassword(sr) != drAuth: # Avoid setting the master auth if not required since on redis 2.2 theres no way to set a null password 130 | dr.config_set('masterauth', redisPassword(sr) or '') 131 | dr.slaveof(redisHost(sr), redisPort(sr)) 132 | 133 | # Wait for dsts to be in sync 134 | while True: 135 | synced = 0 136 | y = 2 137 | for dr,sr in zip(dsts,srcs): 138 | y += 1 139 | info = dr.info() 140 | if info['role'] != 'slave': 141 | writeLn(y, 1, 'Error: dest %s:%s configured as %s'%(redisHost(dr), redisPort(dr), info['role'])) 142 | continue 143 | writeLn(y, 1, '%s:%s ==> %s:%s - link status: %s, sync in progress: %s, %s left, used memory %.2fMB'%(redisHost(sr), redisPort(sr), redisHost(dr), redisPort(dr), info['master_link_status'], 'yes' if info['master_sync_in_progress'] else 'no', bytesToStr(info.get('master_sync_left_bytes', 0)), float(info['used_memory'])/(1024*1024))) 144 | if info['master_link_status'] == 'up': 145 | synced += 1 146 | if synced == len(dsts): 147 | stdscr.move(3,0) 148 | stdscr.clrtobot() 149 | writeLn(3, 1, 'Replication links are up, wait for master replication buffers to flush before disconnecting from sources') 150 | writeLn(1, 0, 'q - Quit, e - Enable writes on destinations', curses.A_BOLD) 151 | break 152 | checkInput() 153 | 154 | # Wait for master client buffers to flush 155 | while True: 156 | y = 5 157 | for dr,sr in zip(dsts,srcs): 158 | maxOutBuff = None 159 | maxOutBuffCommands = None 160 | if compareVersion(sr.ver, '2.4') >= 0: 161 | slaves = [client for client in sr.client_list() if 'S' in client['flags']] 162 | if compareVersion(sr.ver, '2.6') >= 0: 163 | maxOutBuff = max([int(slave['omem']) for slave in slaves]) 164 | maxOutBuffCommands = max([(1 if int(slave['obl']) > 0 else 0) + int(slave['oll']) for slave in slaves]) 165 | readonly = dr.config_get('slave-read-only').get('slave-read-only') if compareVersion(dr.ver, '2.6') >= 0 else 'N/A' 166 | writeLn(y, 1, '%s:%s ==> %s:%s: replication buf size %s, replication buf commands: %s, dst readonly: %s '%(redisHost(sr), redisPort(sr), redisHost(dr), redisPort(dr), bytesToStr(maxOutBuff) if maxOutBuff != None else 'N/A', valOrNA(maxOutBuffCommands), readonly)) 167 | y += 1 168 | c = checkInput() 169 | if c == 'e': 170 | for dr in dsts: 171 | if compareVersion(dr.ver, '2.6') >= 0: 172 | dr.config_set('slave-read-only', 'no') 173 | writeLn(3, 1, 'Replication links are up and writes enabled on destinations, wait for master replication buffers to flush before disconnecting from sources') 174 | writeLn(1, 0, 'q - quit, e - Enable writes on destinations, m - Make destinations masters and quit', curses.A_BOLD) 175 | if c == 'm': 176 | for dr in dsts: 177 | dr.slaveof('no','one') 178 | if compareVersion(dr.ver, '2.6') >= 0: 179 | dr.config_set('slave-read-only', 'no') 180 | if dr.config_get('masterauth')['masterauth']: # Avoid zeroing the master auth if not required, becaues of bug in v2.2 where you can put a null value in the mastaer auth 181 | dr.config_set('masterauth', '') 182 | sys.exit() 183 | 184 | finally: 185 | curses.nocbreak() 186 | curses.echo() 187 | curses.curs_set(1) 188 | curses.endwin() 189 | 190 | 191 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | redis 2 | --------------------------------------------------------------------------------