├── MANIFEST.in ├── screenshot.png ├── redisclu ├── cli │ ├── __init__.py │ ├── helper.py │ └── commands.py ├── failover.py ├── __init__.py ├── exceptions.py ├── utils.py ├── node.py └── cluster.py ├── setup.py ├── LICENSE └── README.md /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include README.md 2 | include LICENSE 3 | -------------------------------------------------------------------------------- /screenshot.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/baranbartu/redis-clu/HEAD/screenshot.png -------------------------------------------------------------------------------- /redisclu/cli/__init__.py: -------------------------------------------------------------------------------- 1 | from helper import CommandParser 2 | from commands import load_commands 3 | 4 | 5 | def main(): 6 | parser = CommandParser(fromfile_prefix_chars='@') 7 | for command in load_commands(): 8 | parser.add_command(command) 9 | 10 | parser.run() 11 | 12 | 13 | if __name__ == "__main__": 14 | main() 15 | -------------------------------------------------------------------------------- /redisclu/failover.py: -------------------------------------------------------------------------------- 1 | import time 2 | from redis.exceptions import TimeoutError 3 | 4 | from utils import echo 5 | 6 | 7 | def on_timeout(f): 8 | def wrapper(*args, **kwargs): 9 | for _ in range(100): 10 | try: 11 | return f(*args, **kwargs) 12 | except TimeoutError as e: 13 | args[0].attempts.append(e) 14 | echo('Timeout error reading from socket. ' 15 | 'Trying again in 10 seconds.', 16 | color='red') 17 | time.sleep(10) 18 | raise 19 | 20 | return wrapper 21 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | import os 5 | from setuptools import setup, find_packages 6 | 7 | README = open(os.path.join(os.path.dirname(__file__), 'README.md')).read() 8 | 9 | # allow setup.py to be run from any path 10 | os.chdir(os.path.normpath(os.path.join(os.path.abspath(__file__), os.pardir))) 11 | 12 | version = '0.0.7' 13 | 14 | setup( 15 | name='redis-clu', 16 | version=version, 17 | description='Redis Cluster Management Tool', 18 | long_description=README, 19 | url='https://github.com/baranbartu/redis-clu', 20 | download_url='https://github.com/baranbartu/redis-clu/tarball/%s' % version, 21 | author='Baran Bartu Demirci', 22 | author_email='bbartu.demirci@gmail.com', 23 | license='MIT', 24 | keywords='redis cluster management replication master slave', 25 | packages=find_packages(), 26 | entry_points={ 27 | 'console_scripts': ['redis-clu = redisclu.cli:main'] 28 | }, 29 | install_requires=[ 30 | 'hiredis', 31 | 'redis', 32 | 'futures==3.0.3', 33 | ] 34 | ) 35 | -------------------------------------------------------------------------------- /redisclu/__init__.py: -------------------------------------------------------------------------------- 1 | from redis import connection 2 | from redis.exceptions import (ResponseError) 3 | from exceptions import (AskError, MovedError) 4 | 5 | import logging 6 | 7 | logging.basicConfig() 8 | 9 | CUSTOM_EXCEPTION_CLASSES = { 10 | 'ASK': AskError, 11 | 'MOVED': MovedError, 12 | } 13 | 14 | 15 | def parse_error(self, response): 16 | "Parse an error response" 17 | error_code = response.split(' ')[0] 18 | if error_code in self.EXCEPTION_CLASSES: 19 | response = response[len(error_code) + 1:] 20 | exception_class = self.EXCEPTION_CLASSES[error_code] 21 | if isinstance(exception_class, dict): 22 | for reason, inner_exception_class in exception_class.items(): 23 | if reason in response: 24 | return inner_exception_class(response) 25 | return ResponseError(response) 26 | return exception_class(response) 27 | return ResponseError(response) 28 | 29 | 30 | connection.BaseParser.EXCEPTION_CLASSES.update(CUSTOM_EXCEPTION_CLASSES) 31 | connection.BaseParser.parse_error = parse_error 32 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2016, Baran Bartu Demirci 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in 13 | all copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 21 | THE SOFTWARE. 22 | -------------------------------------------------------------------------------- /redisclu/exceptions.py: -------------------------------------------------------------------------------- 1 | from redis.exceptions import ( 2 | ResponseError 3 | ) 4 | 5 | 6 | class RedisCluException(Exception): 7 | pass 8 | 9 | 10 | class AskError(ResponseError): 11 | """ 12 | partially keys is slot migrated to another node 13 | 14 | src node: MIGRATING to dst node 15 | get > ASK error 16 | ask dst node > ASKING command 17 | dst node: IMPORTING from src node 18 | asking command only affects next command 19 | any op will be allowed after asking command 20 | """ 21 | 22 | def __init__(self, resp): 23 | """should only redirect to master node""" 24 | self.args = (resp,) 25 | self.message = resp 26 | slot_id, new_node = resp.split(' ') 27 | host, port = new_node.rsplit(':', 1) 28 | self.slot_id = int(slot_id) 29 | self.node_addr = self.host, self.port = host, int(port) 30 | 31 | 32 | class MovedError(AskError): 33 | """ 34 | all keys in slot migrated to another node 35 | """ 36 | pass 37 | 38 | 39 | class ClusterNotHealthy(RedisCluException): 40 | pass 41 | 42 | 43 | class ClusterNotConsistent(RedisCluException): 44 | pass -------------------------------------------------------------------------------- /redisclu/utils.py: -------------------------------------------------------------------------------- 1 | from __future__ import print_function 2 | 3 | import itertools 4 | import os 5 | import sys 6 | 7 | COLOR_MAP = { 8 | "red": 31, 9 | "green": 32, 10 | "yellow": 33, 11 | "blue": 34, 12 | "purple": 35 13 | } 14 | 15 | 16 | def echo(*values, **kwargs): 17 | end = kwargs.get("end", '\n') 18 | color = kwargs.get("color", None) 19 | bold = 0 if kwargs.get("bold", False) is False else 1 20 | disable = kwargs.get("diable", False) 21 | 22 | if disable: 23 | return 24 | 25 | msg = ' '.join(str(v) for v in values) + end 26 | 27 | if not color or os.getenv("ANSI_COLORS_DISABLED") is not None: 28 | sys.stdout.write(msg) 29 | else: 30 | color_prefix = "\033[{};{}m".format(bold, COLOR_MAP[color]) 31 | color_suffix = "\033[0m" 32 | sys.stdout.write(color_prefix + msg + color_suffix) 33 | sys.stdout.flush() 34 | 35 | 36 | def divide(n, m): 37 | """Divide integer n to m chunks 38 | """ 39 | avg = int(n / m) 40 | remain = n - m * avg 41 | data = list(itertools.repeat(avg, m)) 42 | for i in range(len(data)): 43 | if not remain: 44 | break 45 | data[i] += 1 46 | remain -= 1 47 | return data 48 | 49 | 50 | def spread(nodes, n): 51 | """Distrubute master instances in different nodes 52 | 53 | { 54 | "192.168.0.1": [node1, node2], 55 | "192.168.0.2": [node3, node4], 56 | "192.168.0.3": [node5, node6] 57 | } => [node1, node3, node5] 58 | """ 59 | target = [] 60 | 61 | while len(target) < n and nodes: 62 | for ip, node_group in list(nodes.items()): 63 | if not node_group: 64 | nodes.pop(ip) 65 | continue 66 | target.append(node_group.pop(0)) 67 | if len(target) >= n: 68 | break 69 | return target -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # redis-clu 2 | Redis Cluster Management Tool 3 | 4 | Create/Manage sharded+replicated redis cluster easily. 5 | 6 | # Installation 7 | 8 | ```bash 9 | pip install redis-clu 10 | ``` 11 | 12 | # Usage 13 | 14 | ##### Create cluster 15 | 16 | ```bash 17 | # Sharded cluster (master-master) 18 | redis-clu create localhost:6376 localhost:6377 localhost:6378 19 | ``` 20 | 21 | 22 | ##### Show status 23 | 24 | ```bash 25 | redis-clu status localhost:6376 26 | ``` 27 | 28 | 29 | ##### Add masters 30 | 31 | ```bash 32 | # single node: 33 | # 34 | redis-clu add localhost:6376 localhost:6379 35 | (optional: --keyMigrationCount ) pipelined command, default 1 36 | 37 | # multiple nodes: 38 | # recommended for dynamic scaling, it will be split cluster into subclusters 39 | # and each subcluster will be resharding simultaneously 40 | # 41 | redis-clu add_multi localhost:6376 localhost:6381 localhost:6382 42 | (optional: --keyMigrationCount ) pipelined command, default 1 43 | ``` 44 | 45 | 46 | ##### Add slaves 47 | 48 | ```bash 49 | # master-slave replication 50 | # To make redis cluster high available, all master should have at least one slave. 51 | # 52 | redis-clu replicate localhost:6376 localhost:6385 53 | ``` 54 | 55 | 56 | ##### Fix cluster 57 | 58 | ```bash 59 | redis-clu fix localhost:6376 60 | # If occurs "Slot is already busy" execute with "--force 1" 61 | redis-clu fix localhost:6376 --force 1 62 | ``` 63 | 64 | 65 | ##### Reshard cluster (Slot balancing) 66 | 67 | ```bash 68 | redis-clu reshard localhost:6376 69 | (optional: --keyMigrationCount ) pipelined command, default 1 70 | ``` 71 | 72 | 73 | ##### Remove node 74 | 75 | ```bash 76 | # 77 | redis-clu remove localhost:6376 localhost:6380 78 | (optional: --keyMigrationCount ) pipelined command, default 1 79 | ``` 80 | 81 | 82 | ##### Flush/Destroy cluster 83 | 84 | ```bash 85 | # flush cluster (initialize with 0 keys): 86 | redis-clu reset localhost:6376 87 | 88 | # destroy cluster: 89 | redis-clu reset localhost:6376 --hard 1 90 | ``` 91 | 92 | 93 | # Monitoring 94 | 95 | Also you can make your own basic monitoring screen using 'watch'. 96 | 97 | brew install watch (For Mac OSx) 98 | redis-clu status 99 | watch -d -n 1 'redis-clu status localhost:6376' 100 | 101 | Monitoring will help you to make an action. 102 | 103 | ![ScreenShot](https://raw.github.com/baranbartu/redis-clu/master/screenshot.png) 104 | 105 | 106 | -------------------------------------------------------------------------------- /redisclu/cli/helper.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import collections 3 | from redisclu.node import Node 4 | from redisclu.cluster import Cluster 5 | from redisclu.utils import echo, spread, divide 6 | 7 | 8 | class Context(object): 9 | def __init__(self, parser=None): 10 | self.__parser = parser 11 | 12 | def abort(self, message): 13 | if not self.__parser: 14 | return 15 | 16 | self.__parser.error(message) 17 | 18 | 19 | class Command(object): 20 | def __init__(self, args, func): 21 | self.arguments = args 22 | self.parser = None 23 | self.name = func.__name__ 24 | self.doc = func.__doc__ 25 | self.func = func 26 | 27 | def callback(self, args): 28 | ctx = Context(self.parser) 29 | if hasattr(self.func, '__pass_ctx__'): 30 | self.func(ctx, args) 31 | else: 32 | self.func(args) 33 | 34 | @classmethod 35 | def command(cls, func): 36 | if not hasattr(func, '__cmd_args__'): 37 | func.__cmd_args__ = [] 38 | func.__cmd_args__.reverse() 39 | return cls(func.__cmd_args__, func) 40 | 41 | @classmethod 42 | def pass_ctx(cls, func): 43 | func.__pass_ctx__ = True 44 | return func 45 | 46 | @classmethod 47 | def argument(cls, *args, **kwargs): 48 | def deco(func): 49 | if not hasattr(func, '__cmd_args__'): 50 | func.__cmd_args__ = [] 51 | func.__cmd_args__.append((args, kwargs)) 52 | return func 53 | 54 | return deco 55 | 56 | def __call__(self): 57 | self.parser = argparse.ArgumentParser() 58 | for args, kwargs in self.arguments: 59 | self.parser.add_argument(*args, **kwargs) 60 | args = self.parser.parse_args() 61 | self.callback(args) 62 | 63 | 64 | class CommandParser(object): 65 | def __init__(self, *args, **kwargs): 66 | self.parser = argparse.ArgumentParser(*args, **kwargs) 67 | self.subparser = self.parser.add_subparsers(title='Subcommands') 68 | 69 | def add_command(self, command): 70 | parser = self.subparser.add_parser(command.name, help=command.doc) 71 | command.parser = parser 72 | for args, kwargs in command.arguments: 73 | parser.add_argument(*args, **kwargs) 74 | parser.set_defaults(func=command.callback) 75 | 76 | def run(self): 77 | args = self.parser.parse_args() 78 | args.func(args) 79 | 80 | 81 | command = Command.command 82 | argument = Command.argument 83 | pass_ctx = Command.pass_ctx 84 | 85 | 86 | class MasterCandidate(object): 87 | def __init__(self, master): 88 | self.master = master 89 | self.unbinded_slots = [] 90 | 91 | def __getattr__(self, attr): 92 | return getattr(self.master, attr) 93 | 94 | def bind_slots(self): 95 | assert self.unbinded_slots 96 | for chunk in self.unbinded_slots: 97 | self.add_slots(*range(*chunk)) 98 | 99 | def is_enabled(self): 100 | return self.unbinded_slots 101 | 102 | 103 | class ClusterCreator(object): 104 | def __init__(self, masters): 105 | master_candidates = [Node.from_uri(i) for i in masters] 106 | self.master_candidates = [MasterCandidate(i) for i in 107 | master_candidates] 108 | self.masters = [] 109 | self.cluster = None 110 | 111 | def check(self): 112 | # check pre-requirements 113 | if len(self.master_candidates) < 2: 114 | return False 115 | for master in self.master_candidates: 116 | if not master.info().get('cluster_enabled'): 117 | return False 118 | master.execute_command('select', '0') 119 | if master.randomkey(): 120 | return False 121 | if master.cluster_info()['cluster_known_nodes'] != 1: 122 | return False 123 | return True 124 | 125 | def initialize_slots(self): 126 | ips = collections.defaultdict(list) 127 | for candidate in self.master_candidates: 128 | ips[candidate.host].append(candidate) 129 | master_count = len(self.master_candidates) 130 | self.masters = masters = spread(ips, master_count) 131 | chunks = self.split_slot(Cluster.CLUSTER_HASH_SLOTS, master_count) 132 | for master, chunk in zip(masters, chunks): 133 | master.unbinded_slots.append(chunk) 134 | 135 | self.master_candidates = [i for i in self.master_candidates if 136 | i.is_enabled()] 137 | self.cluster = Cluster(self.master_candidates) 138 | 139 | def show_cluster_info(self): 140 | for instance in self.master_candidates: 141 | echo('M', end='', color='green') 142 | name_msg = ': {name} {host}:{port}' 143 | echo(name_msg.format(name=instance.name, 144 | host=instance.host, port=instance.port)) 145 | slot_msg = ','.join(['-'.join([str(s[0]), str(s[1] - 1)]) 146 | for s in instance.unbinded_slots]) 147 | echo('\tslots:', slot_msg) 148 | 149 | def bind_slots(self): 150 | for master in self.masters: 151 | master.bind_slots() 152 | 153 | def join_cluster(self): 154 | if not self.masters: 155 | return 156 | 157 | first_master = self.masters[0] 158 | for master in self.masters[1:]: 159 | master.meet(first_master.host, first_master.port) 160 | 161 | def bind_config_epoch(self): 162 | epoch = 1 163 | for instance in self.masters: 164 | try: 165 | instance.set_config_epoch(epoch) 166 | except: 167 | pass 168 | epoch += 1 169 | 170 | def split_slot(self, n, m): 171 | chunks = divide(n, m) 172 | 173 | res, total = [], 0 174 | for c in chunks: 175 | res.append((total, c + total)) 176 | total += c 177 | return res 178 | -------------------------------------------------------------------------------- /redisclu/node.py: -------------------------------------------------------------------------------- 1 | import socket 2 | import redis 3 | import urlparse 4 | 5 | from exceptions import (AskError, MovedError) 6 | from utils import echo 7 | 8 | 9 | class Node(object): 10 | ignored_exceptions = (AskError, MovedError) 11 | 12 | def __init__(self, host='localhost', port=6379, socket_timeout=4): 13 | self.host = socket.gethostbyname(host) 14 | self.port = port 15 | self.redis = redis.Redis(host, port, socket_timeout=socket_timeout) 16 | self.redis.ping() 17 | self.pipeline = self.redis.pipeline(transaction=False) 18 | self.attempts = [] 19 | 20 | @classmethod 21 | def from_uri(cls, uri): 22 | if not uri.startswith('redis://'): 23 | uri = 'redis://{}'.format(uri) 24 | d = urlparse.urlparse(uri) 25 | return cls(d.hostname, d.port) 26 | 27 | def __repr__(self): 28 | return 'Node<{}:{}>'.format(self.host, self.port) 29 | 30 | def __getattr__(self, attr): 31 | return getattr(self.redis, attr) 32 | 33 | def execute_command(self, *args, **kwargs): 34 | return self.redis.execute_command(*args, **kwargs) 35 | 36 | def is_slave(self, master_id=None): 37 | info = self.node_info 38 | 39 | r = 'slave' in info['flags'] 40 | if master_id is not None: 41 | r = r and info['replicate'] == master_id 42 | return r 43 | 44 | def is_master(self): 45 | return 'master' in self.node_info['flags'] 46 | 47 | @property 48 | def node_info(self): 49 | nodes = self.nodes() 50 | return nodes[0] 51 | 52 | @property 53 | def slots(self): 54 | return self.node_info['slots'] 55 | 56 | @property 57 | def name(self): 58 | return self.node_info['name'] 59 | 60 | def migrate_keys(self, host, port, keys): 61 | for key in keys: 62 | self.pipeline.execute_command('MIGRATE', host, port, key, 0, 15000) 63 | return self.pipeline.execute(raise_on_error=False) 64 | 65 | def migrate_slot(self, dst, slot, cluster): 66 | dst.set_slot('IMPORTING', slot, self.name) 67 | self.set_slot('MIGRATING', slot, dst.name) 68 | 69 | total_keys = 0 70 | for keys in self._scan_keys(slot, cluster.key_migration_count): 71 | results = self.migrate_keys(dst.host, dst.port, keys) 72 | self.attempts.extend(filter(lambda r: any( 73 | isinstance(r, e) for e in self.ignored_exceptions), results)) 74 | total_keys += len(keys) 75 | 76 | echo('{} key(s) migrated from {} to {} in slot {}'.format( 77 | total_keys, self, dst, slot)) 78 | 79 | cluster.update_slot_mapping(slot, dst.name) 80 | 81 | def reset(self, hard=False): 82 | args = [] 83 | if hard: 84 | args = ['HARD'] 85 | return self.execute_command('CLUSTER RESET', *args) 86 | 87 | def set_slot(self, action, slot, node_id=None): 88 | remain = [node_id] if node_id else [] 89 | return self.execute_command('CLUSTER SETSLOT', slot, action, *remain) 90 | 91 | def get_keys_in_slot(self, slot, count): 92 | return self.execute_command('CLUSTER GETKEYSINSLOT', slot, count) 93 | 94 | def count_keys_in_slot(self, slot): 95 | return self.execute_command('CLUSTER COUNTKEYSINSLOT', slot) 96 | 97 | def slaves(self, node_id): 98 | data = self.execute_command('CLUSTER SLAVES', node_id) 99 | return self._parse_node('\n'.join(data)) 100 | 101 | def add_slots(self, *slot): 102 | if not slot: 103 | return 104 | self.execute_command('CLUSTER ADDSLOTS', *slot) 105 | 106 | def forget(self, node_id): 107 | return self.execute_command('CLUSTER FORGET', node_id) 108 | 109 | def set_config_epoch(self, config_epoch): 110 | return self.execute_command('CLUSTER SET-CONFIG-EPOCH', config_epoch) 111 | 112 | def meet(self, ip, port): 113 | return self.execute_command('CLUSTER MEET', ip, port) 114 | 115 | def replicate(self, node_id): 116 | return self.execute_command('CLUSTER REPLICATE', node_id) 117 | 118 | def nodes(self): 119 | info = self.execute_command('CLUSTER NODES').strip() 120 | return self._parse_node(info) 121 | 122 | def cluster_info(self): 123 | data = {} 124 | info = self.execute_command('CLUSTER INFO').strip() 125 | for item in info.split('\r\n'): 126 | k, v = item.split(':') 127 | if k != 'cluster_state': 128 | v = int(v) 129 | data[k] = v 130 | return data 131 | 132 | def _parse_node(self, nodes): 133 | data = [] 134 | for item in nodes.split('\n'): 135 | if not item: 136 | continue 137 | confs = item.split() 138 | node_info = { 139 | 'name': confs[0], 140 | 'addr': confs[1], 141 | 'flags': confs[2].split(','), 142 | 'replicate': confs[3], # master_id 143 | 'ping_sent': int(confs[4]), 144 | 'ping_recv': int(confs[5]), 145 | 'link_status': confs[7], 146 | 'migrating': {}, 147 | 'importing': {}, 148 | 'slots': [] 149 | } 150 | for slot in confs[8:]: 151 | if slot[0] == '[': 152 | if '->-' in slot: 153 | s, dst = slot[1:-1].split('->-') 154 | node_info['migrating'][s] = dst 155 | elif '-<-' in slot: 156 | s, src = slot[1:-1].split('-<-') 157 | node_info['importing'][s] = src 158 | elif '-' in slot: 159 | start, end = slot.split('-') 160 | node_info['slots'].extend(range(int(start), int(end) + 1)) 161 | else: 162 | node_info['slots'].append(int(slot)) 163 | 164 | if 'myself' in node_info['flags']: 165 | data.insert(0, node_info) 166 | else: 167 | data.append(node_info) 168 | return data 169 | 170 | def _scan_keys(self, slot, count): 171 | while True: 172 | keys = self.get_keys_in_slot(slot, count) 173 | if not keys: 174 | break 175 | yield keys 176 | 177 | 178 | class MockNode(object): pass 179 | -------------------------------------------------------------------------------- /redisclu/cluster.py: -------------------------------------------------------------------------------- 1 | import time 2 | import itertools 3 | import hashlib 4 | import failover 5 | 6 | from node import Node 7 | from exceptions import (ClusterNotHealthy, ClusterNotConsistent) 8 | from utils import (divide, echo) 9 | 10 | 11 | class Cluster(object): 12 | CLUSTER_HASH_SLOTS = 16384 13 | 14 | def __init__(self, nodes, hash_slots=CLUSTER_HASH_SLOTS, 15 | parent_nodes=None): 16 | self.nodes = nodes 17 | self.parent_nodes = parent_nodes if parent_nodes else nodes 18 | self.CLUSTER_HASH_SLOTS = hash_slots 19 | self.attempts = [] 20 | self.key_migration_count = 1 21 | 22 | @classmethod 23 | def from_node(cls, node): 24 | nodes = [Node.from_uri(i['addr']) for i in node.nodes() 25 | if i['link_status'] != 'disconnected'] 26 | return cls(nodes) 27 | 28 | @property 29 | def masters(self): 30 | return [i for i in self.nodes if i.is_master()] 31 | 32 | def set_key_migration_count(self, val): 33 | self.key_migration_count = val 34 | 35 | def consistent(self): 36 | sig = set() 37 | for instance in self.nodes: 38 | if not instance.is_master(): 39 | continue 40 | nodes = instance.nodes() 41 | slots, names = [], [] 42 | for node in nodes: 43 | slots.extend(node['slots']) 44 | names.append(node['name']) 45 | info = '{}:{}'.format('|'.join(sorted(names)), 46 | ','.join(str(i) for i in sorted(slots))) 47 | sig.add(hashlib.md5(info).hexdigest()) 48 | return len(sig) == 1 49 | 50 | def healthy(self): 51 | slots = list(itertools.chain(*[i.slots for i in self.nodes])) 52 | return len(slots) == self.CLUSTER_HASH_SLOTS and self.consistent() 53 | 54 | def wait(self): 55 | check = 0 56 | while not self.consistent(): 57 | time.sleep(1) 58 | if check == 10: 59 | raise ClusterNotConsistent('Error: cluster is not consistent') 60 | check += 1 61 | 62 | if not self.healthy(): 63 | raise ClusterNotHealthy('Error: missing slots') 64 | 65 | def get_node(self, node_id): 66 | for i in self.nodes: 67 | if i.name == node_id: 68 | return i 69 | 70 | def fix_open_slots(self): 71 | for master in self.masters: 72 | self.fix_node(master) 73 | 74 | def fix_node(self, node): 75 | info = node.node_info 76 | 77 | for slot, dst_id in info['migrating'].items(): 78 | dst = self.get_node(dst_id) 79 | if not dst or slot not in dst.node_info['importing']: 80 | node.set_slot('STABLE', slot) 81 | continue 82 | 83 | node.migrate_slot(dst, slot, self) 84 | 85 | for slot, target_id in info['importing'].items(): 86 | src = self.get_node(target_id) 87 | if not src or slot not in src.node_info['migrating']: 88 | node.set_slot('STABLE', slot) 89 | continue 90 | 91 | src.migrate_slot(node, slot, self) 92 | 93 | @failover.on_timeout 94 | def reshard(self): 95 | if not self.consistent(): 96 | return 97 | 98 | nodes = [{ 99 | "node": n, 100 | "count": len(n.slots), 101 | "need": [] 102 | } for n in self.masters] 103 | 104 | nodes = self.slot_balance(nodes) 105 | 106 | for n in nodes: 107 | if not n["need"]: 108 | continue 109 | for src, count in n["need"]: 110 | self.migrate(src, n["node"], count) 111 | 112 | @failover.on_timeout 113 | def remove_node(self, node): 114 | if node.is_master(): 115 | self.migrate_node(node) 116 | 117 | self.nodes = [n for n in self.nodes if n.name != node.name] 118 | masters = self.masters 119 | masters.sort(key=lambda x: len(x.slaves(x.name))) 120 | 121 | for n in self.nodes: 122 | if n.is_slave(node.name): 123 | n.replicate(masters[0].name) 124 | n.forget(node.name) 125 | 126 | assert not node.slots 127 | node.reset() 128 | 129 | def add_node(self, master): 130 | """ 131 | Add node to cluster. 132 | """ 133 | new = Node.from_uri(master) 134 | cluster_member = self.nodes[0] 135 | new.meet(cluster_member.host, cluster_member.port) 136 | self.nodes.append(new) 137 | self.wait() 138 | 139 | def fill_slots(self): 140 | masters = self.masters 141 | slots = itertools.chain(*[n.slots for n in masters]) 142 | missing = list(set(range(self.CLUSTER_HASH_SLOTS)).difference(slots)) 143 | 144 | div = divide(len(missing), len(masters)) 145 | masters.sort(key=lambda x: len(x.slots)) 146 | 147 | i = 0 148 | for count, node in zip(div, masters): 149 | node.add_slots(*missing[i:count + i]) 150 | i += count 151 | 152 | def bind_slots_force(self): 153 | masters = self.masters 154 | slots = itertools.chain(*[n.slots for n in masters]) 155 | missing = list(set(range(self.CLUSTER_HASH_SLOTS)).difference(slots)) 156 | 157 | div = divide(len(missing), len(masters)) 158 | masters.sort(key=lambda x: len(x.slots)) 159 | 160 | i = 0 161 | for count, node in zip(div, masters): 162 | for slot in missing[i:count + i]: 163 | self.update_slot_mapping(slot, node.name) 164 | i += count 165 | 166 | def migrate_node(self, src_node): 167 | nodes = [n for n in self.masters if n.name != src_node.name] 168 | slot_count = len(src_node.slots) 169 | if slot_count <= 0: 170 | return 171 | slots = divide(slot_count, len(nodes)) 172 | 173 | nodes.sort(key=lambda x: len(x.slots)) 174 | 175 | for node, count in zip(nodes, slots): 176 | src, dst = (src_node, node) 177 | self.migrate(src, dst, count) 178 | 179 | def migrate(self, src, dst, count): 180 | if count <= 0: 181 | return 182 | 183 | slots = src.slots 184 | slots_count = len(slots) 185 | if count > slots_count: 186 | count = slots_count 187 | 188 | keys = [(s, src.count_keys_in_slot(s)) for s in slots] 189 | keys.sort(key=lambda x: x[1]) 190 | 191 | for slot, _ in keys[:count]: 192 | src.migrate_slot(dst, slot, self) 193 | 194 | def update_slot_mapping(self, slot, dst_name): 195 | for node in self.parent_nodes: 196 | node.set_slot('NODE', slot, dst_name) 197 | 198 | def print_attempts(self): 199 | for node in self.nodes: 200 | self.attempts.extend(node.attempts) 201 | echo('Length of attempts: {}'.format(len(self.attempts))) 202 | for exc, group in itertools.groupby(self.attempts, 203 | lambda a: type(a).__name__): 204 | length = len(list(group)) 205 | echo( 206 | 'Exception: {}, Count: {}'.format(exc, length), 207 | color='yellow') 208 | 209 | def slot_balance(self, seq): 210 | amt = self.CLUSTER_HASH_SLOTS 211 | seq.sort(key=lambda x: x['count'], reverse=True) 212 | chunks = divide(amt, len(seq)) 213 | pairs = list(zip(seq, chunks)) 214 | 215 | i, j = 0, len(pairs) - 1 216 | while i < j: 217 | m, count = pairs[i] 218 | more = m['count'] - count 219 | if more <= 0: 220 | i += 1 221 | continue 222 | 223 | n, count = pairs[j] 224 | need = count - n['count'] 225 | if need <= 0: 226 | j -= 1 227 | continue 228 | 229 | if need < more: 230 | n['need'].append((m['node'], need)) 231 | n['count'] += need 232 | m['count'] -= need 233 | j -= 1 234 | elif need > more: 235 | n['need'].append((m['node'], more)) 236 | n['count'] += more 237 | m['count'] -= more 238 | i += 1 239 | else: 240 | n['need'].append((m['node'], need)) 241 | n['count'] += need 242 | m['count'] -= more 243 | j -= 1 244 | i += 1 245 | 246 | return seq 247 | -------------------------------------------------------------------------------- /redisclu/cli/commands.py: -------------------------------------------------------------------------------- 1 | import sys 2 | import time 3 | import redis 4 | import itertools 5 | import concurrent.futures 6 | 7 | from random import shuffle 8 | 9 | from redisclu.cli import helper as cli_helper 10 | from redisclu.cluster import Cluster 11 | from redisclu.node import Node, MockNode 12 | from redisclu.utils import echo 13 | 14 | 15 | @cli_helper.command 16 | @cli_helper.argument('masters', nargs='+') 17 | def create(args): 18 | creator = cli_helper.ClusterCreator(args.masters) 19 | if not creator.check(): 20 | echo('Pre-requirements to create cluster.\n', color='red') 21 | echo('\t1. At least 2 redis instances must be provided.') 22 | echo('\t2. Should be set on redis server conf.') 23 | echo('\t3. Should be removed all keys in db 0.') 24 | echo('\t4. Any redis instance should not be member of other cluster.') 25 | exit() 26 | creator.initialize_slots() 27 | creator.show_cluster_info() 28 | creator.bind_slots() 29 | creator.bind_config_epoch() 30 | creator.join_cluster() 31 | echo('Waiting for the cluster to join ', end='') 32 | sys.stdout.flush() 33 | time.sleep(1) 34 | while not creator.cluster.consistent(): 35 | echo('.', end='') 36 | sys.stdout.flush() 37 | time.sleep(1) 38 | 39 | 40 | @cli_helper.command 41 | @cli_helper.argument('cluster') 42 | def status(args): 43 | cluster = Cluster.from_node(Node.from_uri(args.cluster)) 44 | dis = [] 45 | for n in cluster.masters: 46 | slaves = ','.join([s['addr'] for s in n.slaves(n.name)]) 47 | msg = '{} {}:{} {} {}'.format(n.name, n.host, n.port, len(n.slots), 48 | slaves) 49 | dis.append(msg) 50 | echo('\n'.join(dis)) 51 | echo('Masters:', len(cluster.masters)) 52 | echo('Slaves:', len(cluster.nodes) - len(cluster.masters)) 53 | covered_slots = sum(len(n.slots) for n in cluster.masters) 54 | echo('Covered Slots:', covered_slots) 55 | if covered_slots == cluster.CLUSTER_HASH_SLOTS: 56 | echo('Cluster is healthy!') 57 | else: 58 | echo('!!!Cluster is not healthy!!!') 59 | echo('Either there is no cluster or exists cluster is not healthy.') 60 | echo('If there is no cluster then run "redis-clu create " or') 61 | echo('"redis-clu fix {}" would be great if there is exists cluster!'.format(args.cluster)) 62 | 63 | echo('\n') 64 | 65 | for master in cluster.masters: 66 | echo(master) 67 | echo('===========================') 68 | echo(master.execute_command('info', 'keyspace')) 69 | echo('\n') 70 | 71 | 72 | @cli_helper.command 73 | @cli_helper.argument('cluster') 74 | @cli_helper.argument('--force', default=0) 75 | def fix(args): 76 | cluster = Cluster.from_node(Node.from_uri(args.cluster)) 77 | cluster.fix_open_slots() 78 | if int(args.force) == 1: 79 | cluster.bind_slots_force() 80 | else: 81 | cluster.fill_slots() 82 | cluster.wait() 83 | cluster.print_attempts() 84 | 85 | 86 | @cli_helper.command 87 | @cli_helper.argument('cluster') 88 | @cli_helper.argument('master') 89 | @cli_helper.argument('--keyMigrationCount', default=1) 90 | @cli_helper.pass_ctx 91 | def add(ctx, args): 92 | """ 93 | add master node to cluster 94 | """ 95 | cluster = Cluster.from_node(Node.from_uri(args.cluster)) 96 | if not cluster.healthy(): 97 | ctx.abort( 98 | 'Cluster not healthy. Run "redis-clu fix {}" first'.format( 99 | args.cluster)) 100 | cluster.set_key_migration_count(int(args.keyMigrationCount)) 101 | cluster.add_node(args.master) 102 | cluster.reshard() 103 | cluster.wait() 104 | cluster.print_attempts() 105 | 106 | 107 | @cli_helper.command 108 | @cli_helper.argument('cluster') 109 | @cli_helper.argument('masters', nargs='+') 110 | @cli_helper.argument('--keyMigrationCount', default=1) 111 | @cli_helper.pass_ctx 112 | def add_multi(ctx, args): 113 | cluster = Cluster.from_node(Node.from_uri(args.cluster)) 114 | if len(args.masters) > len(cluster.nodes): 115 | ctx.abort('Length of new node list should be less than length of ' 116 | 'cluster master nodes.') 117 | if not cluster.healthy(): 118 | ctx.abort( 119 | 'Cluster not healthy. Run "redis-clu fix {}" first'.format( 120 | args.cluster)) 121 | masters = filter(lambda n: n.is_master(), cluster.nodes) 122 | residual_count = len(masters) % len(args.masters) 123 | if residual_count: 124 | for i in range(len(args.masters) - residual_count): 125 | masters.append(MockNode()) 126 | 127 | for master in args.masters: 128 | cluster.add_node(master) 129 | 130 | shard_ratio = len(masters) / len(args.masters) 131 | 132 | shuffle(masters) 133 | 134 | sub_clusters = [] 135 | while masters: 136 | sub_nodes = masters[:shard_ratio] 137 | new_master = args.masters.pop() 138 | nodes = filter(lambda n: not isinstance(n, MockNode), sub_nodes) 139 | nodes.append(Node.from_uri(new_master)) 140 | hash_slots = len(list(itertools.chain( 141 | *[n.slots for n in nodes]))) 142 | sub_cluster = Cluster(nodes, hash_slots=hash_slots, 143 | parent_nodes=cluster.nodes) 144 | sub_cluster.set_key_migration_count(int(args.keyMigrationCount)) 145 | sub_clusters.append(sub_cluster) 146 | for sn in sub_nodes: 147 | masters.pop(masters.index(sn)) 148 | 149 | future_to_args = dict() 150 | executor = concurrent.futures.ThreadPoolExecutor( 151 | max_workers=len(sub_clusters)) 152 | 153 | for sub_cluster in sub_clusters: 154 | args = () 155 | future = executor.submit(sub_cluster.reshard) 156 | future_to_args.setdefault(future, args) 157 | 158 | concurrent.futures.wait(future_to_args) 159 | executor.shutdown(wait=False) 160 | time.sleep(1) 161 | cluster.wait() 162 | 163 | for sub_cluster in sub_clusters: 164 | sub_cluster.print_attempts() 165 | 166 | 167 | @cli_helper.command 168 | @cli_helper.argument('cluster') 169 | @cli_helper.argument('--keyMigrationCount', default=1) 170 | @cli_helper.pass_ctx 171 | def reshard(ctx, args): 172 | cluster = Cluster.from_node(Node.from_uri(args.cluster)) 173 | if not cluster.healthy(): 174 | ctx.abort( 175 | 'Cluster not healthy. Run "redis-clu fix {}" first'.format( 176 | args.cluster)) 177 | cluster.set_key_migration_count(int(args.keyMigrationCount)) 178 | cluster.reshard() 179 | cluster.wait() 180 | cluster.print_attempts() 181 | 182 | 183 | @cli_helper.command 184 | @cli_helper.argument('cluster') 185 | @cli_helper.argument('node') 186 | @cli_helper.argument('--keyMigrationCount', default=1) 187 | @cli_helper.pass_ctx 188 | def remove(ctx, args): 189 | ''' 190 | remove node from cluster 191 | ''' 192 | cluster = Cluster.from_node(Node.from_uri(args.cluster)) 193 | if not cluster.healthy(): 194 | ctx.abort( 195 | 'Cluster not healthy. Run "redis-clu fix {}" first'.format( 196 | args.cluster)) 197 | cluster.set_key_migration_count(int(args.keyMigrationCount)) 198 | cluster.remove_node(Node.from_uri(args.node)) 199 | cluster.wait() 200 | cluster.print_attempts() 201 | 202 | 203 | @cli_helper.command 204 | @cli_helper.argument('master') 205 | @cli_helper.argument('slave') 206 | @cli_helper.pass_ctx 207 | def replicate(ctx, args): 208 | master = Node.from_uri(args.master) 209 | if not master.is_master(): 210 | ctx.abort('Node {} is not a master.'.format(args.master)) 211 | cluster = Cluster.from_node(master) 212 | cluster.add_node(args.slave) 213 | slave = Node.from_uri(args.slave) 214 | try: 215 | slave.replicate(master.name) 216 | except redis.ResponseError as e: 217 | ctx.abort(str(e)) 218 | cluster.wait() 219 | 220 | 221 | @cli_helper.command 222 | @cli_helper.argument('cluster') 223 | @cli_helper.argument('--hard', default=0) 224 | def reset(args): 225 | cluster = Cluster.from_node(Node.from_uri(args.cluster)) 226 | future_to_args = dict() 227 | executor = concurrent.futures.ThreadPoolExecutor( 228 | max_workers=len(cluster.masters)) 229 | 230 | for master in cluster.masters: 231 | f_args = () 232 | future = executor.submit(master.flushall) 233 | future_to_args.setdefault(future, f_args) 234 | 235 | concurrent.futures.wait(future_to_args) 236 | executor.shutdown(wait=False) 237 | 238 | if int(args.hard) == 1: 239 | for node in cluster.nodes: 240 | node.reset(hard=True) 241 | 242 | 243 | def load_commands(): 244 | import commands 245 | 246 | attrs = set(dir(commands)) 247 | return filter(lambda f: isinstance(f, cli_helper.Command), 248 | map(lambda attr: getattr(commands, attr), attrs)) 249 | --------------------------------------------------------------------------------