├── .circleci └── config.yml ├── .gitignore ├── LICENSE ├── README.rst ├── example.py ├── pyproject.toml ├── redis_hashring └── __init__.py ├── requirements.txt ├── setup.py └── tests.py /.circleci/config.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | 3 | workflows: 4 | version: 2 5 | workflow: 6 | jobs: 7 | - lint 8 | - test-3.8 9 | - test-3.9 10 | - test-3.10 11 | - test-3.11 12 | - test-3.12 13 | 14 | defaults: &defaults 15 | working_directory: ~/code 16 | steps: 17 | - checkout 18 | - run: 19 | name: Install dependencies 20 | command: pip install --user -r requirements.txt 21 | - run: 22 | name: Test 23 | command: | 24 | export PATH=$HOME/.local/bin:$PATH 25 | pytest 26 | 27 | jobs: 28 | lint: 29 | working_directory: ~/code 30 | docker: 31 | - image: python:3.12 32 | steps: 33 | - checkout 34 | - run: 35 | name: Install dependencies 36 | command: pip install --user -r requirements.txt 37 | - run: 38 | name: Format 39 | command: | 40 | export PATH=$HOME/.local/bin:$PATH 41 | ruff format --check --no-cache 42 | - run: 43 | name: Lint 44 | command: | 45 | export PATH=$HOME/.local/bin:$PATH 46 | ruff check --no-cache 47 | test-3.8: 48 | <<: *defaults 49 | docker: 50 | - image: python:3.8 51 | - image: redis:7.2.4 52 | test-3.9: 53 | <<: *defaults 54 | docker: 55 | - image: python:3.9 56 | - image: redis:7.2.4 57 | test-3.10: 58 | <<: *defaults 59 | docker: 60 | - image: python:3.10 61 | - image: redis:7.2.4 62 | test-3.11: 63 | <<: *defaults 64 | docker: 65 | - image: python:3.11 66 | - image: redis:7.2.4 67 | test-3.12: 68 | <<: *defaults 69 | docker: 70 | - image: python:3.12 71 | - image: redis:7.2.4 72 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.pyc 2 | venv/ 3 | *.egg 4 | *.egg-info/ 5 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2015-2024 Elastic Inc. (Close) 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE 22 | -------------------------------------------------------------------------------- /README.rst: -------------------------------------------------------------------------------- 1 | ============== 2 | redis-hashring 3 | ============== 4 | .. image:: https://circleci.com/gh/closeio/redis-hashring.svg?style=svg&circle-token=e9b81f0e4bc9a1a0b6150522e854ca0c9b1c2881 5 | :target: https://circleci.com/gh/closeio/redis-hashring/tree/master 6 | 7 | *redis-hashring* is a Python library that implements a consistent hash ring 8 | for building distributed applications, which is stored in Redis. 9 | 10 | The problem 11 | ----------- 12 | 13 | Let's assume you're building a distributed application that's responsible for 14 | syncing accounts. Accounts are synced continuously, e.g. by keeping a 15 | connection open. Given the large amount of accounts, the application can't 16 | run in one process and has to be distributed and split up in multiple 17 | processes. Also, if one of the processes fails or crashes, other machines need 18 | to be able to take over accounts quickly. The load should be balanced equally 19 | between the machines. 20 | 21 | The solution 22 | ------------ 23 | 24 | A solution to this problem is to use a consistent hash ring: Different Python 25 | instances ("nodes") are responsible for a different set of keys. In our account 26 | example, the account IDs could be used as keys. A consistent hash ring is a 27 | large (integer) space that wraps around to form a circle. Each node picks a few 28 | random points ("replicas") on the hash ring when starting. Keys are hashed and 29 | looked up on the hash ring: In order to find the node that's responsible for a 30 | given key, we move on the hash ring until we find the next smaller point that 31 | belongs to a replica. The reason for multiple replicas per node is to ensure 32 | better distribution of the keys amongst the nodes. It can also be used to give 33 | certain nodes more weight. The ring is automatically rebalanced when a node 34 | enters or leaves the ring: If a node crashes or shuts down, its replicas are 35 | removed from the ring. 36 | 37 | How it works 38 | ------------ 39 | 40 | The ring is stored as a sorted set (ZSET) in Redis. Each replica is a member 41 | of the set, scored by it's expiration time. Each node needs to periodically 42 | refresh the score of its replicas to stay on the ring. 43 | 44 | The ring contains 2^32 points, and a replica is created by randomly placing 45 | a point on the ring. A replica of a node is responsible for the range of 46 | points from its randomly generated starting point until the starting point of 47 | the next node / replica. 48 | 49 | To check if a node is responsible for a given key, the key's position on the 50 | ring is determined by hashing the key using CRC-32. 51 | 52 | For example, let's say there are two nodes, having one replica each. The first 53 | node is at 1 000 000 000 (1e9), the second at 2e9. In this case, the first node 54 | is responsible for the range [1e9, 2e9-1], the second node is responsible for 55 | [2e9, 2^32-1] and [0, 1e9-1], since the ring wraps. To check if the key 56 | *hello* is on the ring, we compute CRC-32, which is 907 060 870, and the value 57 | is therefore on the first node. 58 | 59 | Since the node replica points are picked randomly, it is recommended to have 60 | multiple replicas of the node on a ring to ensure a more even distribution of 61 | the nodes. 62 | 63 | Demo 64 | ---- 65 | 66 | As an example, let's assume you have a process that is responsible for syncing 67 | accounts. In this example they are numbered from 0 to 99. Starting node 1 will 68 | assign all accounts to node 1, since it's the only node on the ring. 69 | 70 | We can see this by running the provided example script on node 1: 71 | 72 | .. code:: bash 73 | 74 | % python example.py 75 | INFO:root:PID 80721, 100 keys ([0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99]) 76 | 77 | We can print the ring for debugging and see all the nodes and replicas on the 78 | ring: 79 | 80 | .. code:: bash 81 | 82 | % python example.py --print 83 | Hash ring "ring" replicas: 84 | Start Range Delay Node 85 | 706234936 2.97% 0s mbp.local:80721:249d729d 86 | 833679955 3.58% 0s mbp.local:80721:aa60d44c 87 | 987624694 24.44% 0s mbp.local:80721:aa7d4433 88 | 2037338983 3.41% 0s mbp.local:80721:e810d068 89 | 2183761853 3.55% 0s mbp.local:80721:3917f572 90 | 2336151471 2.82% 0s mbp.local:80721:e42b1b46 91 | 2457297989 4.40% 0s mbp.local:80721:e6bd5726 92 | 2646391033 4.37% 0s mbp.local:80721:6de2fc22 93 | 2834073726 5.30% 0s mbp.local:80721:b6f950b2 94 | 3061910569 3.96% 0s mbp.local:80721:d176c9e2 95 | 3231812046 5.70% 0s mbp.local:80721:65432143 96 | 3476455773 5.71% 0s mbp.local:80721:f2b29682 97 | 3721589736 0.65% 0s mbp.local:80721:51d0cb09 98 | 3749333446 5.53% 0s mbp.local:80721:3572f718 99 | 3986767934 4.39% 0s mbp.local:80721:42147f45 100 | 4175523935 19.22% 0s mbp.local:80721:296c9522 101 | 102 | Hash ring "ring" nodes: 103 | Range Replicas Delay Hostname PID 104 | 100.00% 16 0s mbp.local 80721 105 | 106 | We can see that the node is responsible for the entire ring (range 100%) and 107 | has 16 replicas on the ring. 108 | 109 | Now let's start another node by running the script again. It will add its 110 | replicas to the ring and notify all the remaining nodes. 111 | 112 | .. code:: bash 113 | 114 | % python example.py 115 | INFO:root:PID 80721, 51 keys ([1, 5, 8, 9, 10, 14, 17, 20, 21, 24, 25, 28, 30, 32, 33, 34, 36, 38, 41, 42, 45, 46, 49, 50, 52, 54, 56, 58, 59, 60, 61, 62, 65, 66, 68, 69, 71, 74, 75, 78, 79, 81, 82, 85, 86, 87, 88, 89, 92, 93, 96]) 116 | 117 | Node 1 will rebalance and is now only responsible for keys not in node 2: 118 | 119 | .. code:: bash 120 | 121 | INFO:root:PID 80808, 49 keys ([0, 2, 3, 4, 6, 7, 11, 12, 13, 15, 16, 18, 19, 22, 23, 26, 27, 29, 31, 35, 37, 39, 40, 43, 44, 47, 48, 51, 53, 55, 57, 63, 64, 67, 70, 72, 73, 76, 77, 80, 83, 84, 90, 91, 94, 95, 97, 98, 99]) 122 | 123 | We can inspect the ring: 124 | 125 | .. code:: bash 126 | 127 | % python example.py --print 128 | Hash ring "ring" replicas: 129 | Start Range Delay Node 130 | 204632062 1.06% 0s mbp.local:80808:f933c33c 131 | 250215779 0.36% 0s mbp.local:80808:3b104c45 132 | 265648189 1.15% 0s mbp.local:80808:84d71125 133 | 315059885 2.77% 0s mbp.local:80808:bab5a03c 134 | 434081415 6.34% 0s mbp.local:80808:6eec1b26 135 | 706234936 2.97% 0s mbp.local:80721:249d729d 136 | 833679955 1.59% 0s mbp.local:80721:aa60d44c 137 | 901926411 2.00% 0s mbp.local:80808:bd6f3b27 138 | 987624694 2.87% 0s mbp.local:80721:aa7d4433 139 | 1110943067 5.42% 0s mbp.local:80808:abfa5d78 140 | 1343923832 0.83% 0s mbp.local:80808:5261947f 141 | 1379658747 4.70% 0s mbp.local:80808:cb0904de 142 | 1581392642 1.06% 0s mbp.local:80808:3050daa3 143 | 1627017290 9.55% 0s mbp.local:80808:8e1cef12 144 | 2037338983 3.41% 0s mbp.local:80721:e810d068 145 | 2183761853 3.55% 0s mbp.local:80721:3917f572 146 | 2336151471 2.82% 0s mbp.local:80721:e42b1b46 147 | 2457297989 4.40% 0s mbp.local:80721:e6bd5726 148 | 2646391033 4.37% 0s mbp.local:80721:6de2fc22 149 | 2834073726 2.30% 0s mbp.local:80721:b6f950b2 150 | 2932842903 3.01% 0s mbp.local:80808:58f09769 151 | 3061910569 3.08% 0s mbp.local:80721:d176c9e2 152 | 3194206736 0.88% 0s mbp.local:80808:ce94a1cf 153 | 3231812046 5.70% 0s mbp.local:80721:65432143 154 | 3476455773 0.21% 0s mbp.local:80721:f2b29682 155 | 3485592199 5.49% 0s mbp.local:80808:6fc107a3 156 | 3721589736 0.65% 0s mbp.local:80721:51d0cb09 157 | 3749333446 0.68% 0s mbp.local:80721:3572f718 158 | 3778349273 4.85% 0s mbp.local:80808:e7cc7485 159 | 3986767934 1.29% 0s mbp.local:80721:42147f45 160 | 4042192844 3.10% 0s mbp.local:80808:001590b5 161 | 4175523935 7.55% 0s mbp.local:80721:296c9522 162 | 163 | Hash ring "ring" nodes: 164 | Range Replicas Delay Hostname PID 165 | 47.42% 16 0s mbp.local 80721 166 | 52.58% 16 0s mbp.local 80808 167 | 168 | gevent example 169 | -------------- 170 | 171 | *redis-hashring* provides a ``RingNode`` class, which has helper methods for 172 | `gevent`-based applications. The ``RingNode.gevent_start()`` method spawns a 173 | greenlet that initializes the ring and periodically updates the node's 174 | replicas. 175 | 176 | An example app could look as follows: 177 | 178 | .. code:: python 179 | 180 | from redis import Redis 181 | from redis_hashring import RingNode 182 | 183 | KEY = 'example-ring' 184 | 185 | redis = Redis() 186 | node = RingNode(redis, KEY) 187 | node.gevent_start() 188 | 189 | def get_items(): 190 | """ 191 | Implement this method and return items to be processed. 192 | """ 193 | raise NotImplementedError() 194 | 195 | def process_items(items): 196 | """ 197 | Implement this method and process the given items. 198 | """ 199 | raise NotImplementedError() 200 | 201 | try: 202 | while True: 203 | # Only process items this node is reponsible for. 204 | items = [item for item in get_items() if node.contains(item)] 205 | process_items(items) 206 | except KeyboardInterrupt: 207 | pass 208 | 209 | node.gevent_stop() 210 | 211 | Implementation considerations 212 | ----------------------------- 213 | 214 | When implementing a distributed application using redis-hashring, be aware of 215 | the following: 216 | 217 | - Locking 218 | 219 | When nodes are added to the ring, multiple nodes might assume they're 220 | responsible for the same key until they are notified about the new state of 221 | the ring. Depending on the application, locking may be necessary to avoid 222 | duplicate processing. 223 | 224 | For example, in the demo above the node could add a per-account-ID lock if an 225 | account should never be synced by multiple nodes at the same time. This can 226 | be done using a Redis lock class or any other distributed lock. 227 | 228 | - Limit 229 | 230 | It is recommended to add an upper limit to the number of keys a node can 231 | process to avoid overloading a node when there are few nodes on the ring or 232 | all nodes need to be restarted. 233 | 234 | For example, in the demo above we could implement a limit of 50 accounts, if 235 | we know that a node may not be capable of syncing much more. In this case, 236 | multiple nodes would need to be running to sync all the accounts. Also note 237 | that the ring is not usually equally balanced, so running 2 nodes wouldn't be 238 | enough in this example. 239 | -------------------------------------------------------------------------------- /example.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import logging 3 | import os 4 | import sys 5 | import time 6 | 7 | import redis 8 | 9 | from redis_hashring import RingNode 10 | 11 | N_KEYS = 100 12 | 13 | logging.basicConfig(level=logging.DEBUG) 14 | 15 | 16 | def _parse_arguments(): 17 | parser = argparse.ArgumentParser("Hash ring example.", add_help=False) 18 | parser.add_argument( 19 | "--host", "-h", default="localhost", help="Redis hostname." 20 | ) 21 | parser.add_argument( 22 | "--port", "-p", type=int, default=6379, help="Redis port." 23 | ) 24 | parser.add_argument( 25 | "--print", 26 | action="store_true", 27 | dest="print", 28 | help="Print the hash ring.", 29 | ) 30 | parser.add_argument( 31 | "--help", 32 | action="help", 33 | default=argparse.SUPPRESS, 34 | help="Show this help message and exit.", 35 | ) 36 | return parser.parse_args() 37 | 38 | 39 | if __name__ == "__main__": 40 | args = _parse_arguments() 41 | 42 | print(f"Attempting to connect to Redis at {args.host}:{args.port}.") 43 | r = redis.Redis(args.host, args.port) 44 | try: 45 | r.ping() 46 | except redis.exceptions.ConnectionError: 47 | print("Failed to connect to Redis.") 48 | sys.exit(1) 49 | 50 | node = RingNode(r, "ring") 51 | 52 | if args.print: 53 | node.debug_print() 54 | sys.exit() 55 | 56 | pid = os.getpid() 57 | 58 | node.start() 59 | 60 | try: 61 | while True: 62 | keys = [key for key in range(N_KEYS) if node.contains(str(key))] 63 | logging.info("PID %d, %d keys (%s)", pid, len(keys), repr(keys)) 64 | time.sleep(2) 65 | except KeyboardInterrupt: 66 | pass 67 | 68 | node.stop() 69 | -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [tool.ruff] 2 | target-version = "py38" 3 | line-length = 79 4 | exclude = [ 5 | ".git", 6 | "venv", 7 | ".venv", 8 | "__pycache__", 9 | ] 10 | 11 | [tool.ruff.lint] 12 | extend-select = ["I"] 13 | unfixable = [ 14 | # Variable assigned but never used - automatically removing the assignment 15 | # is annoying when running autofix on work-in-progress code. 16 | "F841", 17 | ] 18 | 19 | [tool.ruff.lint.flake8-tidy-imports] 20 | ban-relative-imports = "all" 21 | 22 | [tool.ruff.lint.flake8-tidy-imports.banned-api] 23 | "functools.partial".msg = "Use a lambda or a named function instead. Partials don't type check correctly." 24 | "datetime.datetime.utcnow".msg = "Use `datetime.datetime.now(datetime.timezone.utc).replace(tzinfo=None)` instead." 25 | "datetime.datetime.utcfromtimestamp".msg = "Use `datetime.datetime.fromtimestamp(timestamp, datetime.timezone.utc).replace(tzinfo=None)` instead." 26 | 27 | [tool.ruff.lint.isort] 28 | combine-as-imports = true 29 | forced-separate = ["tests"] 30 | 31 | [tool.ruff.lint.pydocstyle] 32 | # https://google.github.io/styleguide/pyguide.html#383-functions-and-methods 33 | convention = "google" 34 | 35 | [tool.ruff.lint.flake8-annotations] 36 | ignore-fully-untyped = true 37 | 38 | [tool.pytest.ini_options] 39 | timeout = 180 40 | python_files = "tests.py" 41 | testpaths = ["."] 42 | xfail_strict = true 43 | -------------------------------------------------------------------------------- /redis_hashring/__init__.py: -------------------------------------------------------------------------------- 1 | import binascii 2 | import collections 3 | import operator 4 | import os 5 | import random 6 | import select 7 | import socket 8 | import threading 9 | import time 10 | 11 | # Amount of points on the ring. Must not be higher than 2**32 because we're 12 | # using CRC32 to compute the checksum. 13 | RING_SIZE = 2**32 14 | 15 | # Default amount of replicas per node. 16 | RING_REPLICAS = 16 17 | 18 | # How often to update a node's heartbeat. 19 | POLL_INTERVAL = 10 20 | 21 | # After how much time a node is considered to be dead. 22 | NODE_TIMEOUT = 60 23 | 24 | # How often expired nodes are cleaned up from the ring. 25 | CLEANUP_INTERVAL = 120 26 | 27 | 28 | def _decode(data): 29 | # Compatibility with different redis-py `decode_responses` settings. 30 | if isinstance(data, bytes): 31 | return data.decode() 32 | else: 33 | return data 34 | 35 | 36 | class RingNode(object): 37 | """ 38 | A node in a Redis hash ring. 39 | 40 | Each node may have multiple replicas on the ring for more balanced hashing. 41 | 42 | The ring is stored as follows in Redis: 43 | 44 | ZSET 45 | Represents the ring in Redis. The keys of this ZSET represent 46 | "start:replica_name", where start is the start of the range for which the 47 | replica is responsible. 48 | 49 | CHANNEL 50 | Represents a pubsub channel in Redis which receives a message every time 51 | the ring structure has changed. 52 | 53 | Simple usage example: 54 | 55 | ``` 56 | node = RingNode(redis, key) 57 | node.start() 58 | 59 | while is_running: 60 | # Only process items this node is responsible for. `item` should be an 61 | # object that can be encoded to bytes by calling `item.encode()` on it, 62 | # like a `str`. 63 | items = [item for item in get_items() if node.contains(item)] 64 | process_items(items) 65 | 66 | node.stop() 67 | ``` 68 | 69 | As a context manager: 70 | 71 | ``` 72 | with RingNode(redis, key) as node: 73 | while is_running: 74 | # Only process items this node is responsible for. `item` should be 75 | # an object that can be encoded to bytes by calling `item.encode()` 76 | # on it, like a `str`. 77 | items = [item for item in get_items() if node.contains(item)] 78 | process_items(items) 79 | ``` 80 | """ 81 | 82 | def __init__(self, conn, key, n_replicas=RING_REPLICAS): 83 | """ 84 | Initializes a Redis hash ring node. 85 | 86 | Args: 87 | conn: The Redis connection to use. 88 | key: A key to use for this node. 89 | n_replicas: Number of replicas this node should have on the ring. 90 | """ 91 | self._polling_thread = None 92 | self._stop_polling_fd_r = None 93 | self._stop_polling_fd_w = None 94 | 95 | self._conn = conn 96 | self._key = key 97 | host = socket.gethostname() 98 | pid = os.getpid() 99 | 100 | # Create unique identifiers for the replicas. 101 | self._replicas = [ 102 | ( 103 | random.randrange(2**32), 104 | "{host}:{pid}:{id_}".format( 105 | host=host, 106 | pid=pid, 107 | id_=binascii.hexlify(os.urandom(4)).decode(), 108 | ), 109 | ) 110 | for _ in range(n_replicas) 111 | ] 112 | 113 | # Number of nodes currently active in the ring. 114 | self._node_count = 0 115 | # List of tuples of ranges this node is responsible for, where a tuple 116 | # (a, b) includes any N matching a <= N < b. 117 | self._ranges = [] 118 | 119 | self._select = select.select 120 | 121 | def _fetch_ring(self): 122 | """ 123 | Fetch the ring from Redis. 124 | 125 | The fetched ring only includes active nodes. Returns a list of tuples 126 | (start, replica) (see _fetch_all docs for more details). 127 | """ 128 | expiry_time = time.time() - NODE_TIMEOUT 129 | data = self._conn.zrangebyscore(self._key, expiry_time, "INF") 130 | 131 | ring = [] 132 | for replica_data in data: 133 | start, replica = _decode(replica_data).split(":", 1) 134 | ring.append((int(start), replica)) 135 | return sorted(ring, key=operator.itemgetter(0)) 136 | 137 | def _fetch_ring_all(self): 138 | """ 139 | Fetch the ring from Redis. 140 | 141 | The fetched ring will include inactive nodes. Returns a list of tuples 142 | (start, replica, heartbeat, expired), where: 143 | * start: start of the range for which the replica is responsible. 144 | * replica: name of the replica. 145 | * heartbeat: timestamp of the last heartbeat. 146 | * expired: boolean denoting whether this replica is inactive. 147 | """ 148 | expiry_time = time.time() - NODE_TIMEOUT 149 | data = self._conn.zrange(self._key, 0, -1, withscores=True) 150 | 151 | ring = [] 152 | for replica_data, heartbeat in data: 153 | start, replica = _decode(replica_data).split(":", 1) 154 | ring.append( 155 | (int(start), replica, heartbeat, heartbeat < expiry_time) 156 | ) 157 | return sorted(ring, key=operator.itemgetter(0)) 158 | 159 | def debug_print(self): 160 | """ 161 | Prints the ring for debugging purposes. 162 | """ 163 | ring = self._fetch_ring_all() 164 | 165 | print('Hash ring "{key}" replicas:'.format(key=self._key)) 166 | 167 | now = time.time() 168 | 169 | n_replicas = len(ring) 170 | if ring: 171 | print( 172 | "{:10} {:6} {:7} {}".format("Start", "Range", "Delay", "Node") 173 | ) 174 | else: 175 | print("(no replicas)") 176 | 177 | nodes = collections.defaultdict(list) 178 | 179 | for n, (start, replica, heartbeat, expired) in enumerate(ring): 180 | hostname, pid, _ = replica.split(":") 181 | node = ":".join([hostname, pid]) 182 | 183 | abs_size = (ring[(n + 1) % n_replicas][0] - ring[n][0]) % RING_SIZE 184 | size = 100.0 / RING_SIZE * abs_size 185 | delay = int(now - heartbeat) 186 | expired_str = "(EXPIRED)" if expired else "" 187 | 188 | nodes[node].append((hostname, pid, abs_size, delay, expired)) 189 | 190 | print( 191 | f"{start:10} {size:5.2f}% {delay:6}s {replica} {expired_str}" 192 | ) 193 | 194 | print() 195 | print('Hash ring "{key}" nodes:'.format(key=self._key)) 196 | 197 | if nodes: 198 | print( 199 | "{:8} {:8} {:7} {:20} {:5}".format( 200 | "Range", "Replicas", "Delay", "Hostname", "PID" 201 | ) 202 | ) 203 | else: 204 | print("(no nodes)") 205 | 206 | for _, v in nodes.items(): 207 | hostname, pid = v[0][0], v[0][1] 208 | abs_size = sum(replica[2] for replica in v) 209 | size = 100.0 / RING_SIZE * abs_size 210 | delay = max(replica[3] for replica in v) 211 | expired = any(replica[4] for replica in v) 212 | count = len(v) 213 | expired_str = "(EXPIRED)" if expired else "" 214 | print( 215 | f"{size:5.2f}% {count:8} {delay:6}s {hostname:20} {pid:5}" 216 | f" {expired_str}" 217 | ) 218 | 219 | def heartbeat(self): 220 | """ 221 | Add/update the node in Redis. 222 | 223 | Needs to be called regularly by the node. 224 | """ 225 | pipeline = self._conn.pipeline() 226 | 227 | now = time.time() 228 | 229 | for replica in self._replicas: 230 | pipeline.zadd(self._key, {f"{replica[0]}:{replica[1]}": now}) 231 | ret = pipeline.execute() 232 | 233 | # Only notify the other nodes if we're not in the ring yet. 234 | if any(ret): 235 | self._notify() 236 | 237 | def remove(self): 238 | """ 239 | Remove the node from the ring. 240 | """ 241 | pipeline = self._conn.pipeline() 242 | 243 | for replica in self._replicas: 244 | pipeline.zrem(self._key, f"{replica[0]}:{replica[1]}") 245 | pipeline.execute() 246 | 247 | # Make sure this node won't contain any items. 248 | self._node_count = 0 249 | self._ranges = [] 250 | 251 | self._notify() 252 | 253 | def _notify(self): 254 | """ 255 | Publish an update to the ring's activity channel. 256 | """ 257 | self._conn.publish(self._key, "*") 258 | 259 | def cleanup(self): 260 | """ 261 | Removes expired nodes from the ring. 262 | """ 263 | expired = time.time() - NODE_TIMEOUT 264 | 265 | if self._conn.zremrangebyscore(self._key, 0, expired): 266 | self._notify() 267 | 268 | def update(self): 269 | """ 270 | Fetches the updated ring from Redis and updates the current ranges. 271 | """ 272 | ring = self._fetch_ring() 273 | nodes = set() 274 | n_replicas = len(ring) 275 | 276 | own_replicas = {r[1] for r in self._replicas} 277 | 278 | self._ranges = [] 279 | for n, (start, replica) in enumerate(ring): 280 | host, pid, _ = replica.split(":") 281 | node = ":".join([host, pid]) 282 | nodes.add(node) 283 | 284 | if replica in own_replicas: 285 | end = ring[(n + 1) % n_replicas][0] % RING_SIZE 286 | if start < end: 287 | self._ranges.append((start, end)) 288 | elif end < start: 289 | self._ranges.append((start, RING_SIZE)) 290 | self._ranges.append((0, end)) 291 | else: 292 | self._ranges.append((0, RING_SIZE)) 293 | 294 | self._node_count = len(nodes) 295 | 296 | def get_ranges(self): 297 | """ 298 | Return the hash ring ranges that this node owns. 299 | """ 300 | return self._ranges 301 | 302 | def get_node_count(self): 303 | """ 304 | Return the number of active nodes in the ring. 305 | """ 306 | return self._node_count 307 | 308 | def contains(self, key): 309 | """ 310 | Check whether this node is responsible for the item. 311 | """ 312 | return self._contains_ring_point(self.key_as_ring_point(key)) 313 | 314 | def key_as_ring_point(self, key): 315 | """Turn a key into a point on a hash ring.""" 316 | return binascii.crc32(key.encode()) % RING_SIZE 317 | 318 | def _contains_ring_point(self, n): 319 | """ 320 | Check whether this node is responsible for the ring point. 321 | """ 322 | for start, end in self._ranges: 323 | if start <= n < end: 324 | return True 325 | return False 326 | 327 | def poll(self): 328 | """ 329 | Keep a node in the hash ring. 330 | 331 | This should be kept running for as long as the node needs to stay in 332 | the ring. Can be run in a separate thread or in a greenlet. This takes 333 | care of: 334 | * Updating the heartbeat. 335 | * Checking for ring updates. 336 | * Cleaning up expired nodes periodically. 337 | """ 338 | pubsub = self._conn.pubsub() 339 | pubsub.subscribe(self._key) 340 | pubsub_fd = pubsub.connection._sock.fileno() 341 | 342 | last_heartbeat = time.time() 343 | self.heartbeat() 344 | 345 | last_cleanup = time.time() 346 | self.cleanup() 347 | 348 | self._stop_polling_fd_r, self._stop_polling_fd_w = os.pipe() 349 | 350 | try: 351 | while True: 352 | # Since Redis' `listen` method blocks, we use `select` to 353 | # inspect the underlying socket to see if there is activity. 354 | timeout = max( 355 | 0.0, POLL_INTERVAL - (time.time() - last_heartbeat) 356 | ) 357 | r, _, _ = self._select( 358 | [self._stop_polling_fd_r, pubsub_fd], [], [], timeout 359 | ) 360 | 361 | if self._stop_polling_fd_r in r: 362 | os.close(self._stop_polling_fd_r) 363 | os.close(self._stop_polling_fd_w) 364 | self._stop_polling_fd_r = None 365 | self._stop_polling_fd_w = None 366 | break 367 | 368 | if pubsub_fd in r: 369 | while pubsub.get_message(): 370 | pass 371 | self.update() 372 | 373 | last_heartbeat = time.time() 374 | self.heartbeat() 375 | 376 | now = time.time() 377 | if now - last_cleanup > CLEANUP_INTERVAL: 378 | last_cleanup = now 379 | self.cleanup() 380 | finally: 381 | pubsub.close() 382 | 383 | def start(self): 384 | """ 385 | Start the node for threads-based applications. 386 | """ 387 | self._polling_thread = threading.Thread(target=self.poll, daemon=True) 388 | self._polling_thread.start() 389 | 390 | def stop(self): 391 | """ 392 | Stop the node for threads-based applications. 393 | """ 394 | if self._polling_thread: 395 | while not self._stop_polling_fd_w: 396 | # Let's give the thread some time to create the fd. 397 | time.sleep(0.1) 398 | os.write(self._stop_polling_fd_w, b"1") 399 | self._polling_thread.join() 400 | self._polling_thread = None 401 | self.remove() 402 | 403 | def __enter__(self): 404 | self.start() 405 | return self 406 | 407 | def __exit__(self, *args, **kwargs): 408 | self.stop() 409 | 410 | 411 | class GeventRingNode(RingNode): 412 | """ 413 | A node in a Redis hash ring. 414 | 415 | This works exactly the same as `RingNode`, except that `start` and `stop` 416 | will create a gevent greenlet to maintain the node information up to date 417 | with the hash ring. 418 | 419 | For a usage example, see the documentation for `RingNode`. 420 | """ 421 | 422 | def __init__(self, *args, **kwargs): 423 | self._polling_greenlet = None 424 | super().__init__(*args, **kwargs) 425 | 426 | def start(self): 427 | """ 428 | Start the node for gevent-based applications. 429 | """ 430 | import gevent 431 | import gevent.select 432 | 433 | self._select = gevent.select.select 434 | self._polling_greenlet = gevent.spawn(self.poll) 435 | 436 | # Even though `self.poll` will run `self.heartbeat` and `self.update` 437 | # immediately as it starts, this is gevent and `self.poll` may take a 438 | # while to run, depending on how long the greenlet that creates the 439 | # node takes to yield. So we'll run these functions here to make sure 440 | # the node is up to date immediately. 441 | self.heartbeat() 442 | self.update() 443 | 444 | def stop(self): 445 | """ 446 | Stop the node for gevent-based applications. 447 | """ 448 | if self._polling_greenlet: 449 | while not self._stop_polling_fd_w: 450 | # Let's give the greenlet some time to create the fd. 451 | time.sleep(0.1) 452 | os.write(self._stop_polling_fd_w, b"1") 453 | self._polling_greenlet.join() 454 | self._polling_greenlet = None 455 | self.remove() 456 | self._select = select.select 457 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | pytest==7.2.2 2 | redis==4.6.0 3 | ruff==0.4.3 4 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | from setuptools import setup 2 | 3 | setup( 4 | name="redis-hashring", 5 | version="0.5.0", 6 | author="Close Engineering", 7 | author_email="engineering@close.com", 8 | url="https://github.com/closeio/redis-hashring", 9 | license="MIT", 10 | description=( 11 | "Python library for distributed applications using a Redis hash ring" 12 | ), 13 | install_requires=["redis>=3"], 14 | platforms="any", 15 | classifiers=[ 16 | "Intended Audience :: Developers", 17 | "License :: OSI Approved :: MIT License", 18 | "Operating System :: OS Independent", 19 | "Topic :: Software Development :: Libraries :: Python Modules", 20 | "Programming Language :: Python", 21 | "Programming Language :: Python :: 3", 22 | "Programming Language :: Python :: 3 :: Only", 23 | "Programming Language :: Python :: 3.8", 24 | "Programming Language :: Python :: 3.9", 25 | "Programming Language :: Python :: 3.10", 26 | "Programming Language :: Python :: 3.11", 27 | "Programming Language :: Python :: 3.12", 28 | ], 29 | packages=["redis_hashring"], 30 | ) 31 | -------------------------------------------------------------------------------- /tests.py: -------------------------------------------------------------------------------- 1 | import socket 2 | from unittest.mock import patch 3 | 4 | import pytest 5 | from redis import Redis 6 | 7 | from redis_hashring import RingNode 8 | 9 | TEST_KEY = "hashring-test" 10 | 11 | 12 | @pytest.fixture 13 | def redis(): 14 | redis = Redis() 15 | yield redis 16 | redis.delete(TEST_KEY) 17 | 18 | 19 | def get_node(redis, n_replicas, total_replicas): 20 | node = RingNode(redis, TEST_KEY, n_replicas=n_replicas) 21 | 22 | assert len(node._replicas) == n_replicas 23 | assert redis.zcard(TEST_KEY) == total_replicas - n_replicas 24 | 25 | node.heartbeat() 26 | 27 | assert redis.zcard(TEST_KEY) == total_replicas 28 | assert len(node.get_ranges()) == 0 29 | 30 | return node 31 | 32 | 33 | def test_node(redis): 34 | with patch.object(socket, "gethostname", return_value="host1"): 35 | node1 = get_node(redis, 1, 1) 36 | node1.update() 37 | assert len(node1.get_ranges()) == 1 38 | assert node1.get_node_count() == 1 39 | 40 | with patch.object(socket, "gethostname", return_value="host2"): 41 | node2 = get_node(redis, 1, 2) 42 | node1.update() 43 | node2.update() 44 | assert len(node1.get_ranges()) + len(node2.get_ranges()) == 3 45 | assert node1.get_node_count() == 2 46 | assert node2.get_node_count() == 2 47 | 48 | with patch.object(socket, "gethostname", return_value="host3"): 49 | node3 = get_node(redis, 2, 4) 50 | node1.update() 51 | node2.update() 52 | node3.update() 53 | assert ( 54 | len(node1.get_ranges()) 55 | + len(node2.get_ranges()) 56 | + len(node3.get_ranges()) 57 | == 5 58 | ) 59 | assert node1.get_node_count() == 3 60 | assert node2.get_node_count() == 3 61 | assert node3.get_node_count() == 3 62 | 63 | node1.remove() 64 | node2.update() 65 | node3.update() 66 | assert len(node1.get_ranges()) == 0 67 | assert node1.get_node_count() == 0 68 | assert len(node2.get_ranges()) + len(node3.get_ranges()) == 4 69 | assert node2.get_node_count() == 2 70 | assert node3.get_node_count() == 2 71 | 72 | 73 | def test_contains(redis): 74 | node1 = get_node(redis, 1, 1) 75 | node1.update() 76 | assert node1.contains("item") is True 77 | 78 | node1.remove() 79 | assert node1.contains("item") is False 80 | --------------------------------------------------------------------------------