├── .arcconfig ├── .gitignore ├── .travis.yml ├── AUTHORS ├── CHANGES ├── LICENSE ├── MANIFEST.in ├── Makefile ├── README.rst ├── benchmark.py ├── nydus ├── __init__.py ├── conf.py ├── contrib │ ├── __init__.py │ └── ketama.py ├── db │ ├── __init__.py │ ├── backends │ │ ├── __init__.py │ │ ├── base.py │ │ ├── memcache.py │ │ ├── pycassa.py │ │ ├── redis.py │ │ ├── riak.py │ │ └── thoonk.py │ ├── base.py │ ├── exceptions.py │ ├── map.py │ ├── promise.py │ └── routers │ │ ├── __init__.py │ │ ├── base.py │ │ ├── keyvalue.py │ │ └── redis.py ├── testutils.py └── utils.py ├── setup.cfg ├── setup.py └── tests ├── __init__.py └── nydus ├── __init__.py └── db ├── __init__.py ├── backends ├── __init__.py ├── memcache │ ├── __init__.py │ └── tests.py ├── pycassa │ ├── __init__.py │ └── tests.py ├── redis │ ├── __init__.py │ └── tests.py ├── riak │ ├── __init__.py │ └── tests.py └── thoonk │ ├── __init__.py │ └── tests.py ├── connections ├── __init__.py └── tests.py └── routers ├── __init__.py └── tests.py /.arcconfig: -------------------------------------------------------------------------------- 1 | { 2 | "base": "git:merge-base(origin/master), arc:upstream, git:HEAD^", 3 | "project_id": "nydus", 4 | "conduit_uri" : "http://phabricator.local.disqus.net/", 5 | "arcanist_configuration": "DisqusConfiguration", 6 | "copyright_holder": "Disqus, Inc.", 7 | "immutable_history": false, 8 | "differential.field-selector": "DisqusDifferentialFieldSelector", 9 | "lint_engine": "ComprehensiveLintEngine", 10 | "lint.pep8.options": "--ignore=W391,W292,W293,E501,E225", 11 | "lint.jshint.prefix": "node_modules/jshint/bin", 12 | "lint.jshint.bin": "jshint", 13 | "phutil_libraries": { 14 | "disqus": "/usr/local/include/php/libdisqus/src" 15 | } 16 | } 17 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.pyc 2 | *.egg-info 3 | *.gz 4 | *.egg 5 | .tox 6 | .coverage 7 | /dist 8 | /build 9 | /cover 10 | /htmlcov -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: python 2 | services: 3 | - memcached 4 | - redis-server 5 | python: 6 | - "2.6" 7 | - "2.7" 8 | - "3.4" 9 | env: 10 | - DJANGO=1.2.7 11 | - DJANGO=1.3.1 12 | - DJANGO=1.4.12 13 | - DJANGO=1.5.7 14 | - DJANGO=1.6.4 15 | install: 16 | - pip install Django==$DJANGO 17 | - pip install flake8 18 | - pip install -e . 19 | script: 20 | - make test 21 | -------------------------------------------------------------------------------- /AUTHORS: -------------------------------------------------------------------------------- 1 | http://github.com/disqus/nydus/contributors 2 | 3 | Ketama consistent hash algorithm for python written by Andrey Nikishaev: https://github.com/creotiv/PyKetama -------------------------------------------------------------------------------- /CHANGES: -------------------------------------------------------------------------------- 1 | 0.11.0 2 | ------ 3 | 4 | -The StrictRedis client is now used by default. 5 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | 2 | Apache License 3 | Version 2.0, January 2004 4 | http://www.apache.org/licenses/ 5 | 6 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 7 | 8 | 1. Definitions. 9 | 10 | "License" shall mean the terms and conditions for use, reproduction, 11 | and distribution as defined by Sections 1 through 9 of this document. 12 | 13 | "Licensor" shall mean the copyright owner or entity authorized by 14 | the copyright owner that is granting the License. 15 | 16 | "Legal Entity" shall mean the union of the acting entity and all 17 | other entities that control, are controlled by, or are under common 18 | control with that entity. For the purposes of this definition, 19 | "control" means (i) the power, direct or indirect, to cause the 20 | direction or management of such entity, whether by contract or 21 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 22 | outstanding shares, or (iii) beneficial ownership of such entity. 23 | 24 | "You" (or "Your") shall mean an individual or Legal Entity 25 | exercising permissions granted by this License. 26 | 27 | "Source" form shall mean the preferred form for making modifications, 28 | including but not limited to software source code, documentation 29 | source, and configuration files. 30 | 31 | "Object" form shall mean any form resulting from mechanical 32 | transformation or translation of a Source form, including but 33 | not limited to compiled object code, generated documentation, 34 | and conversions to other media types. 35 | 36 | "Work" shall mean the work of authorship, whether in Source or 37 | Object form, made available under the License, as indicated by a 38 | copyright notice that is included in or attached to the work 39 | (an example is provided in the Appendix below). 40 | 41 | "Derivative Works" shall mean any work, whether in Source or Object 42 | form, that is based on (or derived from) the Work and for which the 43 | editorial revisions, annotations, elaborations, or other modifications 44 | represent, as a whole, an original work of authorship. For the purposes 45 | of this License, Derivative Works shall not include works that remain 46 | separable from, or merely link (or bind by name) to the interfaces of, 47 | the Work and Derivative Works thereof. 48 | 49 | "Contribution" shall mean any work of authorship, including 50 | the original version of the Work and any modifications or additions 51 | to that Work or Derivative Works thereof, that is intentionally 52 | submitted to Licensor for inclusion in the Work by the copyright owner 53 | or by an individual or Legal Entity authorized to submit on behalf of 54 | the copyright owner. For the purposes of this definition, "submitted" 55 | means any form of electronic, verbal, or written communication sent 56 | to the Licensor or its representatives, including but not limited to 57 | communication on electronic mailing lists, source code control systems, 58 | and issue tracking systems that are managed by, or on behalf of, the 59 | Licensor for the purpose of discussing and improving the Work, but 60 | excluding communication that is conspicuously marked or otherwise 61 | designated in writing by the copyright owner as "Not a Contribution." 62 | 63 | "Contributor" shall mean Licensor and any individual or Legal Entity 64 | on behalf of whom a Contribution has been received by Licensor and 65 | subsequently incorporated within the Work. 66 | 67 | 2. Grant of Copyright License. Subject to the terms and conditions of 68 | this License, each Contributor hereby grants to You a perpetual, 69 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 70 | copyright license to reproduce, prepare Derivative Works of, 71 | publicly display, publicly perform, sublicense, and distribute the 72 | Work and such Derivative Works in Source or Object form. 73 | 74 | 3. Grant of Patent License. Subject to the terms and conditions of 75 | this License, each Contributor hereby grants to You a perpetual, 76 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 77 | (except as stated in this section) patent license to make, have made, 78 | use, offer to sell, sell, import, and otherwise transfer the Work, 79 | where such license applies only to those patent claims licensable 80 | by such Contributor that are necessarily infringed by their 81 | Contribution(s) alone or by combination of their Contribution(s) 82 | with the Work to which such Contribution(s) was submitted. If You 83 | institute patent litigation against any entity (including a 84 | cross-claim or counterclaim in a lawsuit) alleging that the Work 85 | or a Contribution incorporated within the Work constitutes direct 86 | or contributory patent infringement, then any patent licenses 87 | granted to You under this License for that Work shall terminate 88 | as of the date such litigation is filed. 89 | 90 | 4. Redistribution. You may reproduce and distribute copies of the 91 | Work or Derivative Works thereof in any medium, with or without 92 | modifications, and in Source or Object form, provided that You 93 | meet the following conditions: 94 | 95 | (a) You must give any other recipients of the Work or 96 | Derivative Works a copy of this License; and 97 | 98 | (b) You must cause any modified files to carry prominent notices 99 | stating that You changed the files; and 100 | 101 | (c) You must retain, in the Source form of any Derivative Works 102 | that You distribute, all copyright, patent, trademark, and 103 | attribution notices from the Source form of the Work, 104 | excluding those notices that do not pertain to any part of 105 | the Derivative Works; and 106 | 107 | (d) If the Work includes a "NOTICE" text file as part of its 108 | distribution, then any Derivative Works that You distribute must 109 | include a readable copy of the attribution notices contained 110 | within such NOTICE file, excluding those notices that do not 111 | pertain to any part of the Derivative Works, in at least one 112 | of the following places: within a NOTICE text file distributed 113 | as part of the Derivative Works; within the Source form or 114 | documentation, if provided along with the Derivative Works; or, 115 | within a display generated by the Derivative Works, if and 116 | wherever such third-party notices normally appear. The contents 117 | of the NOTICE file are for informational purposes only and 118 | do not modify the License. You may add Your own attribution 119 | notices within Derivative Works that You distribute, alongside 120 | or as an addendum to the NOTICE text from the Work, provided 121 | that such additional attribution notices cannot be construed 122 | as modifying the License. 123 | 124 | You may add Your own copyright statement to Your modifications and 125 | may provide additional or different license terms and conditions 126 | for use, reproduction, or distribution of Your modifications, or 127 | for any such Derivative Works as a whole, provided Your use, 128 | reproduction, and distribution of the Work otherwise complies with 129 | the conditions stated in this License. 130 | 131 | 5. Submission of Contributions. Unless You explicitly state otherwise, 132 | any Contribution intentionally submitted for inclusion in the Work 133 | by You to the Licensor shall be under the terms and conditions of 134 | this License, without any additional terms or conditions. 135 | Notwithstanding the above, nothing herein shall supersede or modify 136 | the terms of any separate license agreement you may have executed 137 | with Licensor regarding such Contributions. 138 | 139 | 6. Trademarks. This License does not grant permission to use the trade 140 | names, trademarks, service marks, or product names of the Licensor, 141 | except as required for reasonable and customary use in describing the 142 | origin of the Work and reproducing the content of the NOTICE file. 143 | 144 | 7. Disclaimer of Warranty. Unless required by applicable law or 145 | agreed to in writing, Licensor provides the Work (and each 146 | Contributor provides its Contributions) on an "AS IS" BASIS, 147 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 148 | implied, including, without limitation, any warranties or conditions 149 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 150 | PARTICULAR PURPOSE. You are solely responsible for determining the 151 | appropriateness of using or redistributing the Work and assume any 152 | risks associated with Your exercise of permissions under this License. 153 | 154 | 8. Limitation of Liability. In no event and under no legal theory, 155 | whether in tort (including negligence), contract, or otherwise, 156 | unless required by applicable law (such as deliberate and grossly 157 | negligent acts) or agreed to in writing, shall any Contributor be 158 | liable to You for damages, including any direct, indirect, special, 159 | incidental, or consequential damages of any character arising as a 160 | result of this License or out of the use or inability to use the 161 | Work (including but not limited to damages for loss of goodwill, 162 | work stoppage, computer failure or malfunction, or any and all 163 | other commercial damages or losses), even if such Contributor 164 | has been advised of the possibility of such damages. 165 | 166 | 9. Accepting Warranty or Additional Liability. While redistributing 167 | the Work or Derivative Works thereof, You may choose to offer, 168 | and charge a fee for, acceptance of support, warranty, indemnity, 169 | or other liability obligations and/or rights consistent with this 170 | License. However, in accepting such obligations, You may act only 171 | on Your own behalf and on Your sole responsibility, not on behalf 172 | of any other Contributor, and only if You agree to indemnify, 173 | defend, and hold each Contributor harmless for any liability 174 | incurred by, or claims asserted against, such Contributor by reason 175 | of your accepting any such warranty or additional liability. 176 | 177 | END OF TERMS AND CONDITIONS 178 | 179 | APPENDIX: How to apply the Apache License to your work. 180 | 181 | To apply the Apache License to your work, attach the following 182 | boilerplate notice, with the fields enclosed by brackets "[]" 183 | replaced with your own identifying information. (Don't include 184 | the brackets!) The text should be enclosed in the appropriate 185 | comment syntax for the file format. We also recommend that a 186 | file or class name and description of purpose be included on the 187 | same "printed page" as the copyright notice for easier 188 | identification within third-party archives. 189 | 190 | Copyright 2011-2012 DISQUS 191 | 192 | Licensed under the Apache License, Version 2.0 (the "License"); 193 | you may not use this file except in compliance with the License. 194 | You may obtain a copy of the License at 195 | 196 | http://www.apache.org/licenses/LICENSE-2.0 197 | 198 | Unless required by applicable law or agreed to in writing, software 199 | distributed under the License is distributed on an "AS IS" BASIS, 200 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 201 | See the License for the specific language governing permissions and 202 | limitations under the License. -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include setup.py README.rst MANIFEST.in LICENSE 2 | global-exclude *~ -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | test: 2 | pip install flake8 3 | flake8 --exclude=migrations --ignore=E501,E225,E121,E123,E124,E125,E127,E128 --exit-zero nydus || exit 1 4 | python setup.py test 5 | 6 | publish: 7 | python setup.py sdist bdist_wheel upload 8 | 9 | .PHONY: test publish 10 | -------------------------------------------------------------------------------- /README.rst: -------------------------------------------------------------------------------- 1 | Nydus 2 | ===== 3 | 4 | Generic database utilities, including connection clustering and routing so you can scale like a pro. 5 | 6 | The following example creates a Redis connection cluster which will distribute reads and writes based on a simple modulus lookup of the hashed key: 7 | 8 | .. code:: python 9 | 10 | from nydus.db import create_cluster 11 | 12 | redis = create_cluster({ 13 | 'backend': 'nydus.db.backends.redis.Redis', 14 | 'router': 'nydus.db.routers.keyvalue.PartitionRouter', 15 | 'hosts': { 16 | 0: {'db': 0}, 17 | 1: {'db': 1}, 18 | 2: {'db': 2}, 19 | } 20 | }) 21 | 22 | res = redis.incr('foo') 23 | 24 | assert res == 1 25 | 26 | 27 | nydus.db.create_cluster 28 | ----------------------- 29 | 30 | The create_cluster function is a simple helper to configure a ``Cluster`` based on a simple dict config. 31 | 32 | There are two required keyword arguments: 33 | 34 | * ``backend``: full path to the backend class, which should extend ``nydus.db.backends.BaseConnection``. 35 | 36 | * ``hosts``: a dictionary, where the key is an ordered numeric value, and the result is a dict of connection options. 37 | 38 | (the keys are numeric values simply for readability in configuration) 39 | 40 | * ``defaults``: a dictionary of default settings for all hosts to inherit from. 41 | 42 | Optionally, you may also specify a value for ``router``, which is the full path to the router class, 43 | which must extend ``nydus.db.routers.BaseRouter``. 44 | 45 | Distributed Queries 46 | ------------------- 47 | 48 | In some cases you may want to execute a query on many nodes (in parallel). Nydus has built-in support for this when any routing function 49 | returns a cluster of nodes: 50 | 51 | .. code:: python 52 | 53 | from nydus.db import create_cluster 54 | >>> 55 | # by not specifying a router all queries are executed on all hosts 56 | redis = create_cluster({ 57 | 'backend': 'nydus.db.backends.redis.Redis', 58 | 'hosts': { 59 | 0: {'db': 0}, 60 | 1: {'db': 1}, 61 | 2: {'db': 2}, 62 | } 63 | }) 64 | >>> 65 | # map the query over all connections returned by the default router 66 | res = redis.incr('foo') 67 | >>> 68 | assert type(res) == list 69 | assert len(res) == 3 70 | 71 | You can also map many queries (utilizing an internal queue) over connections (again, returned by the router): 72 | 73 | .. code:: python 74 | 75 | with redis.map() as conn: 76 | results = [conn.incr(k) for k in keys] 77 | 78 | As of release 0.5.0, the map() function now supports pipelines, and the included Redis backend will pipeline commands 79 | wherever possible. 80 | 81 | Redis 82 | ----- 83 | 84 | Nydus was originally designed as a toolkit to expand on the usage of Redis at DISQUS. While it does provide 85 | a framework for building clusters that are not Redis, much of the support has gone into providing utilities 86 | for routing and querying on Redis clusters. 87 | 88 | You can configure the Redis client for a connection by specifying it's full path: 89 | 90 | .. code:: python 91 | 92 | redis = create_cluster({ 93 | 'backend': 'nydus.db.backends.redis.Redis', 94 | 'hosts': { 95 | 0: {'db': 0}, 96 | }, 97 | }) 98 | 99 | The available host options are: 100 | 101 | * host 102 | * port 103 | * db 104 | * timeout 105 | * password 106 | * identifier 107 | 108 | The Redis client also supports pipelines via the map command. This means that all commands will hit servers at most 109 | as of once: 110 | 111 | .. code:: python 112 | 113 | with redis.map() as conn: 114 | conn.set('a', 1) 115 | conn.incr('a') 116 | results = [conn.get('a'), conn.get('b')] 117 | results['a'] == 2 118 | results['b'] == None 119 | 120 | Simple Partition Router 121 | ~~~~~~~~~~~~~~~~~~~~~~~ 122 | 123 | There are also several options for builtin routing. The easiest is a simple partition router, which is just a simple 124 | hash on the key: 125 | 126 | .. code:: python 127 | 128 | redis = create_cluster({ 129 | 'backend': 'nydus.db.backends.redis.Redis', 130 | 'router': 'nydus.db.routers.keyvalue.PartitionRouter', 131 | 'hosts': { 132 | 0: {'db': 0}, 133 | }, 134 | }) 135 | 136 | Consistent Hashing Router 137 | ~~~~~~~~~~~~~~~~~~~~~~~~~ 138 | 139 | An improvement upon hashing, Nydus provides a Ketama-based consistent hashing router: 140 | 141 | .. code:: python 142 | 143 | redis = create_cluster({ 144 | 'backend': 'nydus.db.backends.redis.Redis', 145 | 'router': 'nydus.db.routers.keyvalue.ConsistentHashingRouter', 146 | 'hosts': { 147 | 0: {'db': 0}, 148 | }, 149 | }) 150 | 151 | Round Robin Router 152 | ~~~~~~~~~~~~~~~~~~ 153 | 154 | An additional option for distributing queries is the round robin router: 155 | 156 | .. code:: python 157 | 158 | redis = create_cluster({ 159 | 'backend': 'nydus.db.backends.redis.Redis', 160 | 'router': 'nydus.db.routers.RoundRobinRouter', 161 | 'hosts': { 162 | 0: {'db': 0}, 163 | }, 164 | }) 165 | 166 | Pycassa 167 | ------- 168 | 169 | Basic connection management for pycassa (Cassandra) clusters is supported, but we use a non-standard syntax 170 | for creating clusters as routing behavior and per-connection options are not useful in this context: 171 | 172 | .. code:: python 173 | 174 | pycassa = create_cluster({ 175 | 'backend': 'nydus.db.backends.pycassa.Pycassa', 176 | 'hosts': ['localhost'], 177 | 'keyspace': 'test', 178 | }) 179 | 180 | .. note:: Pycassa handles routing of hosts internally, which means things like ``map`` have no affect. 181 | -------------------------------------------------------------------------------- /benchmark.py: -------------------------------------------------------------------------------- 1 | from nydus.db import create_cluster 2 | import time 3 | 4 | partition_cluster = create_cluster({ 5 | 'engine': 'nydus.db.backends.redis.Redis', 6 | 'router': 'nydus.db.routers.keyvalue.PartitionRouter', 7 | 'hosts': { 8 | 0: {'db': 0}, 9 | 1: {'db': 1}, 10 | 2: {'db': 2}, 11 | 3: {'db': 3}, 12 | }, 13 | }) 14 | 15 | ketama_cluster = create_cluster({ 16 | 'engine': 'nydus.db.backends.redis.Redis', 17 | 'router': 'nydus.db.routers.keyvalue.ConsistentHashingRouter', 18 | 'hosts': { 19 | 0: {'db': 0}, 20 | 1: {'db': 1}, 21 | 2: {'db': 2}, 22 | 3: {'db': 3}, 23 | }, 24 | }) 25 | 26 | roundrobin_cluster = create_cluster({ 27 | 'engine': 'nydus.db.backends.redis.Redis', 28 | 'router': 'nydus.db.routers.RoundRobinRouter', 29 | 'hosts': { 30 | 0: {'db': 0}, 31 | 1: {'db': 1}, 32 | 2: {'db': 2}, 33 | 3: {'db': 3}, 34 | }, 35 | }) 36 | 37 | 38 | def test_redis_normal(cluster): 39 | cluster.set('foo', 'bar') 40 | cluster.get('foo') 41 | cluster.set('biz', 'bar') 42 | cluster.get('biz') 43 | cluster.get('bar') 44 | 45 | 46 | def test_redis_map(cluster): 47 | with cluster.map() as conn: 48 | for n in xrange(5): 49 | conn.set('foo', 'bar') 50 | conn.get('foo') 51 | conn.set('biz', 'bar') 52 | conn.get('biz') 53 | conn.get('bar') 54 | 55 | 56 | def main(iterations=1000): 57 | for cluster in ('partition_cluster', 'ketama_cluster', 'roundrobin_cluster'): 58 | for func in ('test_redis_normal', 'test_redis_map'): 59 | print "Running %r on %r" % (func, cluster) 60 | s = time.time() 61 | for x in xrange(iterations): 62 | globals()[func](globals()[cluster]) 63 | t = (time.time() - s) * 1000 64 | print " %.3fms per iteration" % (t / iterations,) 65 | 66 | 67 | if __name__ == '__main__': 68 | main() 69 | -------------------------------------------------------------------------------- /nydus/__init__.py: -------------------------------------------------------------------------------- 1 | """ 2 | nydus 3 | ~~~~~ 4 | 5 | :copyright: (c) 2011-2012 DISQUS. 6 | :license: Apache License 2.0, see LICENSE for more details. 7 | """ 8 | 9 | try: 10 | VERSION = __import__('pkg_resources') \ 11 | .get_distribution('nydus').version 12 | except Exception: 13 | VERSION = 'unknown' 14 | -------------------------------------------------------------------------------- /nydus/conf.py: -------------------------------------------------------------------------------- 1 | """ 2 | nydus.conf 3 | ~~~~~~~~~~ 4 | 5 | :copyright: (c) 2011-2012 DISQUS. 6 | :license: Apache License 2.0, see LICENSE for more details. 7 | """ 8 | 9 | import warnings 10 | 11 | CONNECTIONS = {} 12 | 13 | 14 | def configure(kwargs): 15 | for k, v in kwargs.iteritems(): 16 | if k.upper() != k: 17 | warnings.warn('Invalid setting, \'%s\' which is not defined by Nydus' % k) 18 | elif k not in globals(): 19 | warnings.warn('Invalid setting, \'%s\' which is not defined by Nydus' % k) 20 | else: 21 | globals()[k] = v 22 | -------------------------------------------------------------------------------- /nydus/contrib/__init__.py: -------------------------------------------------------------------------------- 1 | """ 2 | nydus.contrib 3 | ~~~~~~~~~~~~~ 4 | 5 | :copyright: (c) 2011-2012 DISQUS. 6 | :license: Apache License 2.0, see LICENSE for more details. 7 | """ 8 | -------------------------------------------------------------------------------- /nydus/contrib/ketama.py: -------------------------------------------------------------------------------- 1 | """ 2 | Ketama consistent hash algorithm. 3 | 4 | Rewrited from the original source: http://www.audioscrobbler.net/development/ketama/ 5 | 6 | """ 7 | __author__ = "Andrey Nikishaev" 8 | __email__ = "creotiv@gmail.com" 9 | __version__ = 0.1 10 | __status__ = "productrion" 11 | 12 | __all__ = ['Ketama'] 13 | 14 | import hashlib 15 | import math 16 | from bisect import bisect 17 | 18 | 19 | class Ketama(object): 20 | 21 | def __init__(self, nodes=None, weights=None): 22 | """ 23 | nodes - List of nodes(strings) 24 | weights - Dictionary of node wheights where keys are nodes names. 25 | if not set, all nodes will be equal. 26 | """ 27 | self._hashring = dict() 28 | self._sorted_keys = [] 29 | 30 | self._nodes = set(nodes or []) 31 | self._weights = weights if weights else {} 32 | 33 | self._build_circle() 34 | 35 | def _build_circle(self): 36 | """ 37 | Creates hash ring. 38 | """ 39 | total_weight = 0 40 | for node in self._nodes: 41 | total_weight += self._weights.get(node, 1) 42 | 43 | for node in self._nodes: 44 | weight = self._weights.get(node, 1) 45 | 46 | ks = math.floor((40 * len(self._nodes) * weight) / total_weight) 47 | 48 | for i in xrange(0, int(ks)): 49 | b_key = self._md5_digest('%s-%s-salt' % (node, i)) 50 | 51 | for l in xrange(0, 4): 52 | key = ((b_key[3 + l * 4] << 24) 53 | | (b_key[2 + l * 4] << 16) 54 | | (b_key[1 + l * 4] << 8) 55 | | b_key[l * 4]) 56 | 57 | self._hashring[key] = node 58 | self._sorted_keys.append(key) 59 | 60 | self._sorted_keys.sort() 61 | 62 | def _get_node_pos(self, key): 63 | """ 64 | Return node position(integer) for a given key. Else return None 65 | """ 66 | if not self._hashring: 67 | return None 68 | 69 | key = self._gen_key(key) 70 | 71 | nodes = self._sorted_keys 72 | pos = bisect(nodes, key) 73 | 74 | if pos == len(nodes): 75 | return 0 76 | return pos 77 | 78 | def _gen_key(self, key): 79 | """ 80 | Return long integer for a given key, that represent it place on 81 | the hash ring. 82 | """ 83 | b_key = self._md5_digest(key) 84 | return self._hashi(b_key, lambda x: x) 85 | 86 | def _hashi(self, b_key, fn): 87 | return ((b_key[fn(3)] << 24) 88 | | (b_key[fn(2)] << 16) 89 | | (b_key[fn(1)] << 8) 90 | | b_key[fn(0)]) 91 | 92 | def _md5_digest(self, key): 93 | return map(ord, hashlib.md5(key).digest()) 94 | 95 | def remove_node(self, node): 96 | """ 97 | Removes node from circle and rebuild it. 98 | """ 99 | try: 100 | self._nodes.remove(node) 101 | del self._weights[node] 102 | except (KeyError, ValueError): 103 | pass 104 | self._hashring = dict() 105 | self._sorted_keys = [] 106 | 107 | self._build_circle() 108 | 109 | def add_node(self, node, weight=1): 110 | """ 111 | Adds node to circle and rebuild it. 112 | """ 113 | self._nodes.add(node) 114 | self._weights[node] = weight 115 | self._hashring = dict() 116 | self._sorted_keys = [] 117 | 118 | self._build_circle() 119 | 120 | def get_node(self, key): 121 | """ 122 | Return node for a given key. Else return None. 123 | """ 124 | pos = self._get_node_pos(key) 125 | if pos is None: 126 | return None 127 | return self._hashring[self._sorted_keys[pos]] 128 | 129 | 130 | if __name__ == '__main__': 131 | def test(k): 132 | data = {} 133 | for i in xrange(REQUESTS): 134 | tower = k.get_node('a' + str(i)) 135 | data.setdefault(tower, 0) 136 | data[tower] += 1 137 | print 'Number of caches on each node: ' 138 | print data 139 | print '' 140 | 141 | print k.get_node('Aplple') 142 | print k.get_node('Hello') 143 | print k.get_node('Data') 144 | print k.get_node('Computer') 145 | 146 | NODES = [ 147 | '192.168.0.1:6000', '192.168.0.1:6001', '192.168.0.1:6002', 148 | '192.168.0.1:6003', '192.168.0.1:6004', '192.168.0.1:6005', 149 | '192.168.0.1:6006', '192.168.0.1:6008', '192.168.0.1:6007' 150 | ] 151 | REQUESTS = 1000 152 | 153 | k = Ketama(NODES) 154 | 155 | test(k) 156 | 157 | k.remove_node('192.168.0.1:6007') 158 | 159 | test(k) 160 | 161 | k.add_node('192.168.0.1:6007') 162 | 163 | test(k) 164 | -------------------------------------------------------------------------------- /nydus/db/__init__.py: -------------------------------------------------------------------------------- 1 | """ 2 | nydus.db 3 | ~~~~~~~~ 4 | 5 | Disqus generic connections wrappers. 6 | 7 | >>> from nydus.db import create_cluster 8 | >>> redis = create_cluster({ 9 | >>> 'backend': 'nydus.db.backends.redis.Redis', 10 | >>> }) 11 | >>> res = conn.incr('foo') 12 | >>> assert res == 1 13 | 14 | :copyright: (c) 2011-2012 DISQUS. 15 | :license: Apache License 2.0, see LICENSE for more details. 16 | """ 17 | 18 | __all__ = ('create_cluster', 'connections', 'Cluster') 19 | 20 | import copy 21 | 22 | from nydus import conf 23 | from nydus.db.base import LazyConnectionHandler 24 | from nydus.db.routers.base import BaseRouter 25 | from nydus.utils import import_string, apply_defaults 26 | 27 | 28 | def create_cluster(settings): 29 | """ 30 | Creates a new Nydus cluster from the given settings. 31 | 32 | :param settings: Dictionary of the cluster settings. 33 | :returns: Configured instance of ``nydus.db.base.Cluster``. 34 | 35 | >>> redis = create_cluster({ 36 | >>> 'backend': 'nydus.db.backends.redis.Redis', 37 | >>> 'router': 'nydus.db.routers.redis.PartitionRouter', 38 | >>> 'defaults': { 39 | >>> 'host': 'localhost', 40 | >>> 'port': 6379, 41 | >>> }, 42 | >>> 'hosts': { 43 | >>> 0: {'db': 0}, 44 | >>> 1: {'db': 1}, 45 | >>> 2: {'db': 2}, 46 | >>> } 47 | >>> }) 48 | """ 49 | # Pull in our client 50 | settings = copy.deepcopy(settings) 51 | backend = settings.pop('engine', settings.pop('backend', None)) 52 | if isinstance(backend, basestring): 53 | Conn = import_string(backend) 54 | elif backend: 55 | Conn = backend 56 | else: 57 | raise KeyError('backend') 58 | 59 | # Pull in our cluster 60 | cluster = settings.pop('cluster', None) 61 | if not cluster: 62 | Cluster = Conn.get_cluster() 63 | elif isinstance(cluster, basestring): 64 | Cluster = import_string(cluster) 65 | else: 66 | Cluster = cluster 67 | 68 | # Pull in our router 69 | router = settings.pop('router', None) 70 | if not router: 71 | Router = BaseRouter 72 | elif isinstance(router, basestring): 73 | Router = import_string(router) 74 | else: 75 | Router = router 76 | 77 | # Build the connection cluster 78 | return Cluster( 79 | router=Router, 80 | backend=Conn, 81 | **settings 82 | ) 83 | 84 | connections = LazyConnectionHandler(lambda: conf.CONNECTIONS) 85 | -------------------------------------------------------------------------------- /nydus/db/backends/__init__.py: -------------------------------------------------------------------------------- 1 | """ 2 | nydus.db.backends 3 | ~~~~~~~~~~~~~~~~~ 4 | 5 | :copyright: (c) 2011-2012 DISQUS. 6 | :license: Apache License 2.0, see LICENSE for more details. 7 | """ 8 | 9 | from .base import BaseConnection, BasePipeline 10 | -------------------------------------------------------------------------------- /nydus/db/backends/base.py: -------------------------------------------------------------------------------- 1 | """ 2 | nydus.db.backends.base 3 | ~~~~~~~~~~~~~~~~~~~~~~ 4 | 5 | :copyright: (c) 2011-2012 DISQUS. 6 | :license: Apache License 2.0, see LICENSE for more details. 7 | """ 8 | 9 | __all__ = ('BaseConnection',) 10 | 11 | from nydus.db.base import BaseCluster 12 | 13 | 14 | class BasePipeline(object): 15 | """ 16 | Base Pipeline class. 17 | 18 | This basically is absolutely useless, and just provides a sample 19 | API for dealing with pipelined commands. 20 | """ 21 | def __init__(self, connection): 22 | self.connection = connection 23 | self.pending = [] 24 | 25 | def add(self, command): 26 | """ 27 | Add a command to the pending execution pipeline. 28 | """ 29 | self.pending.append(command) 30 | 31 | def execute(self): 32 | """ 33 | Execute all pending commands and return a list of the results 34 | ordered by call. 35 | """ 36 | raise NotImplementedError 37 | 38 | 39 | class BaseConnection(object): 40 | """ 41 | Base connection class. 42 | 43 | Child classes should implement at least 44 | connect() and disconnect() methods. 45 | """ 46 | 47 | retryable_exceptions = () 48 | supports_pipelines = False 49 | 50 | def __init__(self, num, **options): 51 | self._connection = None 52 | self.num = num 53 | 54 | def __getattr__(self, name): 55 | return getattr(self.connection, name) 56 | 57 | @property 58 | def identifier(self): 59 | return repr(self) 60 | 61 | @property 62 | def connection(self): 63 | if self._connection is None: 64 | self._connection = self.connect() 65 | return self._connection 66 | 67 | def close(self): 68 | """ 69 | Close the connection if it is open. 70 | """ 71 | if self._connection: 72 | self.disconnect() 73 | self._connection = None 74 | 75 | def connect(self): 76 | """ 77 | Connect. 78 | 79 | Must return a connection object. 80 | """ 81 | raise NotImplementedError 82 | 83 | def disconnect(self): 84 | """ 85 | Disconnect. 86 | """ 87 | raise NotImplementedError 88 | 89 | def get_pipeline(self): 90 | """ 91 | Return a new pipeline instance (bound to this connection). 92 | """ 93 | raise NotImplementedError 94 | 95 | @classmethod 96 | def get_cluster(cls): 97 | """ 98 | Return the default cluster type for this backend. 99 | """ 100 | return BaseCluster 101 | -------------------------------------------------------------------------------- /nydus/db/backends/memcache.py: -------------------------------------------------------------------------------- 1 | """ 2 | nydus.db.backends.memcache 3 | ~~~~~~~~~~~~~~~~~~~~~~~~~~ 4 | 5 | :copyright: (c) 2012 DISQUS. 6 | :license: Apache License 2.0, see LICENSE for more details. 7 | """ 8 | 9 | from __future__ import absolute_import 10 | 11 | import pylibmc 12 | 13 | from itertools import izip 14 | from nydus.db.backends import BaseConnection, BasePipeline 15 | from nydus.db.promise import EventualCommand 16 | from nydus.utils import peek 17 | 18 | 19 | class Memcache(BaseConnection): 20 | 21 | retryable_exceptions = frozenset([pylibmc.Error]) 22 | supports_pipelines = True 23 | 24 | def __init__(self, num, host='localhost', port=11211, binary=True, 25 | behaviors=None, **options): 26 | self.host = host 27 | self.port = port 28 | self.binary = binary 29 | self.behaviors = behaviors 30 | super(Memcache, self).__init__(num) 31 | 32 | @property 33 | def identifier(self): 34 | mapping = vars(self) 35 | return "memcache://%(host)s:%(port)s/" % mapping 36 | 37 | def connect(self): 38 | host = "%s:%i" % (self.host, self.port) 39 | return pylibmc.Client([host], binary=self.binary, behaviors=self.behaviors) 40 | 41 | def disconnect(self): 42 | self.connection.disconnect_all() 43 | 44 | def get_pipeline(self, *args, **kwargs): 45 | return MemcachePipeline(self) 46 | 47 | 48 | class MemcachePipeline(BasePipeline): 49 | def execute(self): 50 | grouped = regroup_commands(self.pending) 51 | results = resolve_grouped_commands(grouped, self.connection) 52 | return results 53 | 54 | 55 | def grouped_args_for_command(command): 56 | """ 57 | Returns a list of arguments that are shared for this command. 58 | 59 | When comparing similar commands, these arguments represent the 60 | groupable signature for said commands. 61 | """ 62 | if command.get_name() == 'set': 63 | return command.get_args()[2:] 64 | return command.get_args()[1:] 65 | 66 | 67 | def grouped_command(commands): 68 | """ 69 | Given a list of commands (which are assumed groupable), return 70 | a new command which is a batch (multi) command. 71 | 72 | For ``set`` commands the outcome will be:: 73 | 74 | set_multi({key: value}, **kwargs) 75 | 76 | For ``get`` and ``delete`` commands, the outcome will be:: 77 | 78 | get_multi(list_of_keys, **kwargs) 79 | 80 | (Or respectively ``delete_multi``) 81 | """ 82 | base = commands[0] 83 | name = base.get_name() 84 | multi_command = EventualCommand('%s_multi' % name) 85 | if name in ('get', 'delete'): 86 | args = [c.get_args()[0] for c in commands] 87 | elif base.get_name() == 'set': 88 | args = dict(c.get_args()[0:2] for c in commands) 89 | else: 90 | raise ValueError('Command not supported: %r' % (base.get_name(),)) 91 | 92 | multi_command(args, *grouped_args_for_command(base), **base.get_kwargs()) 93 | 94 | return multi_command 95 | 96 | 97 | def can_group_commands(command, next_command): 98 | """ 99 | Returns a boolean representing whether these commands can be 100 | grouped together or not. 101 | 102 | A few things are taken into account for this decision: 103 | 104 | For ``set`` commands: 105 | 106 | - Are all arguments other than the key/value the same? 107 | 108 | For ``delete`` and ``get`` commands: 109 | 110 | - Are all arguments other than the key the same? 111 | """ 112 | multi_capable_commands = ('get', 'set', 'delete') 113 | 114 | if next_command is None: 115 | return False 116 | 117 | name = command.get_name() 118 | 119 | # TODO: support multi commands 120 | if name not in multi_capable_commands: 121 | return False 122 | 123 | if name != next_command.get_name(): 124 | return False 125 | 126 | # if the shared args (key, or key/value) do not match, we cannot group 127 | if grouped_args_for_command(command) != grouped_args_for_command(next_command): 128 | return False 129 | 130 | # If the keyword arguments do not much (e.g. key_prefix, or timeout on set) 131 | # then we cannot group 132 | if command.get_kwargs() != next_command.get_kwargs(): 133 | return False 134 | 135 | return True 136 | 137 | 138 | def regroup_commands(commands): 139 | """ 140 | Returns a list of tuples: 141 | 142 | [(command_to_run, [list, of, commands])] 143 | 144 | If the list of commands has a single item, the command was not grouped. 145 | """ 146 | grouped = [] 147 | pending = [] 148 | 149 | def group_pending(): 150 | if not pending: 151 | return 152 | 153 | new_command = grouped_command(pending) 154 | result = [] 155 | while pending: 156 | result.append(pending.pop(0)) 157 | grouped.append((new_command, result)) 158 | 159 | for command, next_command in peek(commands): 160 | # if the previous command was a get, and this is a set we must execute 161 | # any pending commands 162 | # TODO: unless this command is a get_multi and it matches the same option 163 | # signature 164 | if can_group_commands(command, next_command): 165 | # if previous command does not match this command 166 | if pending and not can_group_commands(pending[0], command): 167 | group_pending() 168 | 169 | pending.append(command) 170 | else: 171 | # if pending exists for this command, group it 172 | if pending and can_group_commands(pending[0], command): 173 | pending.append(command) 174 | else: 175 | grouped.append((command.clone(), [command])) 176 | 177 | # We couldn't group with previous command, so ensure we bubble up 178 | group_pending() 179 | 180 | group_pending() 181 | 182 | return grouped 183 | 184 | 185 | def resolve_grouped_commands(grouped, connection): 186 | results = {} 187 | 188 | for master_command, grouped_commands in grouped: 189 | result = master_command.resolve(connection) 190 | 191 | # this command was not grouped 192 | if len(grouped_commands) == 1: 193 | results[grouped_commands[0]] = result 194 | else: 195 | if isinstance(result, dict): 196 | # XXX: assume first arg is key 197 | for command in grouped_commands: 198 | results[command] = result.get(command.get_args()[0]) 199 | else: 200 | for command, value in izip(grouped_commands, result): 201 | results[command] = value 202 | 203 | return results 204 | -------------------------------------------------------------------------------- /nydus/db/backends/pycassa.py: -------------------------------------------------------------------------------- 1 | """ 2 | nydus.db.backends.pycassa 3 | ~~~~~~~~~~~~~~~~~~~~~~~~~ 4 | 5 | :copyright: (c) 2011-2012 DISQUS. 6 | :license: Apache License 2.0, see LICENSE for more details. 7 | """ 8 | 9 | from __future__ import absolute_import 10 | 11 | import collections 12 | from nydus.db.base import BaseCluster 13 | from nydus.db.backends import BaseConnection 14 | from pycassa import ConnectionPool 15 | 16 | 17 | class Pycassa(BaseConnection): 18 | def __init__(self, num, keyspace, hosts=['localhost'], **options): 19 | self.keyspace = keyspace 20 | self.hosts = hosts 21 | self.options = options 22 | super(Pycassa, self).__init__(num) 23 | 24 | @property 25 | def identifier(self): 26 | return "pycassa://%(hosts)s/%(keyspace)s" % { 27 | 'hosts': ','.join(self.hosts), 28 | 'keyspace': self.keyspace, 29 | } 30 | 31 | def connect(self): 32 | return ConnectionPool(keyspace=self.keyspace, server_list=self.hosts, **self.options) 33 | 34 | def disconnect(self): 35 | self.connection.dispose() 36 | 37 | @classmethod 38 | def get_cluster(cls): 39 | return PycassaCluster 40 | 41 | 42 | class PycassaCluster(BaseCluster): 43 | """ 44 | A PycassaCluster has a single host as pycassa internally handles routing 45 | and communication within a set of nodes. 46 | """ 47 | def __init__(self, hosts=None, keyspace=None, backend=Pycassa, **kwargs): 48 | assert isinstance(hosts, collections.Iterable), 'hosts must be an iterable' 49 | assert keyspace, 'keyspace must be set' 50 | 51 | return super(PycassaCluster, self).__init__( 52 | hosts={ 53 | 0: { 54 | 'hosts': hosts, 55 | 'keyspace': keyspace, 56 | }, 57 | }, 58 | backend=backend, 59 | **kwargs 60 | ) 61 | -------------------------------------------------------------------------------- /nydus/db/backends/redis.py: -------------------------------------------------------------------------------- 1 | """ 2 | nydus.db.backends.redis 3 | ~~~~~~~~~~~~~~~~~~~~~~~ 4 | 5 | :copyright: (c) 2011-2012 DISQUS. 6 | :license: Apache License 2.0, see LICENSE for more details. 7 | """ 8 | 9 | from __future__ import absolute_import 10 | 11 | from itertools import izip 12 | from redis import Redis as RedisClient, StrictRedis 13 | from redis import ConnectionError, InvalidResponse 14 | 15 | from nydus.db.backends import BaseConnection, BasePipeline 16 | 17 | 18 | class RedisPipeline(BasePipeline): 19 | def __init__(self, connection): 20 | self.pending = [] 21 | self.connection = connection 22 | self.pipe = connection.pipeline() 23 | 24 | def add(self, command): 25 | name, args, kwargs = command.get_command() 26 | self.pending.append(command) 27 | # ensure the command is executed in the pipeline 28 | getattr(self.pipe, name)(*args, **kwargs) 29 | 30 | def execute(self): 31 | return dict(izip(self.pending, self.pipe.execute())) 32 | 33 | 34 | class Redis(BaseConnection): 35 | # Exceptions that can be retried by this backend 36 | retryable_exceptions = frozenset([ConnectionError, InvalidResponse]) 37 | supports_pipelines = True 38 | 39 | def __init__(self, num, host='localhost', port=6379, db=0, timeout=None, 40 | password=None, unix_socket_path=None, identifier=None, 41 | strict=True): 42 | self.host = host 43 | self.port = port 44 | self.db = db 45 | self.unix_socket_path = unix_socket_path 46 | self.timeout = timeout 47 | self.strict = strict 48 | self.__identifier = identifier 49 | self.__password = password 50 | super(Redis, self).__init__(num) 51 | 52 | @property 53 | def identifier(self): 54 | if self.__identifier is not None: 55 | return self.__identifier 56 | mapping = vars(self) 57 | mapping['klass'] = self.__class__.__name__ 58 | return "redis://%(host)s:%(port)s/%(db)s" % mapping 59 | 60 | def connect(self): 61 | if self.strict: 62 | cls = StrictRedis 63 | else: 64 | cls = RedisClient 65 | return cls( 66 | host=self.host, port=self.port, db=self.db, 67 | socket_timeout=self.timeout, password=self.__password, 68 | unix_socket_path=self.unix_socket_path) 69 | 70 | def disconnect(self): 71 | self.connection.disconnect() 72 | 73 | def get_pipeline(self, *args, **kwargs): 74 | return RedisPipeline(self) 75 | -------------------------------------------------------------------------------- /nydus/db/backends/riak.py: -------------------------------------------------------------------------------- 1 | """ 2 | nydus.db.backends.riak 3 | ~~~~~~~~~~~~~~~~~~~~~~ 4 | 5 | :copyright: (c) 2011-2012 DISQUS. 6 | :license: Apache License 2.0, see LICENSE for more details. 7 | """ 8 | 9 | from __future__ import absolute_import 10 | 11 | import socket 12 | import httplib 13 | 14 | from riak import RiakClient, RiakError 15 | 16 | from nydus.db.backends import BaseConnection 17 | 18 | 19 | class Riak(BaseConnection): 20 | # Exceptions that can be retried by this backend 21 | retryable_exceptions = frozenset([socket.error, httplib.HTTPException, RiakError]) 22 | supports_pipelines = False 23 | 24 | def __init__(self, num, host='127.0.0.1', port=8098, prefix='riak', mapred_prefix='mapred', client_id=None, 25 | transport_class=None, solr_transport_class=None, transport_options=None, **options): 26 | 27 | self.host = host 28 | self.port = port 29 | self.prefix = prefix 30 | self.mapred_prefix = mapred_prefix 31 | self.client_id = client_id 32 | self.transport_class = transport_class 33 | self.solr_transport_class = solr_transport_class 34 | self.transport_options = transport_options or {} 35 | super(Riak, self).__init__(num) 36 | 37 | @property 38 | def identifier(self): 39 | mapping = vars(self) 40 | return "http://%(host)s:%(port)s/%(prefix)s" % mapping 41 | 42 | def connect(self): 43 | return RiakClient( 44 | host=self.host, port=self.port, prefix=self.prefix, 45 | mapred_prefix=self.mapred_prefix, client_id=self.client_id, 46 | transport_class=self.transport_class, solr_transport_class=self.solr_transport_class, 47 | transport_options=self.transport_options) 48 | 49 | def disconnect(self): 50 | pass 51 | -------------------------------------------------------------------------------- /nydus/db/backends/thoonk.py: -------------------------------------------------------------------------------- 1 | """ 2 | nydus.db.backends.thoonk 3 | ~~~~~~~~~~~~~~~~~~~~~~~~ 4 | 5 | :copyright: (c) 2011-2012 DISQUS. 6 | :license: Apache License 2.0, see LICENSE for more details. 7 | """ 8 | 9 | from __future__ import absolute_import 10 | 11 | from thoonk import Pubsub 12 | 13 | from redis import RedisError 14 | 15 | from nydus.db.backends import BaseConnection 16 | 17 | 18 | class Thoonk(BaseConnection): 19 | # Exceptions that can be retried by this backend 20 | retryable_exceptions = frozenset([RedisError]) 21 | supports_pipelines = False 22 | 23 | def __init__(self, num, host='localhost', port=6379, db=0, timeout=None, listen=False): 24 | self.host = host 25 | self.port = port 26 | self.db = db 27 | self.timeout = timeout 28 | self.pubsub = None 29 | self.listen = listen 30 | super(Thoonk, self).__init__(num) 31 | 32 | @property 33 | def identifier(self): 34 | mapping = vars(self) 35 | mapping['klass'] = self.__class__.__name__ 36 | return "redis://%(host)s:%(port)s/%(db)s" % mapping 37 | 38 | def connect(self): 39 | return Pubsub(host=self.host, port=self.port, db=self.db, listen=self.listen) 40 | 41 | def disconnect(self): 42 | self.connection.close() 43 | 44 | def flushdb(self): 45 | """the tests assume this function exists for all redis-like backends""" 46 | self.connection.redis.flushdb() 47 | -------------------------------------------------------------------------------- /nydus/db/base.py: -------------------------------------------------------------------------------- 1 | """ 2 | nydus.db.base 3 | ~~~~~~~~~~~~~ 4 | 5 | :copyright: (c) 2011-2012 DISQUS. 6 | :license: Apache License 2.0, see LICENSE for more details. 7 | """ 8 | 9 | __all__ = ('LazyConnectionHandler', 'BaseCluster') 10 | 11 | import collections 12 | from nydus.db.map import DistributedContextManager 13 | from nydus.db.routers import BaseRouter, routing_params 14 | from nydus.utils import apply_defaults 15 | 16 | 17 | def iter_hosts(hosts): 18 | # this can either be a dictionary (with the key acting as the numeric 19 | # index) or it can be a sorted list. 20 | if isinstance(hosts, collections.Mapping): 21 | return hosts.iteritems() 22 | return enumerate(hosts) 23 | 24 | 25 | def create_connection(Connection, num, host_settings, defaults): 26 | # host_settings can be an iterable or a dictionary depending on the style 27 | # of connection (some connections share options and simply just need to 28 | # pass a single host, or a list of hosts) 29 | if isinstance(host_settings, collections.Mapping): 30 | return Connection(num, **apply_defaults(host_settings, defaults or {})) 31 | elif isinstance(host_settings, collections.Iterable): 32 | return Connection(num, *host_settings, **defaults or {}) 33 | return Connection(num, host_settings, **defaults or {}) 34 | 35 | 36 | class BaseCluster(object): 37 | """ 38 | Holds a cluster of connections. 39 | """ 40 | class MaxRetriesExceededError(Exception): 41 | pass 42 | 43 | def __init__(self, hosts, backend, router=BaseRouter, max_connection_retries=20, defaults=None): 44 | self.hosts = dict( 45 | (conn_number, create_connection(backend, conn_number, host_settings, defaults)) 46 | for conn_number, host_settings 47 | in iter_hosts(hosts) 48 | ) 49 | self.max_connection_retries = max_connection_retries 50 | self.install_router(router) 51 | 52 | def __len__(self): 53 | return len(self.hosts) 54 | 55 | def __getitem__(self, name): 56 | return self.hosts[name] 57 | 58 | def __getattr__(self, name): 59 | return CallProxy(self, name) 60 | 61 | def __iter__(self): 62 | for name in self.hosts.iterkeys(): 63 | yield name 64 | 65 | def install_router(self, router): 66 | self.router = router(self) 67 | 68 | def execute(self, path, args, kwargs): 69 | connections = self.__connections_for(path, args=args, kwargs=kwargs) 70 | 71 | results = [] 72 | for conn in connections: 73 | for retry in xrange(self.max_connection_retries): 74 | func = conn 75 | for piece in path.split('.'): 76 | func = getattr(func, piece) 77 | try: 78 | results.append(func(*args, **kwargs)) 79 | except tuple(conn.retryable_exceptions), e: 80 | if not self.router.retryable: 81 | raise e 82 | elif retry == self.max_connection_retries - 1: 83 | raise self.MaxRetriesExceededError(e) 84 | else: 85 | conn = self.__connections_for(path, retry_for=conn.num, args=args, kwargs=kwargs)[0] 86 | else: 87 | break 88 | 89 | # If we only had one db to query, we simply return that res 90 | if len(results) == 1: 91 | return results[0] 92 | else: 93 | return results 94 | 95 | def disconnect(self): 96 | """Disconnects all connections in cluster""" 97 | for connection in self.hosts.itervalues(): 98 | connection.disconnect() 99 | 100 | def get_conn(self, *args, **kwargs): 101 | """ 102 | Returns a connection object from the router given ``args``. 103 | 104 | Useful in cases where a connection cannot be automatically determined 105 | during all steps of the process. An example of this would be 106 | Redis pipelines. 107 | """ 108 | connections = self.__connections_for('get_conn', args=args, kwargs=kwargs) 109 | 110 | if len(connections) is 1: 111 | return connections[0] 112 | else: 113 | return connections 114 | 115 | def map(self, workers=None, **kwargs): 116 | return DistributedContextManager(self, workers, **kwargs) 117 | 118 | @routing_params 119 | def __connections_for(self, attr, args, kwargs, **fkwargs): 120 | return [self[n] for n in self.router.get_dbs(attr=attr, args=args, kwargs=kwargs, **fkwargs)] 121 | 122 | 123 | class CallProxy(object): 124 | """ 125 | Handles routing function calls to the proper connection. 126 | """ 127 | def __init__(self, cluster, path): 128 | self.__cluster = cluster 129 | self.__path = path 130 | 131 | def __call__(self, *args, **kwargs): 132 | return self.__cluster.execute(self.__path, args, kwargs) 133 | 134 | def __getattr__(self, name): 135 | return CallProxy(self.__cluster, self.__path + '.' + name) 136 | 137 | 138 | class LazyConnectionHandler(dict): 139 | """ 140 | Maps clusters of connections within a dictionary. 141 | """ 142 | def __init__(self, conf_callback): 143 | self.conf_callback = conf_callback 144 | self.conf_settings = {} 145 | self.__is_ready = False 146 | 147 | def __getitem__(self, key): 148 | if not self.is_ready(): 149 | self.reload() 150 | return super(LazyConnectionHandler, self).__getitem__(key) 151 | 152 | def is_ready(self): 153 | return self.__is_ready 154 | 155 | def reload(self): 156 | from nydus.db import create_cluster 157 | 158 | for conn_alias, conn_settings in self.conf_callback().iteritems(): 159 | self[conn_alias] = create_cluster(conn_settings) 160 | self._is_ready = True 161 | 162 | def disconnect(self): 163 | """Disconnects all connections in cluster""" 164 | for connection in self.itervalues(): 165 | connection.disconnect() 166 | -------------------------------------------------------------------------------- /nydus/db/exceptions.py: -------------------------------------------------------------------------------- 1 | """ 2 | nydus.db.exceptions 3 | ~~~~~~~~~~~~~~~~~~~ 4 | 5 | :copyright: (c) 2011-2012 DISQUS. 6 | :license: Apache License 2.0, see LICENSE for more details. 7 | """ 8 | 9 | 10 | class CommandError(Exception): 11 | def __init__(self, errors): 12 | self.errors = errors 13 | 14 | def __repr__(self): 15 | return '<%s (%d): %r>' % (type(self), len(self.errors), self.errors) 16 | 17 | def __str__(self): 18 | return '%d command(s) failed: %r' % (len(self.errors), self.errors) 19 | -------------------------------------------------------------------------------- /nydus/db/map.py: -------------------------------------------------------------------------------- 1 | """ 2 | nydus.db.map 3 | ~~~~~~~~~~~~ 4 | 5 | :copyright: (c) 2011 DISQUS. 6 | :license: Apache License 2.0, see LICENSE for more details. 7 | """ 8 | 9 | from collections import defaultdict 10 | from nydus.utils import ThreadPool 11 | from nydus.db.exceptions import CommandError 12 | from nydus.db.promise import EventualCommand, change_resolution 13 | 14 | 15 | class BaseDistributedConnection(object): 16 | def __init__(self, cluster, workers=None, fail_silently=False): 17 | self._commands = [] 18 | self._complete = False 19 | self._errors = [] 20 | self._resolved = False 21 | self._cluster = cluster 22 | self._fail_silently = fail_silently 23 | self._workers = min(workers or len(cluster), 16) 24 | 25 | def __getattr__(self, attr): 26 | command = EventualCommand(attr) 27 | self._commands.append(command) 28 | return command 29 | 30 | def _build_pending_commands(self): 31 | pending_commands = defaultdict(list) 32 | 33 | # build up a list of pending commands and their routing information 34 | for command in self._commands: 35 | if not command.was_called(): 36 | continue 37 | 38 | if self._cluster.router: 39 | name, args, kwargs = command.get_command() 40 | db_nums = self._cluster.router.get_dbs( 41 | cluster=self._cluster, 42 | attr=name, 43 | args=args, 44 | kwargs=kwargs, 45 | ) 46 | else: 47 | db_nums = self._cluster.keys() 48 | 49 | for db_num in db_nums: 50 | # add to pending commands 51 | pending_commands[db_num].append(command) 52 | 53 | return pending_commands 54 | 55 | def get_pool(self, commands): 56 | return ThreadPool(min(self._workers, len(commands))) 57 | 58 | def resolve(self): 59 | pending_commands = self._build_pending_commands() 60 | 61 | num_commands = sum(len(v) for v in pending_commands.itervalues()) 62 | # Don't bother with the pooling if we only need to do one operation on a single machine 63 | if num_commands == 1: 64 | db_num, (command,) = pending_commands.items()[0] 65 | self._commands = [command.resolve(self._cluster[db_num])] 66 | 67 | elif num_commands > 1: 68 | results = self.execute(self._cluster, pending_commands) 69 | 70 | for command in self._commands: 71 | result = results.get(command) 72 | 73 | if result: 74 | for value in result: 75 | if isinstance(value, Exception): 76 | self._errors.append((command.get_name(), value)) 77 | 78 | # XXX: single path routing (implicit) doesnt return a list 79 | if len(result) == 1: 80 | result = result[0] 81 | 82 | change_resolution(command, result) 83 | 84 | self._resolved = True 85 | 86 | if not self._fail_silently and self._errors: 87 | raise CommandError(self._errors) 88 | 89 | def execute(self, cluster, commands): 90 | """ 91 | Execute the given commands on the cluster. 92 | 93 | The result should be a dictionary mapping the original command to the 94 | result value. 95 | """ 96 | raise NotImplementedError 97 | 98 | def get_results(self): 99 | """ 100 | Returns a list of results (once commands have been resolved). 101 | """ 102 | assert self._resolved, 'you must execute the commands before fetching results' 103 | 104 | return self._commands 105 | 106 | def get_errors(self): 107 | """ 108 | Returns a list of errors (once commands have been resolved). 109 | """ 110 | assert self._resolved, 'you must execute the commands before fetching results' 111 | 112 | return self._errors 113 | 114 | 115 | class DistributedConnection(BaseDistributedConnection): 116 | """ 117 | Runs all commands using a simple thread pool, queueing up each command for each database 118 | it needs to run on. 119 | """ 120 | def execute(self, cluster, commands): 121 | # Create the threadpool and pipe jobs into it 122 | pool = self.get_pool(commands) 123 | 124 | # execute our pending commands either in the pool, or using a pipeline 125 | for db_num, command_list in commands.iteritems(): 126 | for command in command_list: 127 | # XXX: its important that we clone the command here so we dont override anything 128 | # in the EventualCommand proxy (it can only resolve once) 129 | pool.add(command, command.clone().resolve, [cluster[db_num]]) 130 | 131 | return dict(pool.join()) 132 | 133 | 134 | class PipelinedDistributedConnection(BaseDistributedConnection): 135 | """ 136 | Runs all commands using pipelines, which will execute a single pipe.execute() call 137 | within a thread pool. 138 | """ 139 | def execute(self, cluster, commands): 140 | # db_num: pipeline object 141 | pipes = {} 142 | 143 | # Create the threadpool and pipe jobs into it 144 | pool = self.get_pool(commands) 145 | 146 | # execute our pending commands either in the pool, or using a pipeline 147 | for db_num, command_list in commands.iteritems(): 148 | pipes[db_num] = cluster[db_num].get_pipeline() 149 | for command in command_list: 150 | # add to pipeline 151 | pipes[db_num].add(command.clone()) 152 | 153 | # We need to finalize our commands with a single execute in pipelines 154 | for db_num, pipe in pipes.iteritems(): 155 | pool.add(db_num, pipe.execute, (), {}) 156 | 157 | # Consolidate commands with their appropriate results 158 | db_result_map = pool.join() 159 | 160 | # Results get grouped by their command signature, so we have to separate the logic 161 | results = defaultdict(list) 162 | 163 | for db_num, db_results in db_result_map.iteritems(): 164 | # Pipelines always execute on a single database 165 | assert len(db_results) == 1 166 | db_results = db_results[0] 167 | 168 | # if pipe.execute (within nydus) fails, this will be an exception object 169 | if isinstance(db_results, Exception): 170 | for command in commands[db_num]: 171 | results[command].append(db_results) 172 | continue 173 | 174 | for command, result in db_results.iteritems(): 175 | results[command].append(result) 176 | 177 | return results 178 | 179 | 180 | class DistributedContextManager(object): 181 | def __init__(self, cluster, workers=None, **kwargs): 182 | if self.can_pipeline(cluster): 183 | cls = PipelinedDistributedConnection 184 | else: 185 | cls = DistributedConnection 186 | self.connection = cls(cluster, workers, **kwargs) 187 | 188 | def __enter__(self): 189 | return self.connection 190 | 191 | def __exit__(self, exc_type, exc_value, tb): 192 | # we need to break up each command and route it 193 | self.connection.resolve() 194 | 195 | def can_pipeline(self, cluster): 196 | return all(cluster[n].supports_pipelines for n in cluster) 197 | -------------------------------------------------------------------------------- /nydus/db/promise.py: -------------------------------------------------------------------------------- 1 | """ 2 | nydus.db.promise 3 | ~~~~~~~~~~~~~~~~ 4 | 5 | :copyright: (c) 2011 DISQUS. 6 | :license: Apache License 2.0, see LICENSE for more details. 7 | """ 8 | 9 | from nydus.db.exceptions import CommandError 10 | from functools import wraps 11 | 12 | 13 | def promise_method(func): 14 | """ 15 | A decorator which ensures that once a method has been marked as resolved 16 | (via Class.__resolved)) will then propagate the attribute (function) call 17 | upstream. 18 | """ 19 | name = func.__name__ 20 | 21 | @wraps(func) 22 | def wrapped(self, *args, **kwargs): 23 | cls_name = type(self).__name__ 24 | if getattr(self, '_%s__resolved' % (cls_name,)): 25 | return getattr(getattr(self, '_%s__wrapped' % (cls_name,)), name)(*args, **kwargs) 26 | return func(self, *args, **kwargs) 27 | return wrapped 28 | 29 | 30 | def change_resolution(command, value): 31 | """ 32 | Public API to change the resolution of an already resolved EventualCommand result value. 33 | """ 34 | command._EventualCommand__wrapped = value 35 | command._EventualCommand__resolved = True 36 | 37 | 38 | class EventualCommand(object): 39 | # introspection support: 40 | __members__ = property(lambda self: self.__dir__()) 41 | 42 | def __init__(self, attr, args=None, kwargs=None): 43 | self.__attr = attr 44 | self.__called = False 45 | self.__wrapped = None 46 | self.__resolved = False 47 | self.__args = args or [] 48 | self.__kwargs = kwargs or {} 49 | self.__ident = ':'.join(map(lambda x: str(hash(str(x))), [self.__attr, self.__args, self.__kwargs])) 50 | 51 | def __call__(self, *args, **kwargs): 52 | self.__called = True 53 | self.__args = args 54 | self.__kwargs = kwargs 55 | self.__ident = ':'.join(map(lambda x: str(hash(str(x))), [self.__attr, self.__args, self.__kwargs])) 56 | return self 57 | 58 | def __hash__(self): 59 | # We return our ident 60 | return hash(self.__ident) 61 | 62 | def __repr__(self): 63 | if self.__resolved: 64 | return repr(self.__wrapped) 65 | return u'' % (self.__attr, self.__args, self.__kwargs) 66 | 67 | def __str__(self): 68 | if self.__resolved: 69 | return str(self.__wrapped) 70 | return repr(self) 71 | 72 | def __unicode__(self): 73 | if self.__resolved: 74 | return unicode(self.__wrapped) 75 | return unicode(repr(self)) 76 | 77 | def __getattr__(self, name): 78 | return getattr(self.__wrapped, name) 79 | 80 | def __setattr__(self, name, value): 81 | if name.startswith('_%s' % (type(self).__name__,)): 82 | return object.__setattr__(self, name, value) 83 | return setattr(self.__wrapped, name, value) 84 | 85 | def __delattr__(self, name): 86 | if name.startswith('_%s' % (type(self).__name__,)): 87 | raise TypeError("can't delete %s." % name) 88 | delattr(self.__wrapped, name) 89 | 90 | def __deepcopy__(self, memo): 91 | from copy import deepcopy 92 | return deepcopy(self.__wrapped, memo) 93 | 94 | # Need to pretend to be the wrapped class, for the sake of objects that care 95 | # about this (especially in equality tests) 96 | def __get_class(self): 97 | return self.__wrapped.__class__ 98 | __class__ = property(__get_class) 99 | 100 | def __dict__(self): 101 | try: 102 | return vars(self.__wrapped) 103 | except RuntimeError: 104 | return AttributeError('__dict__') 105 | __dict__ = property(__dict__) 106 | 107 | def __setitem__(self, key, value): 108 | self.__wrapped[key] = value 109 | 110 | def __delitem__(self, key): 111 | del self.__wrapped[key] 112 | 113 | def __setslice__(self, i, j, seq): 114 | self.__wrapped[i:j] = seq 115 | 116 | def __delslice__(self, i, j): 117 | del self.__wrapped[i:j] 118 | 119 | def __instancecheck__(self, cls): 120 | if self._wrapped is None: 121 | return False 122 | return isinstance(self._wrapped, cls) 123 | 124 | __lt__ = lambda x, o: x.__wrapped < o 125 | __le__ = lambda x, o: x.__wrapped <= o 126 | __eq__ = lambda x, o: x.__wrapped == o 127 | __ne__ = lambda x, o: x.__wrapped != o 128 | __gt__ = lambda x, o: x.__wrapped > o 129 | __ge__ = lambda x, o: x.__wrapped >= o 130 | __cmp__ = lambda x, o: cmp(x.__wrapped, o) 131 | # attributes are currently not callable 132 | # __call__ = lambda x, *a, **kw: x.__wrapped(*a, **kw) 133 | __nonzero__ = lambda x: bool(x.__wrapped) 134 | __len__ = lambda x: len(x.__wrapped) 135 | __getitem__ = lambda x, i: x.__wrapped[i] 136 | __iter__ = lambda x: iter(x.__wrapped) 137 | __contains__ = lambda x, i: i in x.__wrapped 138 | __getslice__ = lambda x, i, j: x.__wrapped[i:j] 139 | __add__ = lambda x, o: x.__wrapped + o 140 | __sub__ = lambda x, o: x.__wrapped - o 141 | __mul__ = lambda x, o: x.__wrapped * o 142 | __floordiv__ = lambda x, o: x.__wrapped // o 143 | __mod__ = lambda x, o: x.__wrapped % o 144 | __divmod__ = lambda x, o: x.__wrapped.__divmod__(o) 145 | __pow__ = lambda x, o: x.__wrapped ** o 146 | __lshift__ = lambda x, o: x.__wrapped << o 147 | __rshift__ = lambda x, o: x.__wrapped >> o 148 | __and__ = lambda x, o: x.__wrapped & o 149 | __xor__ = lambda x, o: x.__wrapped ^ o 150 | __or__ = lambda x, o: x.__wrapped | o 151 | __div__ = lambda x, o: x.__wrapped.__div__(o) 152 | __truediv__ = lambda x, o: x.__wrapped.__truediv__(o) 153 | __neg__ = lambda x: -(x.__wrapped) 154 | __pos__ = lambda x: +(x.__wrapped) 155 | __abs__ = lambda x: abs(x.__wrapped) 156 | __invert__ = lambda x: ~(x.__wrapped) 157 | __complex__ = lambda x: complex(x.__wrapped) 158 | __int__ = lambda x: int(x.__wrapped) 159 | __long__ = lambda x: long(x.__wrapped) 160 | __float__ = lambda x: float(x.__wrapped) 161 | __oct__ = lambda x: oct(x.__wrapped) 162 | __hex__ = lambda x: hex(x.__wrapped) 163 | __index__ = lambda x: x.__wrapped.__index__() 164 | __coerce__ = lambda x, o: x.__coerce__(x, o) 165 | __enter__ = lambda x: x.__enter__() 166 | __exit__ = lambda x, *a, **kw: x.__exit__(*a, **kw) 167 | 168 | @property 169 | def is_error(self): 170 | return isinstance(self.__wrapped, CommandError) 171 | 172 | @promise_method 173 | def was_called(self): 174 | return self.__called 175 | 176 | @promise_method 177 | def resolve(self, conn): 178 | value = getattr(conn, self.__attr)(*self.__args, **self.__kwargs) 179 | return self.resolve_as(value) 180 | 181 | @promise_method 182 | def resolve_as(self, value): 183 | self.__wrapped = value 184 | self.__resolved = True 185 | return value 186 | 187 | @promise_method 188 | def get_command(self): 189 | return (self.__attr, self.__args, self.__kwargs) 190 | 191 | @promise_method 192 | def get_name(self): 193 | return self.__attr 194 | 195 | @promise_method 196 | def get_args(self): 197 | return self.__args 198 | 199 | @promise_method 200 | def get_kwargs(self): 201 | return self.__kwargs 202 | 203 | @promise_method 204 | def set_args(self, args): 205 | self.__args = args 206 | 207 | @promise_method 208 | def set_kwargs(self, kwargs): 209 | self.__kwargs = kwargs 210 | 211 | @promise_method 212 | def clone(self): 213 | return EventualCommand(self.__attr, self.__args, self.__kwargs) 214 | -------------------------------------------------------------------------------- /nydus/db/routers/__init__.py: -------------------------------------------------------------------------------- 1 | """ 2 | nydus.db.routers 3 | ~~~~~~~~~~~~~~~~ 4 | 5 | :copyright: (c) 2011-2012 DISQUS. 6 | :license: Apache License 2.0, see LICENSE for more details. 7 | """ 8 | 9 | from .base import * 10 | -------------------------------------------------------------------------------- /nydus/db/routers/base.py: -------------------------------------------------------------------------------- 1 | """ 2 | nydus.db.base 3 | ~~~~~~~~~~~~~ 4 | 5 | :copyright: (c) 2011-2012 DISQUS. 6 | :license: Apache License 2.0, see LICENSE for more details. 7 | """ 8 | 9 | __all__ = ('BaseRouter', 'RoundRobinRouter', 'routing_params') 10 | 11 | import time 12 | 13 | from functools import wraps 14 | from itertools import cycle 15 | 16 | 17 | def routing_params(func): 18 | @wraps(func) 19 | def wrapped(*args, **kwargs): 20 | if kwargs.get('kwargs') is None: 21 | kwargs['kwargs'] = {} 22 | 23 | if kwargs.get('args') is None: 24 | kwargs['args'] = () 25 | 26 | return func(*args, **kwargs) 27 | wrapped.__wraps__ = getattr(func, '__wraps__', func) 28 | return wrapped 29 | 30 | 31 | class BaseRouter(object): 32 | """ 33 | Handles routing requests to a specific connection in a single cluster. 34 | 35 | For the most part, all public functions will receive arguments as ``key=value`` 36 | pairs and should expect as much. Functions which receive ``args`` and ``kwargs`` 37 | from the calling function will receive default values for those, and need not 38 | worry about handling missing arguments. 39 | """ 40 | retryable = False 41 | 42 | class UnableToSetupRouter(Exception): 43 | pass 44 | 45 | def __init__(self, cluster=None, *args, **kwargs): 46 | self._ready = False 47 | self.cluster = cluster 48 | 49 | @routing_params 50 | def get_dbs(self, attr, args, kwargs, **fkwargs): 51 | """ 52 | Returns a list of db keys to route the given call to. 53 | 54 | :param attr: Name of attribute being called on the connection. 55 | :param args: List of arguments being passed to ``attr``. 56 | :param kwargs: Dictionary of keyword arguments being passed to ``attr``. 57 | 58 | >>> redis = Cluster(router=BaseRouter) 59 | >>> router = redis.router 60 | >>> router.get_dbs('incr', args=('key name', 1)) 61 | [0,1,2] 62 | 63 | """ 64 | if not self._ready: 65 | if not self.setup_router(args=args, kwargs=kwargs, **fkwargs): 66 | raise self.UnableToSetupRouter() 67 | 68 | retval = self._pre_routing(attr=attr, args=args, kwargs=kwargs, **fkwargs) 69 | if retval is not None: 70 | args, kwargs = retval 71 | 72 | if not (args or kwargs): 73 | return self.cluster.hosts.keys() 74 | 75 | try: 76 | db_nums = self._route(attr=attr, args=args, kwargs=kwargs, **fkwargs) 77 | except Exception as e: 78 | self._handle_exception(e) 79 | db_nums = [] 80 | 81 | return self._post_routing(attr=attr, db_nums=db_nums, args=args, kwargs=kwargs, **fkwargs) 82 | 83 | # Backwards compatibilty 84 | get_db = get_dbs 85 | 86 | @routing_params 87 | def setup_router(self, args, kwargs, **fkwargs): 88 | """ 89 | Call method to perform any setup 90 | """ 91 | self._ready = self._setup_router(args=args, kwargs=kwargs, **fkwargs) 92 | 93 | return self._ready 94 | 95 | @routing_params 96 | def _setup_router(self, args, kwargs, **fkwargs): 97 | """ 98 | Perform any initialization for the router 99 | Returns False if setup could not be completed 100 | """ 101 | return True 102 | 103 | @routing_params 104 | def _pre_routing(self, attr, args, kwargs, **fkwargs): 105 | """ 106 | Perform any prerouting with this method and return the key 107 | """ 108 | return args, kwargs 109 | 110 | @routing_params 111 | def _route(self, attr, args, kwargs, **fkwargs): 112 | """ 113 | Perform routing and return db_nums 114 | """ 115 | return self.cluster.hosts.keys() 116 | 117 | @routing_params 118 | def _post_routing(self, attr, db_nums, args, kwargs, **fkwargs): 119 | """ 120 | Perform any postrouting actions and return db_nums 121 | """ 122 | return db_nums 123 | 124 | def _handle_exception(self, e): 125 | """ 126 | Handle/transform exceptions and return it 127 | """ 128 | raise e 129 | 130 | 131 | class RoundRobinRouter(BaseRouter): 132 | """ 133 | Basic retry router that performs round robin 134 | """ 135 | 136 | # Raised if all hosts in the hash have been marked as down 137 | class HostListExhausted(Exception): 138 | pass 139 | 140 | class InvalidDBNum(Exception): 141 | pass 142 | 143 | # If this router can be retried on if a particular db index it gave out did 144 | # not work 145 | retryable = True 146 | 147 | # How many requests to serve in a situation when a host is down before 148 | # the down hosts are assesed for readmittance back into the pool of serving 149 | # requests. 150 | # 151 | # If the attempt_reconnect_threshold is hit, it does not guarantee that the 152 | # down hosts will be put back - only that the router will CHECK to see if 153 | # the hosts CAN be put back. The elegibility of a host being put back is 154 | # handlede in the check_down_connections method, which by default will 155 | # readmit a host if it was marked down more than retry_timeout seconds ago. 156 | attempt_reconnect_threshold = 100000 157 | 158 | # Number of seconds a host must be marked down before it is elligable to be 159 | # put back in the pool and retried. 160 | retry_timeout = 30 161 | 162 | def __init__(self, *args, **kwargs): 163 | self._get_db_attempts = 0 164 | self._down_connections = {} 165 | 166 | super(RoundRobinRouter, self).__init__(*args, **kwargs) 167 | 168 | @classmethod 169 | def ensure_db_num(cls, db_num): 170 | try: 171 | return int(db_num) 172 | except ValueError: 173 | raise cls.InvalidDBNum() 174 | 175 | def check_down_connections(self): 176 | """ 177 | Iterates through all connections which were previously listed as unavailable 178 | and marks any that have expired their retry_timeout as being up. 179 | """ 180 | now = time.time() 181 | 182 | for db_num, marked_down_at in self._down_connections.items(): 183 | if marked_down_at + self.retry_timeout <= now: 184 | self.mark_connection_up(db_num) 185 | 186 | def flush_down_connections(self): 187 | """ 188 | Marks all connections which were previously listed as unavailable as being up. 189 | """ 190 | self._get_db_attempts = 0 191 | for db_num in self._down_connections.keys(): 192 | self.mark_connection_up(db_num) 193 | 194 | def mark_connection_down(self, db_num): 195 | db_num = self.ensure_db_num(db_num) 196 | self._down_connections[db_num] = time.time() 197 | 198 | def mark_connection_up(self, db_num): 199 | db_num = self.ensure_db_num(db_num) 200 | self._down_connections.pop(db_num, None) 201 | 202 | @routing_params 203 | def _setup_router(self, args, kwargs, **fkwargs): 204 | self._hosts_cycler = cycle(self.cluster.hosts.keys()) 205 | 206 | return True 207 | 208 | @routing_params 209 | def _pre_routing(self, attr, args, kwargs, retry_for=None, **fkwargs): 210 | self._get_db_attempts += 1 211 | 212 | if self._get_db_attempts > self.attempt_reconnect_threshold: 213 | self.check_down_connections() 214 | 215 | if retry_for is not None: 216 | self.mark_connection_down(retry_for) 217 | 218 | return args, kwargs 219 | 220 | @routing_params 221 | def _route(self, attr, args, kwargs, **fkwargs): 222 | now = time.time() 223 | 224 | for i in xrange(len(self.cluster)): 225 | db_num = self._hosts_cycler.next() 226 | 227 | marked_down_at = self._down_connections.get(db_num, False) 228 | 229 | if not marked_down_at or (marked_down_at + self.retry_timeout <= now): 230 | return [db_num] 231 | else: 232 | raise self.HostListExhausted() 233 | 234 | @routing_params 235 | def _post_routing(self, attr, db_nums, args, kwargs, **fkwargs): 236 | if db_nums and db_nums[0] in self._down_connections: 237 | self.mark_connection_up(db_nums[0]) 238 | 239 | return db_nums 240 | -------------------------------------------------------------------------------- /nydus/db/routers/keyvalue.py: -------------------------------------------------------------------------------- 1 | """ 2 | nydus.db.routers.keyvalue 3 | ~~~~~~~~~~~~~~~~~~~~~~~~~ 4 | 5 | :copyright: (c) 2011-2012 DISQUS. 6 | :license: Apache License 2.0, see LICENSE for more details. 7 | """ 8 | 9 | from binascii import crc32 10 | 11 | from nydus.contrib.ketama import Ketama 12 | from nydus.db.routers import BaseRouter, RoundRobinRouter, routing_params 13 | 14 | __all__ = ('ConsistentHashingRouter', 'PartitionRouter') 15 | 16 | 17 | def get_key(args, kwargs): 18 | if 'key' in kwargs: 19 | return kwargs['key'] 20 | elif args: 21 | return args[0] 22 | return None 23 | 24 | 25 | class ConsistentHashingRouter(RoundRobinRouter): 26 | """ 27 | Router that returns host number based on a consistent hashing algorithm. 28 | The consistent hashing algorithm only works if a key argument is provided. 29 | 30 | If a key is not provided, then all hosts are returned. 31 | 32 | The first argument is assumed to be the ``key`` for routing. Keyword arguments 33 | are not supported. 34 | """ 35 | 36 | def __init__(self, *args, **kwargs): 37 | self._db_num_id_map = {} 38 | super(ConsistentHashingRouter, self).__init__(*args, **kwargs) 39 | 40 | def mark_connection_down(self, db_num): 41 | db_num = self.ensure_db_num(db_num) 42 | self._hash.remove_node(self._db_num_id_map[db_num]) 43 | 44 | super(ConsistentHashingRouter, self).mark_connection_down(db_num) 45 | 46 | def mark_connection_up(self, db_num): 47 | db_num = self.ensure_db_num(db_num) 48 | self._hash.add_node(self._db_num_id_map[db_num]) 49 | 50 | super(ConsistentHashingRouter, self).mark_connection_up(db_num) 51 | 52 | @routing_params 53 | def _setup_router(self, args, kwargs, **fkwargs): 54 | self._db_num_id_map = dict([(db_num, host.identifier) for db_num, host in self.cluster.hosts.iteritems()]) 55 | self._hash = Ketama(self._db_num_id_map.values()) 56 | 57 | return True 58 | 59 | @routing_params 60 | def _pre_routing(self, *args, **kwargs): 61 | self.check_down_connections() 62 | 63 | return super(ConsistentHashingRouter, self)._pre_routing(*args, **kwargs) 64 | 65 | @routing_params 66 | def _route(self, attr, args, kwargs, **fkwargs): 67 | """ 68 | The first argument is assumed to be the ``key`` for routing. 69 | """ 70 | 71 | key = get_key(args, kwargs) 72 | 73 | found = self._hash.get_node(key) 74 | 75 | if not found and len(self._down_connections) > 0: 76 | raise self.HostListExhausted() 77 | 78 | return [i for i, h in self.cluster.hosts.iteritems() 79 | if h.identifier == found] 80 | 81 | 82 | class PartitionRouter(BaseRouter): 83 | @routing_params 84 | def _route(self, attr, args, kwargs, **fkwargs): 85 | """ 86 | The first argument is assumed to be the ``key`` for routing. 87 | """ 88 | key = get_key(args, kwargs) 89 | 90 | return [crc32(str(key)) % len(self.cluster)] 91 | -------------------------------------------------------------------------------- /nydus/db/routers/redis.py: -------------------------------------------------------------------------------- 1 | """ 2 | nydus.db.routers.redis 3 | ~~~~~~~~~~~~~~~~~~~~~~ 4 | 5 | :copyright: (c) 2011-2012 DISQUS. 6 | :license: Apache License 2.0, see LICENSE for more details. 7 | """ 8 | 9 | from nydus.db.routers import RoundRobinRouter 10 | from nydus.db.routers.keyvalue import ConsistentHashingRouter, PartitionRouter 11 | 12 | __all__ = ('ConsistentHashingRouter', 'PartitionRouter', 'RoundRobinRouter') 13 | -------------------------------------------------------------------------------- /nydus/testutils.py: -------------------------------------------------------------------------------- 1 | import unittest2 2 | 3 | NOTSET = object() 4 | 5 | 6 | class BaseTest(unittest2.TestCase): 7 | def setUp(self): 8 | pass 9 | 10 | 11 | class fixture(object): 12 | """ 13 | >>> class Foo(object): 14 | >>> @fixture 15 | >>> def foo(self): 16 | >>> # calculate something important here 17 | >>> return 42 18 | """ 19 | def __init__(self, func): 20 | self.__name__ = func.__name__ 21 | self.__module__ = func.__module__ 22 | self.__doc__ = func.__doc__ 23 | self.func = func 24 | 25 | def __get__(self, obj, type=None): 26 | if obj is None: 27 | return self 28 | value = obj.__dict__.get(self.__name__, NOTSET) 29 | if value is NOTSET: 30 | value = self.func(obj) 31 | obj.__dict__[self.__name__] = value 32 | return value 33 | -------------------------------------------------------------------------------- /nydus/utils.py: -------------------------------------------------------------------------------- 1 | from collections import defaultdict 2 | from Queue import Queue, Empty 3 | from threading import Thread 4 | 5 | 6 | # import_string comes form Werkzeug 7 | # http://werkzeug.pocoo.org 8 | def import_string(import_name, silent=False): 9 | """Imports an object based on a string. If *silent* is True the return 10 | value will be None if the import fails. 11 | 12 | Simplified version of the function with same name from `Werkzeug`_. 13 | 14 | :param import_name: 15 | The dotted name for the object to import. 16 | :param silent: 17 | If True, import errors are ignored and None is returned instead. 18 | :returns: 19 | The imported object. 20 | """ 21 | import_name = str(import_name) 22 | try: 23 | if '.' in import_name: 24 | module, obj = import_name.rsplit('.', 1) 25 | return getattr(__import__(module, None, None, [obj]), obj) 26 | else: 27 | return __import__(import_name) 28 | except (ImportError, AttributeError): 29 | if not silent: 30 | raise 31 | 32 | 33 | def apply_defaults(host, defaults): 34 | for key, value in defaults.iteritems(): 35 | if key not in host: 36 | host[key] = value 37 | return host 38 | 39 | 40 | def peek(value): 41 | generator = iter(value) 42 | prev = generator.next() 43 | for item in generator: 44 | yield prev, item 45 | prev = item 46 | yield prev, None 47 | 48 | 49 | class Worker(Thread): 50 | def __init__(self, queue): 51 | Thread.__init__(self) 52 | self.queue = queue 53 | self.results = defaultdict(list) 54 | 55 | def run(self): 56 | while True: 57 | try: 58 | ident, func, args, kwargs = self.queue.get_nowait() 59 | except Empty: 60 | break 61 | 62 | try: 63 | result = func(*args, **kwargs) 64 | self.results[ident].append(result) 65 | except Exception as e: 66 | self.results[ident].append(e) 67 | finally: 68 | self.queue.task_done() 69 | 70 | return self.results 71 | 72 | 73 | class ThreadPool(object): 74 | def __init__(self, workers=10): 75 | self.queue = Queue() 76 | self.workers = [] 77 | self.tasks = [] 78 | for worker in xrange(workers): 79 | self.workers.append(Worker(self.queue)) 80 | 81 | def add(self, ident, func, args=None, kwargs=None): 82 | if args is None: 83 | args = () 84 | if kwargs is None: 85 | kwargs = {} 86 | task = (ident, func, args, kwargs) 87 | self.tasks.append(ident) 88 | self.queue.put_nowait(task) 89 | 90 | def join(self): 91 | for worker in self.workers: 92 | worker.start() 93 | 94 | results = defaultdict(list) 95 | for worker in self.workers: 96 | worker.join() 97 | for k, v in worker.results.iteritems(): 98 | results[k].extend(v) 99 | return results 100 | -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [nosetests] 2 | where=tests 3 | 4 | [pytest] 5 | python_files=test*.py 6 | 7 | [wheel] 8 | universal = 1 9 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | import sys 4 | from setuptools import setup, find_packages 5 | 6 | try: 7 | __import__('multiprocessing') 8 | except: 9 | pass 10 | 11 | if 'nosetests' in sys.argv: 12 | setup_requires = ['nose'] 13 | else: 14 | setup_requires = [] 15 | 16 | tests_require = [ 17 | 'mock', 18 | 'nose', 19 | 'pycassa', 20 | 'pylibmc', 21 | 'redis', 22 | 'riak', 23 | 'thoonk', 24 | 'unittest2', 25 | ] 26 | 27 | dependency_links = [ 28 | 'https://github.com/andyet/thoonk.py/tarball/master#egg=thoonk', 29 | ] 30 | 31 | 32 | install_requires = [ 33 | ] 34 | 35 | setup( 36 | name='nydus', 37 | version='0.11.0', 38 | author='DISQUS', 39 | author_email='opensource@disqus.com', 40 | url='https://github.com/disqus/nydus', 41 | description='Connection utilities', 42 | packages=find_packages(exclude=('tests',)), 43 | zip_safe=False, 44 | setup_requires=setup_requires, 45 | install_requires=install_requires, 46 | dependency_links=dependency_links, 47 | tests_require=tests_require, 48 | extras_require={'test': tests_require}, 49 | test_suite='nose.collector', 50 | include_package_data=True, 51 | license='Apache License 2.0', 52 | classifiers=[ 53 | 'Intended Audience :: Developers', 54 | 'Intended Audience :: System Administrators', 55 | 'Operating System :: OS Independent', 56 | 'Topic :: Software Development' 57 | ], 58 | ) 59 | -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/disqus/nydus/9b505840da47a34f758a830c3992fa5dcb7bb7ad/tests/__init__.py -------------------------------------------------------------------------------- /tests/nydus/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/disqus/nydus/9b505840da47a34f758a830c3992fa5dcb7bb7ad/tests/nydus/__init__.py -------------------------------------------------------------------------------- /tests/nydus/db/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/disqus/nydus/9b505840da47a34f758a830c3992fa5dcb7bb7ad/tests/nydus/db/__init__.py -------------------------------------------------------------------------------- /tests/nydus/db/backends/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/disqus/nydus/9b505840da47a34f758a830c3992fa5dcb7bb7ad/tests/nydus/db/backends/__init__.py -------------------------------------------------------------------------------- /tests/nydus/db/backends/memcache/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/disqus/nydus/9b505840da47a34f758a830c3992fa5dcb7bb7ad/tests/nydus/db/backends/memcache/__init__.py -------------------------------------------------------------------------------- /tests/nydus/db/backends/memcache/tests.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | 3 | from nydus.db import create_cluster 4 | from nydus.db.base import BaseCluster 5 | from nydus.db.backends.memcache import Memcache, regroup_commands, grouped_args_for_command, \ 6 | can_group_commands 7 | from nydus.db.promise import EventualCommand 8 | from nydus.testutils import BaseTest, fixture 9 | 10 | import mock 11 | import pylibmc 12 | 13 | 14 | class CanGroupCommandsTest(BaseTest): 15 | def test_groupable_set_commands(self): 16 | command = EventualCommand('set', ['foo', 1]) 17 | other = EventualCommand('set', ['bar', 2]) 18 | self.assertEquals(can_group_commands(command, other), True) 19 | 20 | def test_ungroupable_set_commands(self): 21 | command = EventualCommand('set', ['foo', 1], {'timeout': 1}) 22 | other = EventualCommand('set', ['bar', 2], {'timeout': 2}) 23 | self.assertEquals(can_group_commands(command, other), False) 24 | 25 | def test_groupable_get_commands(self): 26 | command = EventualCommand('get', ['foo']) 27 | other = EventualCommand('get', ['bar']) 28 | self.assertEquals(can_group_commands(command, other), True) 29 | 30 | def test_ungroupable_get_commands(self): 31 | command = EventualCommand('get', ['foo'], {'timeout': 1}) 32 | other = EventualCommand('get', ['bar'], {'timeout': 2}) 33 | self.assertEquals(can_group_commands(command, other), False) 34 | 35 | def test_groupable_delete_commands(self): 36 | command = EventualCommand('delete', ['foo']) 37 | other = EventualCommand('delete', ['bar']) 38 | self.assertEquals(can_group_commands(command, other), True) 39 | 40 | def test_ungroupable_delete_commands(self): 41 | command = EventualCommand('delete', ['foo'], {'timeout': 1}) 42 | other = EventualCommand('delete', ['bar'], {'timeout': 2}) 43 | self.assertEquals(can_group_commands(command, other), False) 44 | 45 | 46 | class GroupedArgsForCommandTest(BaseTest): 47 | def test_set_excludes_first_two_args(self): 48 | command = EventualCommand('set', ['foo', 1, 'biz']) 49 | result = grouped_args_for_command(command) 50 | self.assertEquals(result, ['biz']) 51 | 52 | def test_get_excludes_first_arg(self): 53 | command = EventualCommand('get', ['foo', 1]) 54 | result = grouped_args_for_command(command) 55 | self.assertEquals(result, [1]) 56 | 57 | def test_delete_excludes_first_arg(self): 58 | command = EventualCommand('delete', ['foo', 1]) 59 | result = grouped_args_for_command(command) 60 | self.assertEquals(result, [1]) 61 | 62 | 63 | class RegroupCommandsTest(BaseTest): 64 | def get_grouped_results(self, commands, num_expected): 65 | grouped = regroup_commands(commands) 66 | self.assertEquals(len(grouped), num_expected, grouped) 67 | return grouped 68 | 69 | def test_set_basic(self): 70 | commands = [ 71 | EventualCommand('set', ['foo', 1], {'timeout': 1}), 72 | EventualCommand('set', ['bar', 2], {'timeout': 1}), 73 | EventualCommand('set', ['baz', 3], {'timeout': 2}), 74 | ] 75 | 76 | items = self.get_grouped_results(commands, 2) 77 | 78 | new_command, grouped_commands = items[0] 79 | self.assertEquals(len(grouped_commands), 2) 80 | self.assertEquals(grouped_commands, commands[0:2]) 81 | self.assertEquals(new_command.get_name(), 'set_multi') 82 | self.assertEquals(new_command.get_args(), ({ 83 | 'foo': 1, 84 | 'bar': 2, 85 | },)) 86 | self.assertEquals(new_command.get_kwargs(), { 87 | 'timeout': 1, 88 | }) 89 | 90 | new_command, grouped_commands = items[1] 91 | self.assertEquals(len(grouped_commands), 1) 92 | self.assertEquals(grouped_commands, commands[2:3]) 93 | self.assertEquals(new_command.get_name(), 'set') 94 | self.assertEquals(new_command.get_args(), ['baz', 3]) 95 | self.assertEquals(new_command.get_kwargs(), { 96 | 'timeout': 2, 97 | }) 98 | 99 | def test_get_basic(self): 100 | commands = [ 101 | EventualCommand('get', ['foo'], {'key_prefix': 1}), 102 | EventualCommand('get', ['bar'], {'key_prefix': 1}), 103 | EventualCommand('get', ['baz'], {'key_prefix': 2}), 104 | ] 105 | items = self.get_grouped_results(commands, 2) 106 | 107 | new_command, grouped_commands = items[0] 108 | self.assertEquals(len(grouped_commands), 2) 109 | self.assertEquals(grouped_commands, commands[0:2]) 110 | self.assertEquals(new_command.get_name(), 'get_multi') 111 | self.assertEquals(new_command.get_args(), (['foo', 'bar'],)) 112 | self.assertEquals(new_command.get_kwargs(), { 113 | 'key_prefix': 1, 114 | }) 115 | 116 | new_command, grouped_commands = items[1] 117 | self.assertEquals(len(grouped_commands), 1) 118 | self.assertEquals(grouped_commands, commands[2:3]) 119 | self.assertEquals(new_command.get_name(), 'get') 120 | self.assertEquals(new_command.get_args(), ['baz']) 121 | self.assertEquals(new_command.get_kwargs(), { 122 | 'key_prefix': 2, 123 | }) 124 | 125 | def test_delete_basic(self): 126 | commands = [ 127 | EventualCommand('delete', ['foo'], {'key_prefix': 1}), 128 | EventualCommand('delete', ['bar'], {'key_prefix': 1}), 129 | EventualCommand('delete', ['baz'], {'key_prefix': 2}), 130 | ] 131 | items = self.get_grouped_results(commands, 2) 132 | 133 | new_command, grouped_commands = items[0] 134 | self.assertEquals(len(grouped_commands), 2) 135 | self.assertEquals(grouped_commands, commands[0:2]) 136 | self.assertEquals(new_command.get_name(), 'delete_multi') 137 | self.assertEquals(new_command.get_args(), (['foo', 'bar'],)) 138 | self.assertEquals(new_command.get_kwargs(), { 139 | 'key_prefix': 1, 140 | }) 141 | 142 | new_command, grouped_commands = items[1] 143 | self.assertEquals(len(grouped_commands), 1) 144 | self.assertEquals(grouped_commands, commands[2:3]) 145 | self.assertEquals(new_command.get_name(), 'delete') 146 | self.assertEquals(new_command.get_args(), ['baz']) 147 | self.assertEquals(new_command.get_kwargs(), { 148 | 'key_prefix': 2, 149 | }) 150 | 151 | def test_mixed_commands(self): 152 | commands = [ 153 | EventualCommand('get', ['foo']), 154 | EventualCommand('set', ['bar', 1], {'timeout': 1}), 155 | EventualCommand('set', ['baz', 2], {'timeout': 1}), 156 | EventualCommand('get', ['bar']), 157 | ] 158 | 159 | items = self.get_grouped_results(commands, 3) 160 | 161 | new_command, grouped_commands = items[0] 162 | self.assertEquals(len(grouped_commands), 1) 163 | self.assertEquals(grouped_commands, commands[2:3]) 164 | self.assertEquals(new_command.get_name(), 'get') 165 | self.assertEquals(new_command.get_args(), ['foo']) 166 | self.assertEquals(new_command.get_kwargs(), {}) 167 | 168 | new_command, grouped_commands = items[1] 169 | self.assertEquals(len(grouped_commands), 2) 170 | self.assertEquals(grouped_commands, commands[0:2]) 171 | self.assertEquals(new_command.get_name(), 'set_multi') 172 | self.assertEquals(new_command.get_args(), ({ 173 | 'bar': 1, 174 | 'baz': 2, 175 | },)) 176 | self.assertEquals(new_command.get_kwargs(), { 177 | 'timeout': 1, 178 | }) 179 | 180 | new_command, grouped_commands = items[2] 181 | self.assertEquals(len(grouped_commands), 1) 182 | self.assertEquals(grouped_commands, commands[2:3]) 183 | self.assertEquals(new_command.get_name(), 'get') 184 | self.assertEquals(new_command.get_args(), ['bar']) 185 | self.assertEquals(new_command.get_kwargs(), {}) 186 | 187 | 188 | class MemcacheTest(BaseTest): 189 | 190 | @fixture 191 | def memcache(self): 192 | return Memcache(num=0) 193 | 194 | def test_provides_retryable_exceptions(self): 195 | self.assertEquals(Memcache.retryable_exceptions, frozenset([pylibmc.Error])) 196 | 197 | def test_provides_identifier(self): 198 | self.assertEquals(self.memcache.identifier, str(self.memcache.identifier)) 199 | 200 | @mock.patch('pylibmc.Client') 201 | def test_client_instantiates_with_kwargs(self, Client): 202 | client = Memcache(num=0) 203 | client.connect() 204 | 205 | self.assertEquals(Client.call_count, 1) 206 | Client.assert_any_call(['localhost:11211'], binary=True, behaviors=None) 207 | 208 | @mock.patch('pylibmc.Client.get') 209 | def test_with_cluster(self, get): 210 | p = BaseCluster( 211 | backend=Memcache, 212 | hosts={0: {}}, 213 | ) 214 | result = p.get('MemcacheTest_with_cluster') 215 | get.assert_called_once_with('MemcacheTest_with_cluster') 216 | self.assertEquals(result, get.return_value) 217 | 218 | @mock.patch('pylibmc.Client') 219 | def test_pipeline_behavior(self, Client): 220 | cluster = create_cluster({ 221 | 'engine': 'nydus.db.backends.memcache.Memcache', 222 | 'hosts': { 223 | 0: {'binary': True}, 224 | } 225 | }) 226 | 227 | with cluster.map() as conn: 228 | conn.set('a', 1) 229 | conn.set('b', 2) 230 | conn.set('c', 3) 231 | conn.get('a') 232 | conn.get('b') 233 | conn.get('c') 234 | 235 | Client.return_value.set_multi.assert_any_call({'a': 1, 'b': 2, 'c': 3}) 236 | Client.return_value.get_multi.assert_any_call(['a', 'b', 'c']) 237 | 238 | def test_pipeline_integration(self): 239 | cluster = create_cluster({ 240 | 'engine': 'nydus.db.backends.memcache.Memcache', 241 | 'hosts': { 242 | 0: {'binary': True}, 243 | } 244 | }) 245 | 246 | with cluster.map() as conn: 247 | conn.set('a', 1) 248 | conn.set('b', 2) 249 | conn.set('c', 3) 250 | conn.get('a') 251 | conn.get('b') 252 | conn.get('c') 253 | 254 | results = conn.get_results() 255 | self.assertEquals(len(results), 6, results) 256 | self.assertEquals(results[0:3], [None, None, None]) 257 | self.assertEquals(results[3:6], [1, 2, 3]) 258 | -------------------------------------------------------------------------------- /tests/nydus/db/backends/pycassa/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/disqus/nydus/9b505840da47a34f758a830c3992fa5dcb7bb7ad/tests/nydus/db/backends/pycassa/__init__.py -------------------------------------------------------------------------------- /tests/nydus/db/backends/pycassa/tests.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | 3 | from nydus.db import create_cluster 4 | from nydus.db.backends.pycassa import Pycassa, PycassaCluster 5 | from nydus.testutils import BaseTest, fixture 6 | import mock 7 | 8 | 9 | class PycassCreateClusterTest(BaseTest): 10 | @fixture 11 | def cluster(self): 12 | return create_cluster({ 13 | 'backend': 'nydus.db.backends.pycassa.Pycassa', 14 | 'hosts': ['localhost'], 15 | 'keyspace': 'test', 16 | }) 17 | 18 | def test_is_pycassa_cluster(self): 19 | self.assertEquals(type(self.cluster), PycassaCluster) 20 | 21 | 22 | class PycassClusterTest(BaseTest): 23 | @fixture 24 | def cluster(self): 25 | return PycassaCluster( 26 | hosts=['localhost'], 27 | keyspace='test', 28 | ) 29 | 30 | def test_has_one_connection(self): 31 | self.assertEquals(len(self.cluster), 1) 32 | 33 | def test_backend_is_pycassa(self): 34 | self.assertEquals(type(self.cluster[0]), Pycassa) 35 | 36 | 37 | class PycassaTest(BaseTest): 38 | @fixture 39 | def connection(self): 40 | return Pycassa(num=0, keyspace='test', hosts=['localhost']) 41 | 42 | @mock.patch('nydus.db.backends.pycassa.ConnectionPool') 43 | def test_client_instantiates_with_kwargs(self, ConnectionPool): 44 | client = Pycassa( 45 | keyspace='test', hosts=['localhost'], prefill=True, 46 | timeout=5, foo='bar', num=0, 47 | ) 48 | client.connect() 49 | ConnectionPool.assert_called_once_with( 50 | keyspace='test', prefill=True, timeout=5, 51 | server_list=['localhost'], foo='bar' 52 | ) 53 | 54 | @mock.patch('nydus.db.backends.pycassa.ConnectionPool') 55 | def test_disconnect_calls_dispose(self, ConnectionPool): 56 | self.connection.disconnect() 57 | ConnectionPool().dispose.assert_called_once_with() 58 | -------------------------------------------------------------------------------- /tests/nydus/db/backends/redis/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/disqus/nydus/9b505840da47a34f758a830c3992fa5dcb7bb7ad/tests/nydus/db/backends/redis/__init__.py -------------------------------------------------------------------------------- /tests/nydus/db/backends/redis/tests.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | 3 | from nydus.db import create_cluster 4 | from nydus.db.base import BaseCluster 5 | from nydus.db.backends.redis import Redis 6 | from nydus.db.promise import EventualCommand 7 | from nydus.testutils import BaseTest, fixture 8 | import mock 9 | import redis as redis_ 10 | 11 | 12 | class RedisPipelineTest(BaseTest): 13 | @fixture 14 | def cluster(self): 15 | return create_cluster({ 16 | 'backend': 'nydus.db.backends.redis.Redis', 17 | 'router': 'nydus.db.routers.keyvalue.PartitionRouter', 18 | 'hosts': { 19 | 0: {'db': 5}, 20 | 1: {'db': 6}, 21 | 2: {'db': 7}, 22 | 3: {'db': 8}, 23 | 4: {'db': 9}, 24 | } 25 | }) 26 | 27 | # XXX: technically we're testing the Nydus map code, and not ours 28 | def test_pipelined_map(self): 29 | chars = ('a', 'b', 'c', 'd', 'e', 'f') 30 | with self.cluster.map() as conn: 31 | [conn.set(c, i) for i, c in enumerate(chars)] 32 | res = [conn.get(c) for c in chars] 33 | self.assertEqual(range(len(chars)), [int(r) for r in res]) 34 | 35 | def test_map_single_connection(self): 36 | with self.cluster.map() as conn: 37 | conn.set('a', '1') 38 | self.assertEquals(self.cluster.get('a'), '1') 39 | 40 | def test_no_proxy_without_call_on_map(self): 41 | with self.cluster.map() as conn: 42 | result = conn.incr 43 | 44 | assert type(result) is EventualCommand 45 | assert not result.was_called() 46 | 47 | 48 | class RedisTest(BaseTest): 49 | 50 | def setUp(self): 51 | self.redis = Redis(num=0, db=1) 52 | self.redis.flushdb() 53 | 54 | def test_proxy(self): 55 | self.assertEquals(self.redis.incr('RedisTest_proxy'), 1) 56 | 57 | def test_with_cluster(self): 58 | p = BaseCluster( 59 | backend=Redis, 60 | hosts={0: {'db': 1}}, 61 | ) 62 | self.assertEquals(p.incr('RedisTest_with_cluster'), 1) 63 | 64 | def test_provides_retryable_exceptions(self): 65 | self.assertEquals(Redis.retryable_exceptions, frozenset([redis_.ConnectionError, redis_.InvalidResponse])) 66 | 67 | def test_provides_identifier(self): 68 | self.assertEquals(self.redis.identifier, str(self.redis.identifier)) 69 | 70 | @mock.patch('nydus.db.backends.redis.StrictRedis') 71 | def test_client_instantiates_with_kwargs(self, RedisClient): 72 | client = Redis(num=0) 73 | client.connect() 74 | 75 | self.assertEquals(RedisClient.call_count, 1) 76 | RedisClient.assert_any_call(host='localhost', port=6379, db=0, socket_timeout=None, 77 | password=None, unix_socket_path=None) 78 | 79 | @mock.patch('nydus.db.backends.redis.StrictRedis') 80 | def test_map_does_pipeline(self, RedisClient): 81 | redis = create_cluster({ 82 | 'backend': 'nydus.db.backends.redis.Redis', 83 | 'router': 'nydus.db.routers.keyvalue.PartitionRouter', 84 | 'hosts': { 85 | 0: {'db': 0}, 86 | 1: {'db': 1}, 87 | } 88 | }) 89 | 90 | with redis.map() as conn: 91 | conn.set('a', 0) 92 | conn.set('d', 1) 93 | 94 | # ensure this was actually called through the pipeline 95 | self.assertFalse(RedisClient().set.called) 96 | 97 | self.assertEquals(RedisClient().pipeline.call_count, 2) 98 | RedisClient().pipeline.assert_called_with() 99 | 100 | self.assertEquals(RedisClient().pipeline().set.call_count, 2) 101 | RedisClient().pipeline().set.assert_any_call('a', 0) 102 | RedisClient().pipeline().set.assert_any_call('d', 1) 103 | 104 | self.assertEquals(RedisClient().pipeline().execute.call_count, 2) 105 | RedisClient().pipeline().execute.assert_called_with() 106 | 107 | @mock.patch('nydus.db.backends.redis.StrictRedis') 108 | def test_map_only_runs_on_required_nodes(self, RedisClient): 109 | redis = create_cluster({ 110 | 'engine': 'nydus.db.backends.redis.Redis', 111 | 'router': 'nydus.db.routers.keyvalue.PartitionRouter', 112 | 'hosts': { 113 | 0: {'db': 0}, 114 | 1: {'db': 1}, 115 | } 116 | }) 117 | with redis.map() as conn: 118 | conn.set('a', 0) 119 | conn.set('b', 1) 120 | 121 | # ensure this was actually called through the pipeline 122 | self.assertFalse(RedisClient().set.called) 123 | 124 | self.assertEquals(RedisClient().pipeline.call_count, 1) 125 | RedisClient().pipeline.assert_called_with() 126 | 127 | self.assertEquals(RedisClient().pipeline().set.call_count, 2) 128 | RedisClient().pipeline().set.assert_any_call('a', 0) 129 | RedisClient().pipeline().set.assert_any_call('b', 1) 130 | 131 | self.assertEquals(RedisClient().pipeline().execute.call_count, 1) 132 | RedisClient().pipeline().execute.assert_called_with() 133 | 134 | def test_normal_exceptions_dont_break_the_cluster(self): 135 | redis = create_cluster({ 136 | 'engine': 'nydus.db.backends.redis.Redis', 137 | 'router': 'nydus.db.routers.keyvalue.ConsistentHashingRouter', 138 | 'hosts': { 139 | 0: {'db': 0}, 140 | 1: {'db': 1}, 141 | } 142 | }) 143 | 144 | # Create a normal key 145 | redis.set('a', 0) 146 | 147 | with self.assertRaises(redis_.ResponseError): 148 | # We are going to preform an operation on a key that is not a set 149 | # This call *should* raise the actual Redis exception, and 150 | # not continue on to think the host is down. 151 | redis.scard('a') 152 | 153 | # This shouldn't raise a HostListExhausted exception 154 | redis.get('a') 155 | 156 | def test_custom_identifier_specified(self): 157 | cluster_config = { 158 | 'backend': 'nydus.db.backends.redis.Redis', 159 | 'hosts': { 160 | 0: {'db': 0, 'identifier': 'redis://127.0.0.1:6379/0'}, 161 | 1: {'db': 1, 'identifier': 'redis://127.0.0.1:6380/1'}, 162 | }, 163 | } 164 | 165 | redis = create_cluster(cluster_config) 166 | for idx in cluster_config['hosts'].keys(): 167 | self.assertEquals(redis.hosts[idx].identifier, 168 | cluster_config['hosts'][idx]['identifier']) 169 | -------------------------------------------------------------------------------- /tests/nydus/db/backends/riak/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/disqus/nydus/9b505840da47a34f758a830c3992fa5dcb7bb7ad/tests/nydus/db/backends/riak/__init__.py -------------------------------------------------------------------------------- /tests/nydus/db/backends/riak/tests.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | 3 | import mock 4 | from httplib import HTTPException 5 | from nydus.db.backends.riak import Riak 6 | from nydus.testutils import BaseTest 7 | from riak import RiakClient, RiakError 8 | from socket import error as SocketError 9 | 10 | 11 | class RiakTest(BaseTest): 12 | def setUp(self): 13 | self.expected_defaults = { 14 | 'host': '127.0.0.1', 15 | 'port': 8098, 16 | 'prefix': 'riak', 17 | 'mapred_prefix': 'mapred', 18 | 'client_id': None, 19 | } 20 | 21 | self.modified_props = { 22 | 'host': '127.0.0.254', 23 | 'port': 8908, 24 | 'prefix': 'kair', 25 | 'mapred_prefix': 'derpam', 26 | 'client_id': 'MjgxMDg2MzQx', 27 | 'transport_options': {}, 28 | 'transport_class': mock.Mock, 29 | 'solr_transport_class': mock.Mock, 30 | } 31 | 32 | self.conn = Riak(0) 33 | self.modified_conn = Riak(1, **self.modified_props) 34 | 35 | def test_init_defaults(self): 36 | self.assertDictContainsSubset(self.expected_defaults, self.conn.__dict__) 37 | 38 | def test_init_properties(self): 39 | self.assertDictContainsSubset(self.modified_props, self.modified_conn.__dict__) 40 | 41 | def test_identifier(self): 42 | expected_identifier = 'http://%(host)s:%(port)s/%(prefix)s' % self.conn.__dict__ 43 | self.assertEquals(expected_identifier, self.conn.identifier) 44 | 45 | def test_identifier_properties(self): 46 | expected_identifier = 'http://%(host)s:%(port)s/%(prefix)s' % self.modified_props 47 | self.assertEquals(expected_identifier, self.modified_conn.identifier) 48 | 49 | @mock.patch('nydus.db.backends.riak.RiakClient') 50 | def test_connect_riakclient_options(self, _RiakClient): 51 | self.conn.connect() 52 | 53 | _RiakClient.assert_called_with(host=self.conn.host, port=self.conn.port, prefix=self.conn.prefix, \ 54 | mapred_prefix=self.conn.mapred_prefix, client_id=self.conn.client_id, \ 55 | transport_options=self.conn.transport_options, transport_class=self.conn.transport_class, \ 56 | solr_transport_class=self.conn.solr_transport_class) 57 | 58 | @mock.patch('nydus.db.backends.riak.RiakClient') 59 | def test_connect_riakclient_modified_options(self, _RiakClient): 60 | self.modified_conn.connect() 61 | 62 | _RiakClient.assert_called_with(host=self.modified_conn.host, port=self.modified_conn.port, prefix=self.modified_conn.prefix, \ 63 | mapred_prefix=self.modified_conn.mapred_prefix, client_id=self.modified_conn.client_id, \ 64 | transport_options=self.modified_conn.transport_options, transport_class=self.modified_conn.transport_class, \ 65 | solr_transport_class=self.modified_conn.solr_transport_class) 66 | 67 | def test_connect_returns_riakclient(self): 68 | client = self.conn.connect() 69 | 70 | self.assertIsInstance(client, RiakClient) 71 | 72 | def test_provides_retryable_exceptions(self): 73 | self.assertItemsEqual([RiakError, HTTPException, SocketError], self.conn.retryable_exceptions) 74 | -------------------------------------------------------------------------------- /tests/nydus/db/backends/thoonk/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/disqus/nydus/9b505840da47a34f758a830c3992fa5dcb7bb7ad/tests/nydus/db/backends/thoonk/__init__.py -------------------------------------------------------------------------------- /tests/nydus/db/backends/thoonk/tests.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | 3 | import unittest2 4 | from nydus.db.backends.thoonk import Thoonk 5 | from nydus.db import create_cluster 6 | 7 | 8 | class ThoonkTest(unittest2.TestCase): 9 | def get_cluster(self, router): 10 | cluster = create_cluster({ 11 | 'backend': 'nydus.db.backends.thoonk.Thoonk', 12 | 'router': router, 13 | 'hosts': { 14 | 0: {'db': 5}, 15 | 1: {'db': 6}, 16 | 2: {'db': 7}, 17 | 3: {'db': 8}, 18 | 4: {'db': 9}, 19 | } 20 | }) 21 | self.flush_custer(cluster) 22 | return cluster 23 | 24 | def flush_custer(self, cluster): 25 | for x in range(len(cluster)): 26 | c = cluster.get_conn()[x] 27 | c.redis.flushdb() 28 | 29 | def setUp(self): 30 | self.ps = Thoonk(0, db=1) 31 | self.redis = self.ps.redis 32 | self.redis.flushdb() 33 | 34 | def tearDown(self): 35 | pass 36 | 37 | def test_flush_db(self): 38 | pubsub = self.get_cluster('nydus.db.routers.keyvalue.ConsistentHashingRouter') 39 | pubsub.flushdb() 40 | 41 | def test_job_with_ConsistentHashingRouter(self): 42 | pubsub = self.get_cluster('nydus.db.routers.keyvalue.ConsistentHashingRouter') 43 | job = pubsub.job("test1") 44 | jid = job.put("10") 45 | 46 | jid_found = False 47 | 48 | for ps in pubsub.get_conn(): 49 | jps = ps.job('test1') 50 | if jid in jps.get_ids(): 51 | self.assertFalse(jid_found) 52 | jid_found = True 53 | 54 | self.assertTrue(jid_found) 55 | 56 | def test_job_with_RoundRobinRouter(self): 57 | pubsub = self.get_cluster('nydus.db.routers.RoundRobinRouter') 58 | 59 | jobs = {} 60 | size = 20 61 | 62 | # put jobs onto the queue 63 | for x in xrange(0, size): 64 | jps = pubsub.job('testjob') 65 | jid = jps.put(str(x)) 66 | if id(jps) not in jobs: 67 | jobs[id(jps)] = [] 68 | jobs[id(jps)].append(jid) 69 | 70 | # make sure that we are reusing the job items 71 | self.assertEqual(len(jobs), 5) 72 | for k, v in jobs.iteritems(): 73 | self.assertEqual(len(v), size / 5) 74 | 75 | # make sure we fishi 76 | for x in xrange(0, size): 77 | jps = pubsub.job('testjob') 78 | jid, job, cancel_count = jps.get() 79 | jps.finish(jid) 80 | 81 | self.assertEqual(len(jobs), 5) 82 | 83 | for x in range(len(pubsub)): 84 | ps = pubsub.get_conn('testjob') 85 | jps = ps.job('testjob') 86 | self.assertEqual(jps.get_ids(), []) 87 | -------------------------------------------------------------------------------- /tests/nydus/db/connections/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/disqus/nydus/9b505840da47a34f758a830c3992fa5dcb7bb7ad/tests/nydus/db/connections/__init__.py -------------------------------------------------------------------------------- /tests/nydus/db/connections/tests.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | 3 | import mock 4 | 5 | from nydus.db import create_cluster 6 | from nydus.db.backends.base import BaseConnection 7 | from nydus.db.base import BaseCluster, create_connection 8 | from nydus.db.exceptions import CommandError 9 | from nydus.db.routers.base import BaseRouter 10 | from nydus.db.routers.keyvalue import get_key 11 | from nydus.db.promise import EventualCommand 12 | from nydus.testutils import BaseTest, fixture 13 | from nydus.utils import apply_defaults 14 | 15 | 16 | class DummyConnection(BaseConnection): 17 | def __init__(self, num, resp='foo', **kwargs): 18 | self.resp = resp 19 | super(DummyConnection, self).__init__(num, **kwargs) 20 | 21 | def foo(self, *args, **kwargs): 22 | return self.resp 23 | 24 | 25 | class DummyErroringConnection(DummyConnection): 26 | def foo(self, *args, **kwargs): 27 | if self.resp == 'error': 28 | raise ValueError(self.resp) 29 | return self.resp 30 | 31 | 32 | class DummyRouter(BaseRouter): 33 | def get_dbs(self, attr, args, kwargs, **fkwargs): 34 | key = get_key(args, kwargs) 35 | if key == 'foo': 36 | return [1] 37 | return [0] 38 | 39 | 40 | class ConnectionTest(BaseTest): 41 | @fixture 42 | def connection(self): 43 | return BaseConnection(0) 44 | 45 | @mock.patch('nydus.db.backends.base.BaseConnection.disconnect') 46 | def test_close_calls_disconnect(self, disconnect): 47 | self.connection._connection = mock.Mock() 48 | self.connection.close() 49 | disconnect.assert_called_once_with() 50 | 51 | @mock.patch('nydus.db.backends.base.BaseConnection.disconnect', mock.Mock(return_value=None)) 52 | def test_close_unsets_connection(self): 53 | self.connection.close() 54 | self.assertEquals(self.connection._connection, None) 55 | 56 | @mock.patch('nydus.db.backends.base.BaseConnection.disconnect') 57 | def test_close_propagates_noops_if_not_connected(self, disconnect): 58 | self.connection.close() 59 | self.assertFalse(disconnect.called) 60 | 61 | @mock.patch('nydus.db.backends.base.BaseConnection.connect') 62 | def test_connection_forces_connect(self, connect): 63 | self.connection.connection 64 | connect.assert_called_once_with() 65 | 66 | @mock.patch('nydus.db.backends.base.BaseConnection.connect') 67 | def test_connection_doesnt_reconnect_with_existing_connection(self, connect): 68 | self.connection._connection = mock.Mock() 69 | self.connection.connection 70 | self.assertFalse(connect.called) 71 | 72 | @mock.patch('nydus.db.backends.base.BaseConnection.connect') 73 | def test_connection_returns_result_of_connect(self, connect): 74 | val = self.connection.connection 75 | self.assertEquals(val, connect.return_value) 76 | 77 | def test_attrs_proxy(self): 78 | conn = mock.Mock() 79 | self.connection._connection = conn 80 | val = self.connection.foo(biz='baz') 81 | conn.foo.assert_called_once_with(biz='baz') 82 | self.assertEquals(val, conn.foo.return_value) 83 | 84 | 85 | class CreateConnectionTest(BaseTest): 86 | def test_does_apply_defaults(self): 87 | conn = mock.Mock() 88 | create_connection(conn, 0, {'resp': 'bar'}, {'foo': 'baz'}) 89 | conn.assert_called_once_with(0, foo='baz', resp='bar') 90 | 91 | def test_handles_arg_list_with_defaults(self): 92 | conn = mock.Mock() 93 | create_connection(conn, 0, ['localhost'], {'foo': 'baz'}) 94 | conn.assert_called_once_with(0, 'localhost', foo='baz') 95 | 96 | 97 | class CreateClusterTest(BaseTest): 98 | def test_creates_cluster(self): 99 | c = create_cluster({ 100 | 'backend': DummyConnection, 101 | 'router': DummyRouter, 102 | 'hosts': { 103 | 0: {'resp': 'bar'}, 104 | } 105 | }) 106 | self.assertEquals(len(c), 1) 107 | 108 | @mock.patch('nydus.db.base.create_connection') 109 | def test_does_create_connection_with_defaults(self, create_connection): 110 | create_cluster({ 111 | 'backend': DummyConnection, 112 | 'defaults': {'foo': 'baz'}, 113 | 'hosts': { 114 | 0: {'resp': 'bar'}, 115 | } 116 | }) 117 | create_connection.assert_called_once_with(DummyConnection, 0, {'resp': 'bar'}, {'foo': 'baz'}) 118 | 119 | 120 | class ClusterTest(BaseTest): 121 | def test_len_returns_num_backends(self): 122 | p = BaseCluster( 123 | backend=BaseConnection, 124 | hosts={0: {}}, 125 | ) 126 | self.assertEquals(len(p), 1) 127 | 128 | def test_proxy(self): 129 | p = BaseCluster( 130 | backend=DummyConnection, 131 | hosts={0: {'resp': 'bar'}}, 132 | ) 133 | self.assertEquals(p.foo(), 'bar') 134 | 135 | def test_disconnect(self): 136 | c = mock.Mock() 137 | p = BaseCluster( 138 | backend=c, 139 | hosts={0: {'resp': 'bar'}}, 140 | ) 141 | p.disconnect() 142 | c.disconnect.assert_called_once() 143 | 144 | def test_with_split_router(self): 145 | p = BaseCluster( 146 | router=DummyRouter, 147 | backend=DummyConnection, 148 | hosts={ 149 | 0: {'resp': 'foo'}, 150 | 1: {'resp': 'bar'}, 151 | }, 152 | ) 153 | self.assertEquals(p.foo(), 'foo') 154 | self.assertEquals(p.foo('foo'), 'bar') 155 | 156 | def test_default_routing_with_multiple_hosts(self): 157 | p = BaseCluster( 158 | backend=DummyConnection, 159 | hosts={ 160 | 0: {'resp': 'foo'}, 161 | 1: {'resp': 'bar'}, 162 | }, 163 | ) 164 | self.assertEquals(p.foo(), ['foo', 'bar']) 165 | self.assertEquals(p.foo('foo'), ['foo', 'bar']) 166 | 167 | def test_get_conn_with_split_router(self): 168 | # test dummy router 169 | p = BaseCluster( 170 | backend=DummyConnection, 171 | hosts={ 172 | 0: {'resp': 'foo'}, 173 | 1: {'resp': 'bar'}, 174 | }, 175 | router=DummyRouter, 176 | ) 177 | self.assertEquals(p.get_conn().num, 0) 178 | self.assertEquals(p.get_conn('foo').num, 1) 179 | 180 | def test_get_conn_default_routing_with_multiple_hosts(self): 181 | # test default routing behavior 182 | p = BaseCluster( 183 | backend=DummyConnection, 184 | hosts={ 185 | 0: {'resp': 'foo'}, 186 | 1: {'resp': 'bar'}, 187 | }, 188 | ) 189 | self.assertEquals(map(lambda x: x.num, p.get_conn()), [0, 1]) 190 | self.assertEquals(map(lambda x: x.num, p.get_conn('foo')), [0, 1]) 191 | 192 | 193 | class MapTest(BaseTest): 194 | @fixture 195 | def cluster(self): 196 | return BaseCluster( 197 | backend=DummyConnection, 198 | hosts={ 199 | 0: {'resp': 'foo'}, 200 | 1: {'resp': 'bar'}, 201 | }, 202 | ) 203 | 204 | def test_handles_single_routing_results(self): 205 | self.cluster.install_router(DummyRouter) 206 | 207 | with self.cluster.map() as conn: 208 | foo = conn.foo() 209 | bar = conn.foo('foo') 210 | self.assertEquals(foo, None) 211 | self.assertEquals(bar, None) 212 | 213 | self.assertEquals(bar, 'bar') 214 | self.assertEquals(foo, 'foo') 215 | 216 | def test_handles_groups_of_results(self): 217 | with self.cluster.map() as conn: 218 | foo = conn.foo() 219 | bar = conn.foo('foo') 220 | self.assertEquals(foo, None) 221 | self.assertEquals(bar, None) 222 | 223 | self.assertEquals(foo, ['foo', 'bar']) 224 | self.assertEquals(bar, ['foo', 'bar']) 225 | 226 | 227 | class MapWithFailuresTest(BaseTest): 228 | @fixture 229 | def cluster(self): 230 | return BaseCluster( 231 | backend=DummyErroringConnection, 232 | hosts={ 233 | 0: {'resp': 'foo'}, 234 | 1: {'resp': 'error'}, 235 | }, 236 | router=DummyRouter, 237 | ) 238 | 239 | def test_propagates_errors(self): 240 | with self.assertRaises(CommandError): 241 | with self.cluster.map() as conn: 242 | foo = conn.foo() 243 | bar = conn.foo('foo') 244 | self.assertEquals(foo, None) 245 | self.assertEquals(bar, None) 246 | 247 | def test_fail_silenlty(self): 248 | with self.cluster.map(fail_silently=True) as conn: 249 | foo = conn.foo() 250 | bar = conn.foo('foo') 251 | self.assertEquals(foo, None) 252 | self.assertEquals(bar, None) 253 | 254 | self.assertEquals(len(conn.get_errors()), 1, conn.get_errors()) 255 | self.assertEquals(type(conn.get_errors()[0][1]), ValueError) 256 | 257 | self.assertEquals(foo, 'foo') 258 | self.assertNotEquals(foo, 'bar') 259 | 260 | 261 | class FlakeyConnection(DummyConnection): 262 | 263 | retryable_exceptions = [Exception] 264 | 265 | def foo(self, *args, **kwargs): 266 | if hasattr(self, 'already_failed'): 267 | super(FlakeyConnection, self).foo() 268 | else: 269 | self.already_failed = True 270 | raise Exception('boom!') 271 | 272 | 273 | class RetryableRouter(DummyRouter): 274 | retryable = True 275 | 276 | def __init__(self, *args, **kwargs): 277 | self.kwargs_seen = [] 278 | self.key_args_seen = [] 279 | super(RetryableRouter, self).__init__(*args, **kwargs) 280 | 281 | def get_dbs(self, attr, args, kwargs, **fkwargs): 282 | key = get_key(args, kwargs) 283 | self.kwargs_seen.append(fkwargs) 284 | self.key_args_seen.append(key) 285 | return [0] 286 | 287 | 288 | class ScumbagConnection(DummyConnection): 289 | 290 | retryable_exceptions = [Exception] 291 | 292 | def foo(self): 293 | raise Exception("Says it's a connection / Never actually connects.") 294 | 295 | 296 | class RetryClusterTest(BaseTest): 297 | 298 | def build_cluster(self, connection=FlakeyConnection, router=RetryableRouter): 299 | return create_cluster({ 300 | 'backend': connection, 301 | 'router': router, 302 | 'hosts': { 303 | 0: {'resp': 'bar'}, 304 | } 305 | }) 306 | 307 | def test_returns_correctly(self): 308 | cluster = self.build_cluster(connection=DummyConnection) 309 | self.assertEquals(cluster.foo(), 'bar') 310 | 311 | def test_retry_router_when_receives_error(self): 312 | cluster = self.build_cluster() 313 | 314 | cluster.foo() 315 | self.assertEquals({'retry_for': 0}, cluster.router.kwargs_seen.pop()) 316 | 317 | def test_protection_from_infinate_loops(self): 318 | cluster = self.build_cluster(connection=ScumbagConnection) 319 | with self.assertRaises(Exception): 320 | cluster.foo() 321 | 322 | 323 | class EventualCommandTest(BaseTest): 324 | def test_unevaled_repr(self): 325 | ec = EventualCommand('foo') 326 | ec('bar', baz='foo') 327 | 328 | self.assertEquals(repr(ec), u"") 329 | 330 | def test_evaled_repr(self): 331 | ec = EventualCommand('foo') 332 | ec('bar', baz='foo') 333 | ec.resolve_as('biz') 334 | 335 | self.assertEquals(repr(ec), u"'biz'") 336 | 337 | def test_coersion(self): 338 | ec = EventualCommand('foo')() 339 | ec.resolve_as('5') 340 | 341 | self.assertEquals(int(ec), 5) 342 | 343 | def test_nonzero(self): 344 | ec = EventualCommand('foo')() 345 | ec.resolve_as(None) 346 | 347 | self.assertEquals(int(ec or 0), 0) 348 | 349 | def test_evaled_unicode(self): 350 | ec = EventualCommand('foo') 351 | ec.resolve_as('biz') 352 | 353 | self.assertEquals(unicode(ec), u'biz') 354 | 355 | def test_command_error_returns_as_error(self): 356 | ec = EventualCommand('foo') 357 | ec.resolve_as(CommandError([ValueError('test')])) 358 | self.assertEquals(ec.is_error, True) 359 | 360 | def test_other_error_does_not_return_as_error(self): 361 | ec = EventualCommand('foo') 362 | ec.resolve_as(ValueError('test')) 363 | self.assertEquals(ec.is_error, False) 364 | 365 | def test_isinstance_check(self): 366 | ec = EventualCommand('foo') 367 | ec.resolve_as(['foo', 'bar']) 368 | 369 | self.assertEquals(isinstance(ec, list), True) 370 | 371 | 372 | class ApplyDefaultsTest(BaseTest): 373 | def test_does_apply(self): 374 | host = {'port': 6379} 375 | defaults = {'host': 'localhost'} 376 | results = apply_defaults(host, defaults) 377 | self.assertEquals(results, { 378 | 'port': 6379, 379 | 'host': 'localhost', 380 | }) 381 | 382 | def test_does_not_overwrite(self): 383 | host = {'port': 6379} 384 | defaults = {'port': 9000} 385 | results = apply_defaults(host, defaults) 386 | self.assertEquals(results, { 387 | 'port': 6379, 388 | }) 389 | -------------------------------------------------------------------------------- /tests/nydus/db/routers/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/disqus/nydus/9b505840da47a34f758a830c3992fa5dcb7bb7ad/tests/nydus/db/routers/__init__.py -------------------------------------------------------------------------------- /tests/nydus/db/routers/tests.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | 3 | import mock 4 | import time 5 | 6 | from collections import Iterable 7 | from inspect import getargspec 8 | 9 | from nydus.db.base import BaseCluster 10 | from nydus.db.backends import BaseConnection 11 | from nydus.db.routers import BaseRouter, RoundRobinRouter 12 | from nydus.db.routers.keyvalue import ConsistentHashingRouter 13 | from nydus.testutils import BaseTest 14 | 15 | 16 | def _get_func(func): 17 | return getattr(func, '__wraps__', func) 18 | 19 | 20 | class DummyConnection(BaseConnection): 21 | def __init__(self, i): 22 | self.host = 'dummyhost' 23 | self.i = i 24 | super(DummyConnection, self).__init__(i) 25 | 26 | @property 27 | def identifier(self): 28 | return "%s:%s" % (self.host, self.i) 29 | 30 | 31 | class BaseRouterTest(BaseTest): 32 | Router = BaseRouter 33 | 34 | class TestException(Exception): 35 | pass 36 | 37 | def setUp(self): 38 | self.hosts = dict((i, {}) for i in xrange(5)) 39 | self.cluster = BaseCluster(router=self.Router, hosts=self.hosts, backend=DummyConnection) 40 | self.router = self.cluster.router 41 | 42 | def get_dbs(self, *args, **kwargs): 43 | return self.router.get_dbs(*args, **kwargs) 44 | 45 | def test_not_ready(self): 46 | self.assertTrue(not self.router._ready) 47 | 48 | def test_get_dbs_iterable(self): 49 | db_nums = self.get_dbs(attr='test', args=('foo',)) 50 | self.assertIsInstance(db_nums, Iterable) 51 | 52 | def test_get_dbs_unabletosetuproute(self): 53 | with mock.patch.object(self.router, '_setup_router', return_value=False): 54 | with self.assertRaises(BaseRouter.UnableToSetupRouter): 55 | self.get_dbs(attr='test', args=('foo',)) 56 | 57 | def test_setup_router_returns_true(self): 58 | self.assertTrue(self.router.setup_router()) 59 | 60 | def test_offers_router_interface(self): 61 | func = _get_func(self.router.get_dbs) 62 | self.assertTrue(callable(func)) 63 | dbargs, _, _, dbdefaults = getargspec(func) 64 | self.assertTrue(set(dbargs) >= set(['self', 'attr', 'args', 'kwargs'])) 65 | self.assertIsNone(dbdefaults) 66 | 67 | func = _get_func(self.router.setup_router) 68 | self.assertTrue(callable(func)) 69 | setupargs, _, _, setupdefaults = getargspec(func) 70 | self.assertTrue(set(setupargs) >= set(['self', 'args', 'kwargs'])) 71 | self.assertIsNone(setupdefaults) 72 | 73 | def test_returns_whole_cluster_without_key(self): 74 | self.assertEquals(self.hosts.keys(), self.get_dbs(attr='test')) 75 | 76 | def test_get_dbs_handles_exception(self): 77 | with mock.patch.object(self.router, '_route') as _route: 78 | with mock.patch.object(self.router, '_handle_exception') as _handle_exception: 79 | _route.side_effect = self.TestException() 80 | 81 | self.get_dbs(attr='test', args=('foo',)) 82 | 83 | self.assertTrue(_handle_exception.called) 84 | 85 | 86 | class BaseBaseRouterTest(BaseRouterTest): 87 | def test__setup_router_returns_true(self): 88 | self.assertTrue(self.router._setup_router()) 89 | 90 | def test__pre_routing_returns_args_and_kwargs(self): 91 | self.assertEqual((('foo',), {}), self.router._pre_routing(attr='test', args=('foo',))) 92 | 93 | def test__route_returns_first_db_num(self): 94 | self.assertEqual(self.cluster.hosts.keys()[0], self.router._route(attr='test', args=('foo',))[0]) 95 | 96 | def test__post_routing_returns_db_nums(self): 97 | db_nums = self.hosts.keys() 98 | 99 | self.assertEqual(db_nums, self.router._post_routing(attr='test', db_nums=db_nums, args=('foo',))) 100 | 101 | def test__handle_exception_raises_same_exception(self): 102 | e = self.TestException() 103 | 104 | with self.assertRaises(self.TestException): 105 | self.router._handle_exception(e) 106 | 107 | def test_returns_sequence_with_one_item_when_given_key(self): 108 | self.assertEqual(len(self.get_dbs(attr='test', args=('foo',))), len(self.hosts)) 109 | 110 | 111 | class BaseRoundRobinRouterTest(BaseRouterTest): 112 | Router = RoundRobinRouter 113 | 114 | def setUp(self): 115 | super(BaseRoundRobinRouterTest, self).setUp() 116 | assert self.router._setup_router() 117 | 118 | def test_ensure_db_num(self): 119 | db_num = 0 120 | s_db_num = str(db_num) 121 | 122 | self.assertEqual(self.router.ensure_db_num(db_num), db_num) 123 | self.assertEqual(self.router.ensure_db_num(s_db_num), db_num) 124 | 125 | def test_esnure_db_num_raises(self): 126 | with self.assertRaises(RoundRobinRouter.InvalidDBNum): 127 | self.router.ensure_db_num('a') 128 | 129 | def test_flush_down_connections(self): 130 | self.router._get_db_attempts = 9001 131 | self._down_connections = {0: time.time()} 132 | 133 | self.router.flush_down_connections() 134 | 135 | self.assertEqual(self.router._get_db_attempts, 0) 136 | self.assertEqual(self.router._down_connections, {}) 137 | 138 | def test_mark_connection_down(self): 139 | db_num = 0 140 | 141 | self.router.mark_connection_down(db_num) 142 | 143 | self.assertAlmostEqual(self.router._down_connections[db_num], time.time(), delta=10) 144 | 145 | def test_mark_connection_up(self): 146 | db_num = 0 147 | 148 | self.router.mark_connection_down(db_num) 149 | 150 | self.assertIn(db_num, self.router._down_connections) 151 | 152 | self.router.mark_connection_up(db_num) 153 | 154 | self.assertNotIn(db_num, self.router._down_connections) 155 | 156 | def test__pre_routing_updates__get_db_attempts(self): 157 | self.router._pre_routing(attr='test', args=('foo',)) 158 | 159 | self.assertEqual(self.router._get_db_attempts, 1) 160 | 161 | @mock.patch('nydus.db.routers.base.RoundRobinRouter.check_down_connections') 162 | def test__pre_routing_check_down_connections(self, _check_down_connections): 163 | self.router._get_db_attempts = RoundRobinRouter.attempt_reconnect_threshold + 1 164 | 165 | self.router._pre_routing(attr='test', args=('foo',)) 166 | 167 | self.assertTrue(_check_down_connections.called) 168 | 169 | @mock.patch('nydus.db.routers.base.RoundRobinRouter.mark_connection_down') 170 | def test__pre_routing_retry_for(self, _mark_connection_down): 171 | db_num = 0 172 | 173 | self.router._pre_routing(attr='test', args=('foo',), retry_for=db_num) 174 | 175 | _mark_connection_down.assert_called_with(db_num) 176 | 177 | @mock.patch('nydus.db.routers.base.RoundRobinRouter.mark_connection_up') 178 | def test_online_connections_dont_get_marked_as_up(self, mark_connection_up): 179 | db_nums = [0] 180 | 181 | self.assertEqual(self.router._post_routing(attr='test', db_nums=db_nums, args=('foo',)), db_nums) 182 | self.assertFalse(mark_connection_up.called) 183 | 184 | @mock.patch('nydus.db.routers.base.RoundRobinRouter.mark_connection_up') 185 | def test_offline_connections_get_marked_as_up(self, mark_connection_up): 186 | self.router.mark_connection_down(0) 187 | db_nums = [0] 188 | 189 | self.assertEqual(self.router._post_routing(attr='test', db_nums=db_nums, args=('foo',)), db_nums) 190 | mark_connection_up.assert_called_with(db_nums[0]) 191 | 192 | 193 | class RoundRobinRouterTest(BaseRoundRobinRouterTest): 194 | def test__setup_router(self): 195 | self.assertTrue(self.router._setup_router()) 196 | self.assertIsInstance(self.router._hosts_cycler, Iterable) 197 | 198 | def test__route_cycles_through_keys(self): 199 | db_nums = self.hosts.keys() * 2 200 | results = [self.router._route(attr='test', args=('foo',))[0] for _ in db_nums] 201 | 202 | self.assertEqual(results, db_nums) 203 | 204 | def test__route_retry(self): 205 | self.router.retry_timeout = 0 206 | 207 | db_num = 0 208 | 209 | self.router.mark_connection_down(db_num) 210 | 211 | db_nums = self.router._route(attr='test', args=('foo',)) 212 | 213 | self.assertEqual(db_nums, [db_num]) 214 | 215 | def test__route_skip_down(self): 216 | db_num = 0 217 | 218 | self.router.mark_connection_down(db_num) 219 | 220 | db_nums = self.router._route(attr='test', args=('foo',)) 221 | 222 | self.assertNotEqual(db_nums, [db_num]) 223 | self.assertEqual(db_nums, [db_num + 1]) 224 | 225 | def test__route_hostlistexhausted(self): 226 | [self.router.mark_connection_down(db_num) for db_num in self.hosts.keys()] 227 | 228 | with self.assertRaises(RoundRobinRouter.HostListExhausted): 229 | self.router._route(attr='test', args=('foo',)) 230 | 231 | 232 | class ConsistentHashingRouterTest(BaseRoundRobinRouterTest): 233 | Router = ConsistentHashingRouter 234 | 235 | def get_dbs(self, *args, **kwargs): 236 | kwargs['attr'] = 'test' 237 | return super(ConsistentHashingRouterTest, self).get_dbs(*args, **kwargs) 238 | 239 | def test_retry_gives_next_host_if_primary_is_offline(self): 240 | self.assertEquals([2], self.get_dbs(args=('foo',))) 241 | self.assertEquals([4], self.get_dbs(args=('foo',), retry_for=2)) 242 | 243 | def test_retry_host_change_is_sticky(self): 244 | self.assertEquals([2], self.get_dbs(args=('foo',))) 245 | self.assertEquals([4], self.get_dbs(args=('foo',), retry_for=2)) 246 | 247 | self.assertEquals([4], self.get_dbs(args=('foo',))) 248 | 249 | def test_raises_host_list_exhaused_if_no_host_can_be_found(self): 250 | # Kill the first 4 251 | [self.get_dbs(retry_for=i) for i in range(4)] 252 | 253 | # And the 5th should raise an error 254 | self.assertRaises( 255 | ConsistentHashingRouter.HostListExhausted, 256 | self.get_dbs, **dict(args=('foo',), retry_for=4)) 257 | --------------------------------------------------------------------------------