├── docs └── tutorial_zh.md ├── zyredis ├── utils │ ├── qconf_py.so │ ├── tree.py │ ├── singleton.py │ ├── __init__.py │ └── parse_url.py ├── key.py ├── exceptions.py ├── __init__.py ├── model.py ├── serialization.py ├── manager.py ├── client.py └── types.py ├── INSTALL ├── MANIFEST ├── PKG-INFO ├── LICENSE ├── test ├── test_key.py ├── test_manager.py ├── test_qconf.py ├── test_model.py └── test_client.py ├── setup.py └── README.md /docs/tutorial_zh.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ireaderlab/zyredis/HEAD/docs/tutorial_zh.md -------------------------------------------------------------------------------- /zyredis/utils/qconf_py.so: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ireaderlab/zyredis/HEAD/zyredis/utils/qconf_py.so -------------------------------------------------------------------------------- /INSTALL: -------------------------------------------------------------------------------- 1 | 2 | Please use 3 | python setup.py install 4 | 5 | and report errors to wanglichao@zhangyue.com or xidianwlc@qq.com 6 | -------------------------------------------------------------------------------- /MANIFEST: -------------------------------------------------------------------------------- 1 | # file GENERATED by distutils, do NOT edit 2 | setup.py 3 | test/test_client.py 4 | test/test_key.py 5 | test/test_manager.py 6 | test/test_model.py 7 | test/test_qconf.py 8 | zyredis/__init__.py 9 | zyredis/client.py 10 | zyredis/exceptions.py 11 | zyredis/key.py 12 | zyredis/manager.py 13 | zyredis/model.py 14 | zyredis/serialization.py 15 | zyredis/types.py 16 | zyredis/utils/__init__.py 17 | zyredis/utils/parse_url.py 18 | zyredis/utils/qconf_py.so 19 | zyredis/utils/singleton.py 20 | zyredis/utils/tree.py 21 | -------------------------------------------------------------------------------- /zyredis/key.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | """ 5 | Copyright (c) 2014,掌阅科技 6 | All rights reserved. 7 | 8 | 摘 要: key.py 9 | 创 建 者: WangLichao 10 | 创建日期: 2014-10-24 11 | """ 12 | 13 | 14 | class Key(unicode): 15 | 16 | """使用[]操作来拼接redis的key 17 | 默认下划线分隔符 18 | """ 19 | 20 | def __getitem__(self, key): 21 | if not isinstance(key, basestring): 22 | key = str(key) 23 | if not self: 24 | return Key(key) 25 | return Key(u"{}_{}".format(self, key)) 26 | -------------------------------------------------------------------------------- /zyredis/exceptions.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | """ 5 | Copyright (c) 2015,掌阅科技 6 | All rights reserved. 7 | 8 | 摘 要: exceptions.py 9 | 创 建 者: WangLichao 10 | 创建日期: 2015-08-18 11 | """ 12 | 13 | 14 | class ZyRedisError(StandardError): 15 | 16 | """错误类统一处理 17 | """ 18 | pass 19 | 20 | 21 | class NotSupportCommandError(ZyRedisError): 22 | 23 | """命令不支持 24 | """ 25 | pass 26 | 27 | 28 | class ConfError(ZyRedisError): 29 | 30 | """获取配置失败 31 | """ 32 | pass 33 | 34 | if __name__ == '__main__': 35 | pass 36 | -------------------------------------------------------------------------------- /zyredis/__init__.py: -------------------------------------------------------------------------------- 1 | # zyredis 2 | from zyredis.key import Key 3 | from zyredis.client import * 4 | from zyredis.serialization import * 5 | from zyredis.exceptions import * 6 | from zyredis.types import * 7 | from zyredis.manager import RedisManager 8 | from zyredis.model import Model 9 | 10 | __version__ = '1.0.0' 11 | VERSION = tuple(map(int, __version__.split('.'))) 12 | 13 | __all__ = [ 14 | 'Client', 15 | 'Key', 16 | 'List', 17 | 'Set', 18 | 'SortedSet', 19 | 'Dict', 20 | 'Plain', 21 | 'Pickler', 22 | 'JSON', 23 | 'ZSet', 24 | 'NotSupportCommandError', 25 | 'Model' 26 | 'RedisManager' 27 | ] 28 | -------------------------------------------------------------------------------- /PKG-INFO: -------------------------------------------------------------------------------- 1 | Metadata-Version: 1.0 2 | Name: zyredis 3 | Version: 0.1.4 4 | Summary: ireader redis client 5 | Home-page: UNKNOWN 6 | Author: WangLichao 7 | Author-email: wanglichao@zhangyue.com 8 | License: BSD 9 | Description: ======= 10 | zyredis 11 | ======= 12 | ireader python接入redis客户端 13 | 14 | 项目描述 15 | -------- 16 | - 实现redis接入以及redis的failover机制 17 | - 实现redis客户端级别的负载均衡 18 | - 解决长连接失效后需要重启服务问题 19 | - 提供redis的基础model层 20 | 21 | 项目依赖 22 | -------- 23 | - QConf agent 24 | - QConf python client 25 | - redis client 26 | 27 | 版本变更 28 | -------- 29 | 30 | - v0.0.1 31 | 提供基本功能与failover机制 32 | - v0.1.3 33 | 解决qconf的python包内存泄露，增加SortedSet初始化对dict类型支持 34 | - v0.1.4 35 | 完善文档以及使用举例，增加开源协议 36 | 37 | Platform: UNKNOWN 38 | -------------------------------------------------------------------------------- /zyredis/utils/tree.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | ''' 5 | Copyright (c) 2014,掌阅科技 6 | All rights reserved. 7 | 8 | 摘 要: tree.py 9 | 创 建 者: WangLichao 10 | 创建日期: 2014-10-25 11 | ''' 12 | from collections import defaultdict 13 | 14 | class DefaultDict(defaultdict): 15 | '''为defaultdict添加属性操作 16 | ''' 17 | def __getattr__(self, attr): 18 | return self[attr] 19 | def __setattr__(self, attr, val): 20 | self[attr] = val 21 | 22 | def create_tree(): 23 | '''使用字典表示树的节点，非字典类型表示叶子节点 24 | 具体使用请参考测试数据 25 | ''' 26 | return DefaultDict(create_tree) 27 | 28 | def dicts(t): 29 | '''获取树的叶子节点 30 | ''' 31 | if isinstance(t, dict): 32 | return {k: dicts(t[k]) for k in t} 33 | return t 34 | 35 | if __name__ == '__main__': 36 | a = create_tree() 37 | a['x']['z'] = 6 38 | a['x']['d'] = [4, 6, 9] 39 | a['x']['y']['dd'] = 1 40 | a['x']['y']['xd'] = 1 41 | if a.z.y: 42 | print 'xxx' 43 | a.x.y.z = 248 44 | import ujson 45 | print ujson.dumps(a) 46 | print dicts(a) 47 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2015 zhangyue 2 | 3 | Permission is hereby granted, free of charge, to any person 4 | obtaining a copy of this software and associated documentation 5 | files (the "Software"), to deal in the Software without 6 | restriction, including without limitation the rights to use, 7 | copy, modify, merge, publish, distribute, sublicense, and/or sell 8 | copies of the Software, and to permit persons to whom the 9 | Software is furnished to do so, subject to the following 10 | conditions: 11 | 12 | The above copyright notice and this permission notice shall be 13 | included in all copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 16 | EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES 17 | OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 18 | NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT 19 | HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, 20 | WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 21 | FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 22 | OTHER DEALINGS IN THE SOFTWARE. 23 | -------------------------------------------------------------------------------- /test/test_key.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | """ 5 | Copyright (c) 2015,掌阅科技 6 | All rights reserved. 7 | 8 | 摘 要: test_key.py 9 | 创 建 者: WangLichao 10 | 创建日期: 2015-08-18 11 | """ 12 | import sys 13 | import os 14 | sys.path.append(os.path.dirname(os.path.split(os.path.realpath(__file__))[0])) 15 | import unittest 16 | from zyredis.key import Key 17 | 18 | 19 | class TestKeyModel(unittest.TestCase): 20 | 21 | """测试key生成逻辑 22 | """ 23 | 24 | def test_key_with_prefix(self): 25 | """测试带前缀情况 26 | """ 27 | key = Key("prefix") # 有前缀 28 | mykey = key["abc"]["def"]["ddd"] 29 | assert mykey == "prefix_abc_def_ddd" 30 | 31 | def test_key_no_prefix(self): 32 | 33 | """测试没有前缀的情况 34 | """ 35 | key = Key() # 无前缀 36 | key1 = Key("") 37 | mykey = key["abc"]["def"] 38 | mykey1 = key1["abc"]["def"] 39 | assert mykey == "abc_def" 40 | assert mykey1 == "abc_def" 41 | key2 = Key("") 42 | mykey2 = key2["ttt"]["xxx"] 43 | assert mykey2 == "ttt_xxx" 44 | 45 | if __name__ == '__main__': 46 | unittest.main() 47 | -------------------------------------------------------------------------------- /zyredis/utils/singleton.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | """ 5 | Copyright (c) 2015,掌阅科技 6 | All rights reserved. 7 | 8 | 摘 要: thread safe singleton pattern 9 | 创 建 者: WangLichao 10 | 创建日期: 2015-08-20 11 | """ 12 | import threading 13 | 14 | 15 | class SingletonMixin(object): 16 | 17 | """Based on tornado.ioloop.IOLoop.instance() approach. 18 | See https://github.com/facebook/tornado 19 | """ 20 | __singleton_lock = threading.Lock() 21 | __singleton_instance = None 22 | 23 | @classmethod 24 | def instance(cls): 25 | if not cls.__singleton_instance: 26 | with cls.__singleton_lock: 27 | if not cls.__singleton_instance: 28 | cls.__singleton_instance = cls() 29 | return cls.__singleton_instance 30 | 31 | 32 | if __name__ == '__main__': 33 | class A(SingletonMixin): 34 | pass 35 | 36 | class B(SingletonMixin): 37 | pass 38 | 39 | a, a2 = A.instance(), A.instance() 40 | b, b2 = B.instance(), B.instance() 41 | 42 | assert a is a2 43 | assert b is b2 44 | assert a is not b 45 | 46 | print('a: %s\na2: %s' % (a, a2)) 47 | print('b: %s\nb2: %s' % (b, b2)) 48 | -------------------------------------------------------------------------------- /zyredis/model.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | """ 5 | Copyright (c) 2015,掌阅科技 6 | All rights reserved. 7 | 8 | 摘 要: 基础model类实现用于统一管理redis的Model 9 | 创 建 者: WangLichao 10 | 创建日期: 2015-08-19 11 | """ 12 | from zyredis.manager import RedisManager 13 | from zyredis.key import Key 14 | 15 | 16 | class Model(object): 17 | 18 | """redis所有model的基类 19 | Attributes: 20 | client_name: 选择redis 21 | db_num: 使用db编号 22 | """ 23 | client_name = "" 24 | db_num = 0 25 | prefix = "" 26 | zk_path = "" 27 | init_from = "local" 28 | redis_manager = None 29 | redis_conf = { 30 | 0: "redis://localhost:6371/myclient?weight=1&db=0&transaction=1", 31 | 1:"redis://localhost:6371/myclient?weight=1&db=1&transaction=0" 32 | } 33 | 34 | @property 35 | def db(self): 36 | """当前使用的redis,RedisManager使用前需要进行初始化 37 | """ 38 | if self.redis_manager is None: 39 | if self.init_from == "qconf": 40 | self.redis_manager = RedisManager.instance().init_from_qconf(self.zk_path) 41 | else: 42 | self.redis_manager = RedisManager.instance().init_from_local(self.redis_conf) 43 | return self.redis_manager.select_db(self.client_name, self.db_num) 44 | 45 | @property 46 | def key(self): 47 | """构造redis的key 48 | """ 49 | return Key(self.prefix) 50 | -------------------------------------------------------------------------------- /zyredis/utils/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | """ 5 | Copyright (c) 2014,掌阅科技 6 | All rights reserved. 7 | 8 | 摘 要: client.py 9 | 创 建 者: WangLichao 10 | 创建日期: 2014-10-24 11 | """ 12 | import logging 13 | import time 14 | from datetime import datetime 15 | 16 | LOG = logging.getLogger("zyredis") 17 | 18 | def sublist(lst, slice_size): 19 | """列表切片 20 | """ 21 | if not slice_size: 22 | return lst 23 | sub = [] 24 | result = [] 25 | for i in lst: 26 | sub += [i] 27 | if len(sub) == slice_size: 28 | result += [sub] 29 | sub = [] 30 | if sub: 31 | result += [sub] 32 | return result 33 | 34 | 35 | def maybe_list(value): 36 | """maybe list 37 | """ 38 | if hasattr(value, "__iter__"): 39 | return value 40 | if value is None: 41 | return [] 42 | return [value] 43 | 44 | 45 | def mkey(names, flag="_"): 46 | """mkey 47 | """ 48 | return flag.join(maybe_list(names)) 49 | 50 | 51 | def dt_to_timestamp(datetime_): 52 | """Convert :class:`datetime` to UNIX timestamp.""" 53 | return time.mktime(datetime_.timetuple()) 54 | 55 | 56 | def maybe_datetime(timestamp): 57 | """Convert datetime to timestamp, only if timestamp 58 | is a datetime object.""" 59 | if isinstance(timestamp, datetime): 60 | return dt_to_timestamp(timestamp) 61 | return timestamp 62 | -------------------------------------------------------------------------------- /test/test_manager.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | """ 5 | Copyright (c) 2015,掌阅科技 6 | All rights reserved. 7 | 8 | 摘 要: test_manager.py 9 | 创 建 者: WangLichao 10 | 创建日期: 2015-08-21 11 | """ 12 | import sys 13 | import os 14 | import time 15 | import logging.config 16 | sys.path.append(os.path.dirname(os.path.split(os.path.realpath(__file__))[0])) 17 | import unittest 18 | from zyredis.manager import RedisManager 19 | 20 | LOGGING = { 21 | 'version': 1, 22 | 'disable_existing_loggers': False, 23 | 'handlers': { 24 | 'console': { 25 | 'class': 'logging.StreamHandler', 26 | }, 27 | }, 28 | 'loggers': { 29 | 'zyredis': { 30 | 'handlers': ['console'], 31 | 'level': 'DEBUG', 32 | }, 33 | }, 34 | } 35 | 36 | logging.config.dictConfig(LOGGING) 37 | 38 | 39 | class TestRedisManager(unittest.TestCase): 40 | 41 | """测试redis manager基本用法 42 | """ 43 | 44 | def test_singleton(self): 45 | """测试基本操作 46 | """ 47 | r1, r2 = RedisManager.instance(), RedisManager.instance() 48 | assert r1 is r2 49 | 50 | def test_init(self): 51 | """测试初始化方法,以及配置变更后是否自动识别 52 | """ 53 | count = 0 54 | while True: 55 | rm = RedisManager.instance().init_from_qconf("/test_group/service/codis") 56 | db = rm.select_db("test_codis") 57 | db.set("wlctest", "wlctest") 58 | assert db.get("wlctest") == "wlctest" 59 | time.sleep(1) 60 | count += 1 61 | if count > 1: 62 | break 63 | 64 | 65 | if __name__ == '__main__': 66 | unittest.main() 67 | -------------------------------------------------------------------------------- /zyredis/utils/parse_url.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | """ 5 | Copyright (c) 2015,掌阅科技 6 | All rights reserved. 7 | 8 | 摘 要: parse_url.py 9 | 创 建 者: WangLichao 10 | 创建日期: 2015-08-21 11 | """ 12 | from urlparse import urlparse, parse_qs 13 | 14 | def parse_from_url(url): 15 | """从url解析配置信息 16 | Example: 17 | url = "redis://host:port/client_name?db=0&weight=1&transaction=1" 18 | url = "redis://localhsot:6379/myredis?db=0&weight=1" 19 | Args: 20 | url: string 21 | """ 22 | url_string = url 23 | url = urlparse(url) 24 | qs = '' 25 | 26 | # in python2.6, custom URL schemes don't recognize querystring values 27 | # they're left as part of the url.path. 28 | if '?' in url.path and not url.query: 29 | # chop the querystring including the ? off the end of the url 30 | # and reparse it. 31 | qs = url.path.split('?', 1)[1] 32 | url = urlparse(url_string[:-(len(qs) + 1)]) 33 | else: 34 | qs = url.query 35 | if url.scheme != "redis" or not url.path: 36 | raise ValueError("url conf error, url must be redis protocol") 37 | 38 | url_options = {} 39 | 40 | for name, value in parse_qs(qs).iteritems(): 41 | if value and len(value) > 0: 42 | url_options[name] = value[0] 43 | url_options['db'] = int(url_options.get('db', 0)) 44 | url_options['weight'] = int(url_options.get('weight', 1)) 45 | url_options['client_name'] = url.path.lstrip("/") 46 | host, port = url.netloc.split(":") # 如果不符合规则抛出ValueError异常 47 | url_options['host'] = host 48 | url_options['port'] = port 49 | url_options['transaction'] = int(url_options.get("transaction", 0)) 50 | return url_options 51 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env python 2 | import os 3 | import sys 4 | from distutils.core import setup 5 | from distutils.command.install_data import install_data 6 | from distutils.command.install import INSTALL_SCHEMES 7 | from zyredis import __version__ 8 | 9 | # perform the setup action 10 | 11 | packages, data_files = [], [] 12 | 13 | cmdclasses = {'install_data': install_data} 14 | 15 | for scheme in INSTALL_SCHEMES.values(): 16 | scheme['data'] = scheme['purelib'] 17 | 18 | 19 | def fullsplit(path, result=None): 20 | """ 21 | Split a pathname into components (the opposite of os.path.join) in a 22 | platform-neutral way. 23 | """ 24 | if result is None: 25 | result = [] 26 | head, tail = os.path.split(path) 27 | if head == '': 28 | return [tail] + result 29 | if head == path: 30 | return result 31 | return fullsplit(head, [tail] + result) 32 | 33 | 34 | def is_not_module(filename): 35 | """check filename 36 | """ 37 | return os.path.splitext(filename)[1] not in ['.py', '.pyc', '.pyo'] 38 | 39 | for zyredis_dir in ['zyredis']: 40 | for dirpath, dirnames, filenames in os.walk(zyredis_dir): 41 | # Ignore dirnames that start with '.' 42 | for i, dirname in enumerate(dirnames): 43 | if dirname.startswith('.'): 44 | del dirnames[i] 45 | if '__init__.py' in filenames: 46 | packages.append('.'.join(fullsplit(dirpath))) 47 | data = [f for f in filenames if is_not_module(f)] 48 | if data: 49 | data_files.append([dirpath, [os.path.join(dirpath, f) for f in data]]) 50 | elif filenames: 51 | data_files.append([dirpath, [os.path.join(dirpath, f) for f in filenames]]) 52 | data_files.append(['.', ['README.md']]) 53 | 54 | 55 | setup_args = { 56 | 'name': 'zyredis', 57 | 'version': __version__, 58 | 'description': 'ireader redis client', 59 | 'long_description': open('README.md').read(), 60 | 'author': 'WangLichao', 61 | 'author_email': "wanglichao@zhangyue.com", 62 | 'packages': packages, 63 | 'data_files': data_files, 64 | 'include_package_data': True, 65 | } 66 | 67 | setup(**setup_args) 68 | -------------------------------------------------------------------------------- /zyredis/serialization.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | """ 5 | Copyright (c) 2014,掌阅科技 6 | All rights reserved. 7 | 8 | 摘 要: serialization.py 9 | 创 建 者: WangLichao 10 | 创建日期: 2014-10-24 11 | """ 12 | 13 | try: 14 | import cPickle as pickle 15 | except ImportError: 16 | import pickle 17 | 18 | try: 19 | import ujson as json 20 | except ImportError: 21 | import json 22 | 23 | from zyredis.utils import maybe_list 24 | 25 | 26 | class Serializer(object): 27 | """序列化的基类 28 | 序列化和反序列化一定要成对 29 | Attributes: 30 | encoding: 编码方式 31 | Example: 32 | >>> s = Pickler(encoding='zlib') 33 | >>> val = s.encode({"foo": "bar"}) 34 | >>> s.decode(val) 35 | """ 36 | 37 | def __init__(self, encoding=None): 38 | self.encoding = encoding 39 | 40 | def encode(self, value): 41 | """Encode value.""" 42 | value = self.serialize(value) 43 | if self.encoding: 44 | value = value.encode(self.encoding) 45 | return value 46 | 47 | def decode(self, value): 48 | """Decode value.""" 49 | if self.encoding: 50 | value = value.decode(self.encoding) 51 | return self.deserialize(value) 52 | 53 | def serialize(self, value): 54 | """序列化 55 | """ 56 | raise NotImplementedError("Serializers must implement serialize()") 57 | 58 | def deserialize(self, value): 59 | """反序列化 60 | """ 61 | raise NotImplementedError("Serializers must implement deserialize()") 62 | 63 | 64 | class Plain(Serializer): 65 | """不做操作的方式 66 | """ 67 | 68 | def serialize(self, value): 69 | return value 70 | 71 | def deserialize(self, value): 72 | return value 73 | 74 | 75 | class Pickler(Serializer): 76 | """pickle序列化方式""" 77 | protocol = 2 78 | 79 | def serialize(self, value): 80 | return pickle.dumps(value, protocol=self.protocol) 81 | 82 | def deserialize(self, value): 83 | return pickle.loads(value) 84 | 85 | 86 | class JSON(Serializer): 87 | '''JSON序列化方式''' 88 | def serialize(self, value): 89 | return json.dumps(value) 90 | 91 | def deserialize(self, value): 92 | try: 93 | return json.loads(value) 94 | except ValueError: 95 | return value 96 | -------------------------------------------------------------------------------- /test/test_qconf.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | """ 5 | Copyright (c) 2015,掌阅科技 6 | All rights reserved. 7 | 8 | 摘 要: test_qconf.py 9 | 创 建 者: WangLichao 10 | 创建日期: 2015-08-20 11 | """ 12 | import sys 13 | import os 14 | sys.path.append(os.path.dirname(os.path.split(os.path.realpath(__file__))[0])) 15 | import unittest 16 | from zyredis.utils import qconf_py as qconf 17 | 18 | 19 | def get_qconf_data(zk_path): 20 | """解决qconf get_batch_conf有内存泄露问题 21 | """ 22 | qconf_data = {} 23 | data_keys = qconf.get_batch_keys(zk_path) 24 | for _key in data_keys: 25 | node_key = "{}/{}".format(zk_path, _key) 26 | qconf_data[_key] = qconf.get_conf(node_key) 27 | return qconf_data 28 | 29 | 30 | class TestQconf(unittest.TestCase): 31 | 32 | """测试qconf读取配置 33 | """ 34 | 35 | def test_api(self): 36 | """测试基本api 37 | """ 38 | key = "/test_group/service/codis" 39 | value = qconf.get_conf(key) 40 | keys = qconf.get_batch_keys(key) 41 | assert isinstance(keys, list) 42 | children = qconf.get_batch_conf(key) 43 | assert isinstance(children, dict) 44 | 45 | def test_memleak_get_conf(self): 46 | """测试get_conf是否有内存泄露 47 | """ 48 | key = "/test_group/service/codis" 49 | i = 0 50 | while i < 1: 51 | qconf.get_conf(key) 52 | i = i + 1 53 | 54 | def test_memleak_get_batch_keys(self): 55 | """测试get_batch_keys是否有内存泄露 56 | """ 57 | key = "/test_group/service/codis" 58 | i = 0 59 | while i < 1: 60 | qconf.get_batch_keys(key) 61 | i = i + 1 62 | 63 | def test_memleak_get_batch_conf(self): 64 | """测试get_batch_conf是否有内存泄露 65 | """ 66 | key = "/test_group/service/codis" 67 | i = 0 68 | while i < 1: 69 | qconf.get_batch_conf(key) 70 | i = i + 1 71 | 72 | def test_diff_get_batch_conf(self): 73 | """批量获取对比接口测试 74 | """ 75 | key = "/test_group/service/codis" 76 | children1 = qconf.get_batch_conf(key) 77 | children2 = get_qconf_data(key) 78 | assert len(children1) == len(children2) 79 | assert children1['0'] == children2["0"] 80 | assert children1['1'] == children2["1"] 81 | 82 | if __name__ == '__main__': 83 | unittest.main() 84 | -------------------------------------------------------------------------------- /zyredis/manager.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | """ 5 | Copyright (c) 2015,掌阅科技 6 | All rights reserved. 7 | 8 | 摘 要: 管理redis实例 9 | 创 建 者: WangLichao 10 | 创建日期: 2015-08-19 11 | """ 12 | import random 13 | from zyredis.exceptions import ConfError 14 | from zyredis.client import Client 15 | from zyredis.utils.singleton import SingletonMixin 16 | from zyredis.utils import tree 17 | from zyredis.utils.parse_url import parse_from_url 18 | from zyredis.utils import LOG 19 | try: 20 | from zyredis.utils import qconf_py as qconf 21 | except: 22 | pass 23 | 24 | 25 | class RedisManager(SingletonMixin): 26 | 27 | """用于多个redis的ip:port的管理 28 | """ 29 | redis_pool = tree.create_tree() 30 | redis_local_flag = True 31 | zk_path = None 32 | 33 | def init_from_local(self, redis_conf): 34 | """从配置中读取redis的配置""" 35 | self.redis_local_flag = True 36 | self.redis_conf = redis_conf 37 | if not self.redis_pool: 38 | self._init_from_conf() 39 | return self 40 | 41 | def init_from_qconf(self, zk_path): 42 | self.redis_local_flag = False 43 | if self.zk_path is None: 44 | self.zk_path = zk_path 45 | 46 | if not self.redis_pool: 47 | self._init_from_conf() 48 | return self 49 | 50 | def _get_conf(self): 51 | """使用qconf从zookeeper获取节点信息 52 | """ 53 | try: 54 | # children = qconf.get_batch_conf(self.zk_path) # 官方qconf_so有内存泄露 55 | children = self._get_qconf_data() 56 | except qconf.Error as e: 57 | raise ConfError(e) 58 | 59 | if not children: 60 | raise ConfError("cant get conf from qconf, please check conf %s" % self.zk_path) 61 | return children 62 | 63 | def _get_qconf_data(self): 64 | """解决qconf get_batch_conf有内存泄露问题 65 | """ 66 | qconf_data = {} 67 | data_keys = qconf.get_batch_keys(self.zk_path) 68 | for _key in data_keys: 69 | node_key = "{}/{}".format(self.zk_path, _key) 70 | qconf_data[_key] = qconf.get_conf(node_key) 71 | return qconf_data 72 | 73 | def _init_from_conf(self): 74 | """根据配置初始化redis客户端连接 75 | """ 76 | self.redis_pool = tree.create_tree() # 重置redis_pool 77 | # qconf的话特殊对待 78 | if self.redis_local_flag is False: 79 | self.redis_conf = self._get_conf() 80 | LOG.info("init redis conf, redis_conf=%s", self.redis_conf) 81 | for url in self.redis_conf.itervalues(): 82 | options = parse_from_url(url) # 如果url不符合标准将会直接ValueError异常 83 | client_name = options["client_name"] 84 | db = options["db"] 85 | weight = options["weight"] 86 | if int(weight) == 0: 87 | continue 88 | client = Client( 89 | host=options["host"], 90 | port=options["port"], 91 | db=db, 92 | transaction=options.get("transaction", False), 93 | ) 94 | if self.redis_pool[client_name][db]: 95 | self.redis_pool[client_name][db].extend([client] * weight) 96 | else: 97 | self.redis_pool[client_name][db] = [client] * weight 98 | 99 | def _do_check_conf(self): 100 | """检测配置是否更新,如果配置有变更则重新加载配置信息 101 | """ 102 | new_conf = self._get_conf() 103 | for k, v in new_conf.iteritems(): 104 | if self.redis_conf.get(k, None) != v: 105 | self._init_from_conf() 106 | LOG.info("run _do_check_conf, config has changed") 107 | break 108 | 109 | def select_db(self, client_name, db=0): 110 | """根据client和db选择redis,根据weight配比随机选择 111 | """ 112 | if not self.redis_local_flag: 113 | self._do_check_conf() 114 | if not self.redis_pool[client_name][db]: 115 | raise ConfError('Redis db Conf Error') 116 | return random.choice(self.redis_pool[client_name][db]) 117 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | ======= 2 | zyredis yet another redis client that support failover and codis, zyredis is also a redis orm that supports define redis model 3 | ======= 4 | iReader python接入redis客户端 5 | 6 | 项目描述 7 | -------- 8 | 9 | - 实现redis接入以及redis的failover机制 10 | - 实现redis客户端级别的负载均衡 11 | - 解决长连接失效后需要重启服务问题 12 | - 提供redis的基础model层 13 | - 支持codis 14 | 15 | 项目当前状态 16 | --------- 17 | 18 | - 服务线上所有redis的接入，日请求量大于5亿 19 | - failover机制可以灵活切换redis proxy，并配置不同不服务器的负载 20 | 21 | 项目依赖 22 | -------- 23 | 24 | - [QConf](https://github.com/Qihoo360/QConf) 可以用也可以不用，非必须依赖 25 | - QConf python client目前已经集成到本目录代码中，zyredis/utils/qconf_py.so是通过QConf项目编译的python客户端，可以用也可以不用，非必须依赖 26 | - [redis client](https://github.com/andymccurdy/redis-py) 必须依赖 27 | - [Codis](https://github.com/wandoulabs/codis) 如果使用codis的话可以参考，非必须依赖项 28 | 29 | 当前版本 30 | -------- 31 | 32 | - v1.0.0 兼容redis所有原生命令和codis支持的命令，额外提供pythonic的数据结构，此外zyredis还是一个redis ORM用于规范redis的key的定义规则，数据类型的使用，zyredis还对阻塞redis后端的操作进行了统一优化避免因为开发的失误导致对后端redis的阻塞 33 | 34 | 35 | 安装 36 | -------- 37 | 38 | ``` 39 | git clone https://github.com/ireaderlab/zyredis.git 40 | cd zyredis 41 | python setup.py install 42 | ``` 43 | 44 | 使用举例 45 | ------- 46 | 47 | - zyredis封装了pythonic的对象有以下几个 48 | - Client: 用于和redis进行交互的对象 49 | - Key: 提供了统一的key的封装方式 50 | - List: redis官方的List数据结构的封装，支持python原生的List方法 51 | - Set: redis官方的Set数据结构的封装，支持python原生的set操作方法 52 | - SortedSet: redis官方的SortedSet数据结构的封装 53 | - Dict: redis官方的Dict数据结构的封装，支持python原生的dict操作方法 54 | - ZSet: ZSet官方的zset数据结构的封装 55 | - Plain: 对写入redis的数据的默认编解码操作，Plain是不对数据进行处理 56 | - Pickler: 使用python的Cpickle对数据进行编解码 57 | - JSON: 使用JSON对写入和读取的数据进行编解码 58 | - NotSupportCommandError: 可能会触发的异常，codis客户端存在个别命令不支持，通过异常的方式进行抛出 59 | - Model: 所有的redis相关的操作都通过一个model进行交互 60 | - RedisManager: 管理所有redis的连接 61 | 62 | 使用zyredis的orm举例 63 | ------ 64 | - 如果使用纯粹redis的orm管理的话可以用下面举例的方式来创建BaseRedisModel 65 | ```python 66 | # 定义完redis的基础model后其他所有的model均继承自此model即可 67 | class BaseRedisModel(Model): 68 | 69 | client_name = "test_redis" 70 | db_num = 0 71 | # 说明使用本地配置项 72 | init_from = "local" 73 | # 初始化redis配置 74 | redis_conf = { 75 | 0:"redis://localhost:6389/test_redis?weight=1&transaction=1&db=0", 76 | 1:"redis://localhost:6389/test_redis2?weight=3&transaction=1&db=0", 77 | } 78 | ``` 79 | - 如果使用qconf提供的配置项来做配置管理的话请参考下面配置 80 | qconf对应zookeeper配置项路径：/test_group/service/codis 81 | 该路径下节点codis0的值为：redis://localhost:6389/test_cache?weight=1 82 | 该路径下节点codis1的值为：redis://localhost:6339/test_cache?weight=3 83 | 该路径下节点codis2的值为：redis://localhost:6339/test_db?weight=1 84 | 该路径下节点codis3的值为：redis://localhost:6339/test_db?weight=2 85 | 86 | 这样的配置项目声明了两种proxy的客户端分别为用做缓存的test_cache和用做db的test_db 87 | ```python 88 | from zyredis import Model 89 | 90 | class BaseCacheModel(Model): 91 | 92 | client_name = "test_cache" # 声明使用的客户端名称 93 | db_num = 0 94 | init_from = "qconf" 95 | zk_path = "/test_group/service/codis" # qconf对应的zookeeper路径 96 | 97 | class BaseDbModel(Model): 98 | 99 | client_name = "test_db" # 声明使用的客户端名称 100 | db_num = 0 101 | init_from = "qconf" 102 | zk_path = "/test_group/service/codis" # qconf对应的zookeeper路径 103 | 104 | class TestCacheModel(BaseCacheModel): 105 | 106 | prefix = "wlctest" 107 | 108 | def set_key(self, arg1, arg2, val): 109 | # redis key = wlctest_{arg1}_{arg2} 默认使用下划线连接 110 | self.db[self.key[arg1][arg2]] = val 111 | 112 | def set_key1(self, arg1, val): 113 | # redis key = wlctest_{arg1} 114 | self.db.set(self.key[arg1], val) 115 | 116 | def get_key(self, arg1, arg2): 117 | return self.db[self.key[arg1][arg2]] 118 | 119 | def get_key1(self, arg1): 120 | return self.db.get(self.key[arg1]) 121 | 122 | class TestDbModel(BaseDbModel): 123 | 124 | prefix = "wlctest" 125 | 126 | def set_key(self, arg1, arg2, val): 127 | # redis key = wlctest_{arg1}_{arg2} 默认使用下划线连接 128 | self.db[self.key[arg1][arg2]] = val 129 | 130 | def set_key1(self, arg1, val): 131 | # redis key = wlctest_{arg1} 132 | self.db.set(self.key[arg1], val) 133 | 134 | def get_key(self, arg1, arg2): 135 | return self.db[self.key[arg1][arg2]] 136 | 137 | def get_key1(self, arg1): 138 | return self.db.get(self.key[arg1]) 139 | ``` 140 | - 使用zyredis做为codis的客户端举例 141 | ```python 142 | import zyredis 143 | 144 | db = zyredis.Client(host='localhost', port=6389, serializer=JSON(), transaction=True) # transaction=True使用zyredis当做原生redis的client使用 145 | codis_db = zyredis.Client(host='localhost', port=6389, serializer=JSON(), transaction=False) # transaction=False使用zyredis 当做codis的redis client使用，最大区别是对pipeline事务支持以及codis本身部分api不支持时日志输出 146 | 147 | db.set('key', "mydata") 148 | codis_db.set('codis_db', 'codis_data') 149 | 150 | db.Dict("test_dict", {'1': 1, '2': 2}) # 对应redis的map数据结构 151 | td = Dict("test_dict") 152 | td.get("1") # will return 1 153 | td.get("2") # will return 2 154 | td['3'] = 3 # 新增字段3 155 | ``` 156 | 157 | 开源协议 158 | ------- 159 | 本软件使用FreeBSD协议开源 160 | 161 | 技术交流 162 | ----- 163 | QQ群：329474804 申请备注请说明zkredis技术交流 164 | -------------------------------------------------------------------------------- /test/test_model.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | """ 5 | Copyright (c) 2015,掌阅科技 6 | All rights reserved. 7 | 8 | 摘 要: test_model.py 9 | 创 建 者: WangLichao 10 | 创建日期: 2015-08-21 11 | """ 12 | import sys 13 | import os 14 | import time 15 | import logging.config 16 | sys.path.append(os.path.dirname(os.path.split(os.path.realpath(__file__))[0])) 17 | import unittest 18 | from zyredis.model import Model 19 | 20 | LOGGING = { 21 | 'version': 1, 22 | 'disable_existing_loggers': False, 23 | 'handlers': { 24 | 'console': { 25 | 'class': 'logging.StreamHandler', 26 | }, 27 | }, 28 | 'loggers': { 29 | 'zyredis': { 30 | 'handlers': ['console'], 31 | 'level': 'DEBUG', 32 | }, 33 | }, 34 | } 35 | 36 | logging.config.dictConfig(LOGGING) 37 | 38 | 39 | class BaseRedisModel(Model): 40 | 41 | client_name = "test_codis" 42 | db_num = 0 43 | init_from = "local" 44 | redis_conf = { 45 | 0:"redis://localhost:6389/test_codis?weight=1&transaction=1&db=0", 46 | } 47 | 48 | 49 | class BaseQconfModel(Model): 50 | 51 | client_name = "test_codis" 52 | db_num = 0 53 | init_from = "qconf" 54 | zk_path = "/test_group/service/codis" 55 | 56 | class TestQconfModel(BaseQconfModel): 57 | 58 | prefix = "wlctest" 59 | 60 | def set_key(self, arg1, arg2, val): 61 | self.db[self.key[arg1][arg2]] = val 62 | 63 | def set_key1(self, arg1, val): 64 | self.db.set(self.key[arg1], val) 65 | 66 | def get_key(self, arg1, arg2): 67 | return self.db[self.key[arg1][arg2]] 68 | 69 | def get_key1(self, arg1): 70 | return self.db.get(self.key[arg1]) 71 | 72 | 73 | 74 | class TestModel(BaseRedisModel): 75 | 76 | prefix = "wlctest" 77 | 78 | def set_key(self, arg1, arg2, val): 79 | self.db[self.key[arg1][arg2]] = val 80 | 81 | def set_key1(self, arg1, val): 82 | self.db.set(self.key[arg1], val) 83 | 84 | def get_key(self, arg1, arg2): 85 | return self.db[self.key[arg1][arg2]] 86 | 87 | def get_key1(self, arg1): 88 | return self.db.get(self.key[arg1]) 89 | 90 | class TestNoPrefixQconfModel(BaseQconfModel): 91 | 92 | def set_key(self, arg1, arg2, val): 93 | self.db[self.key[arg1][arg2]] = val 94 | 95 | def set_key1(self, arg1, val): 96 | self.db.set(self.key[arg1], val) 97 | 98 | def get_key(self, arg1, arg2): 99 | return self.db[self.key[arg1][arg2]] 100 | 101 | def get_key1(self, arg1): 102 | return self.db.get(self.key[arg1]) 103 | 104 | 105 | class TestNoPrefixModel(BaseRedisModel): 106 | 107 | def set_key(self, arg1, arg2, val): 108 | self.db[self.key[arg1][arg2]] = val 109 | 110 | def set_key1(self, arg1, val): 111 | self.db.set(self.key[arg1], val) 112 | 113 | def get_key(self, arg1, arg2): 114 | return self.db[self.key[arg1][arg2]] 115 | 116 | def get_key1(self, arg1): 117 | return self.db.get(self.key[arg1]) 118 | 119 | 120 | class TestRedisModel(unittest.TestCase): 121 | 122 | """测试redis manager基本用法 123 | """ 124 | 125 | def test_redis_basic(self): 126 | """测试基本操作 127 | """ 128 | i = 0 129 | tb = TestModel() 130 | while i < 1: 131 | tb.set_key("abc", "def", "value") 132 | assert tb.key["abc"]["def"] == "wlctest_abc_def" 133 | assert tb.get_key("abc", "def") == "value" 134 | i = i + 1 135 | tb.set_key1("ddd", "newvalue") 136 | assert tb.key["ddd"] == "wlctest_ddd" 137 | assert tb.get_key1("ddd") == "newvalue" 138 | # 无前缀key测试 139 | tb_noprefix = TestNoPrefixModel() 140 | tb_noprefix.set_key("abc", "def", "value") 141 | assert tb_noprefix.key["abc"]["def"] == "abc_def" 142 | assert tb_noprefix.get_key("abc", "def") == "value" 143 | tb_noprefix.set_key1("ddd", "newvalue") 144 | assert tb_noprefix.key["ddd"] == "ddd" 145 | assert tb_noprefix.get_key1("ddd") == "newvalue" 146 | 147 | def test_qconf_basic(self): 148 | """测试基本操作 149 | """ 150 | i = 0 151 | tb = TestQconfModel() 152 | while i < 1: 153 | tb.set_key("abc", "def", "value") 154 | assert tb.key["abc"]["def"] == "wlctest_abc_def" 155 | assert tb.get_key("abc", "def") == "value" 156 | i = i + 1 157 | tb.set_key1("ddd", "newvalue") 158 | assert tb.key["ddd"] == "wlctest_ddd" 159 | assert tb.get_key1("ddd") == "newvalue" 160 | # 无前缀key测试 161 | tb_noprefix = TestNoPrefixModel() 162 | tb_noprefix.set_key("abc", "def", "value") 163 | assert tb_noprefix.key["abc"]["def"] == "abc_def" 164 | assert tb_noprefix.get_key("abc", "def") == "value" 165 | tb_noprefix.set_key1("ddd", "newvalue") 166 | assert tb_noprefix.key["ddd"] == "ddd" 167 | assert tb_noprefix.get_key1("ddd") == "newvalue" 168 | 169 | if __name__ == '__main__': 170 | unittest.main() 171 | -------------------------------------------------------------------------------- /zyredis/client.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | """ 5 | Copyright (c) 2014,掌阅科技 6 | All rights reserved. 7 | 8 | 摘 要: client.py 9 | 创 建 者: WangLichao 10 | 创建日期: 2014-10-24 11 | """ 12 | # pylint: disable=invalid-name,superfluous-parens 13 | import functools 14 | # third 15 | from redis import Redis as _RedisClient 16 | from redis.exceptions import ResponseError 17 | # self 18 | from zyredis import types 19 | from zyredis.utils import mkey 20 | from zyredis.utils import LOG 21 | from zyredis.serialization import JSON 22 | from zyredis.exceptions import NotSupportCommandError 23 | 24 | # codis不支持的命令 25 | NOT_SUPPORT_COMMANDS = frozenset([ 26 | 'KEYS', 'MOVE', 'OBJECT', 'RENAME', 'RENAMENX', 'SORT', 'SCAN', 27 | 'BITOP', 'MSETNX', 'BLPOP', 'BRPOP', 'BRPOPLPUSH', 'PSUBSCRIBE', 'PUBLISH', 28 | 'PUNSUBSCRIBE', 'SUBSCRIBE', 'UNSUBSCRIBE', 'DISCARD', 'EXEC', 'MULTI', 29 | 'UNWATCH', 'WATCH', 'AUTH', 'ECHO', 'SELECT', 'BGREWRITEAOF', 'BGSAVE', 30 | 'DBSIZE', 'FLUSHALL', 'FLUSHDB', 'INFO', 'LASTSAVE', 'MONITOR', 'SAVE', 31 | 'SHUTDOWN', 'SLAVEOF', 'SLOWLOG', 'SYNC', 'TIME' 32 | ]) 33 | 34 | class Client(object): 35 | 36 | """Redis Client 37 | Attributes: 38 | host: redis 服务器地址,默认localhost 39 | port: redis 端口,默认6379 40 | db: 数据库名称 41 | serializer: 使用的序列化方式 42 | """ 43 | 44 | host = "localhost" 45 | port = 6379 46 | db = 0 47 | serializer = JSON() 48 | 49 | def __init__(self, host=None, port=None, db=0, 50 | serializer=None, transaction=False, **kwargs): 51 | self.host = host or self.host 52 | self.port = port or self.port 53 | self.transaction = transaction # codis不支持事务机制 54 | self.serializer = serializer or self.serializer 55 | self.db = db or self.db 56 | # 将redis的超时设置统一处理 57 | socket_timeout = kwargs.get("socket_timeout", 1) 58 | socket_connect_timeout = kwargs.get("socket_connect_timeout", 1) 59 | # 使用长连接 60 | socket_keepalive = kwargs.get("socket_keepalive", True) 61 | retry_on_timeout = kwargs.get("retry_on_timeout", True) 62 | self.api = _RedisClient(self.host, 63 | self.port, 64 | self.db, 65 | socket_timeout=socket_timeout, 66 | socket_connect_timeout=socket_connect_timeout, 67 | socket_keepalive=socket_keepalive, 68 | retry_on_timeout=retry_on_timeout, 69 | connection_pool=None, 70 | **kwargs) 71 | 72 | def List(self, name, initial=None, auto_slice=True): 73 | """list 类型 74 | Args: 75 | name: redis key 76 | initial: 初始化值(list类型) 77 | Returns: 78 | 返回支持Python list操作的对象 79 | """ 80 | return types.List(name, self.api, initial=initial, auto_slice=auto_slice) 81 | 82 | def Set(self, name, initial=None): 83 | """set 类型 84 | Args: 85 | name: redis key 86 | initial: 初始化值(set类型) 87 | Returns: 88 | 返回支持Python set操作的对象 89 | """ 90 | return types.Set(name, self.api, initial) 91 | 92 | def SortedSet(self, name, initial=None, auto_slice=True): 93 | """sorted set类型 94 | Args: 95 | name: redis key 96 | initial: 初始化值(ZSet类型,自定义类型) 97 | Returns: 98 | 返回SortedSet对象 99 | """ 100 | return types.SortedSet(name, self.api, initial, auto_slice=auto_slice) 101 | 102 | def Dict(self, name, initial=None, auto_slice=True, **extra): 103 | """Hash 类型 104 | Args: 105 | name: redis key 106 | initial: 初始化值(dict类型,自定义类型) 107 | Returns: 108 | 返回支持python dict操作的对象 109 | """ 110 | return types.Dict(name, self.api, initial=initial, auto_slice=auto_slice, **extra) 111 | 112 | def Queue(self, name, initial=None, maxsize=None): 113 | """queue队列 114 | Args: 115 | name: The name of the queue. 116 | initial: Initial items in the queue. 117 | """ 118 | return types.Queue(name, self.api, initial=initial, maxsize=maxsize) 119 | 120 | def LifoQueue(self, name, initial=None, maxsize=None): 121 | """后进先出队列类型，部分命令codis不支持 122 | Args: 123 | name: string The name of the queue. 124 | initial: Initial items in the queue. 125 | """ 126 | return types.LifoQueue(name, self.api, 127 | initial=initial, maxsize=maxsize) 128 | 129 | def prepare_value(self, value): 130 | """编码 131 | Args: 132 | value: 需要编码的值 133 | """ 134 | return self.serializer.encode(value) 135 | 136 | def value_to_python(self, value): 137 | """解码""" 138 | return self.serializer.decode(value) 139 | 140 | def dbsize(self): 141 | """获取dbsize数量，改命令不支持codis 142 | """ 143 | if self.transaction: 144 | return self.api.dbsize() 145 | LOG.warning("dbsize is not supported by codis") 146 | 147 | def info(self): 148 | """获取info信息，该命令不支持codis 149 | """ 150 | if self.transaction: 151 | return self.api.info() 152 | LOG.warning("info is not supported by codis") 153 | 154 | def clear(self): 155 | """清理所有的key,该命令不支持codis 156 | """ 157 | if self.transaction: 158 | return self.api.flushdb() 159 | LOG.warning("flushdb is not supported by codis") 160 | 161 | def update(self, mapping): 162 | """使用key/values批量更新""" 163 | return self.api.mset(dict((key, self.prepare_value(value)) 164 | for key, value in mapping.items())) 165 | 166 | def rename(self, old_name, new_name): 167 | """重命名redis的key 168 | Args: 169 | old_name: 旧版本key 170 | new_name: 新key 171 | Returns: 172 | False: 如果key不存在 173 | True: key存在并且设置成功 174 | """ 175 | try: 176 | self.api.rename(mkey(old_name), mkey(new_name)) 177 | return True 178 | except ResponseError as exc: 179 | LOG.error("zyredis rename error, error info=%s", exc) 180 | return False 181 | 182 | def keys(self, pattern="*"): 183 | '''获取所有的key,不建议使用,codis本身也不支持 184 | Args: 185 | pattern: 正则 186 | Returns: 187 | list类型key的列表 188 | ''' 189 | if self.transaction: 190 | return self.api.keys(pattern) 191 | LOG.warning("keys is not supported by codis") 192 | 193 | def iterkeys(self, pattern="*"): 194 | '''使用迭代器遍历key 195 | ''' 196 | if self.transaction: 197 | return iter(self.keys(pattern)) 198 | LOG.warning("keys is not supported by codis") 199 | 200 | def iteritems(self, pattern="*"): 201 | '''使用迭代器获取所有的key和value 202 | ''' 203 | if self.transaction: 204 | for key in self.keys(pattern): 205 | yield (key, self[key]) 206 | 207 | def items(self, pattern="*"): 208 | '''获取所有的key/value对 209 | ''' 210 | if self.transaction: 211 | return list(self.iteritems(pattern)) 212 | 213 | def itervalues(self, pattern="*"): 214 | '''获取所有的value 215 | ''' 216 | if self.transaction: 217 | for key in self.keys(pattern): 218 | yield self[key] 219 | 220 | def values(self, pattern="*"): 221 | """values 222 | """ 223 | if self.transaction: 224 | return list(self.itervalues(pattern)) 225 | 226 | def pop(self, name): 227 | """Get and remove key from database (atomic).""" 228 | name = mkey(name) 229 | temp = mkey((name, "__poptmp__")) 230 | self.rename(name, temp) 231 | value = self[temp] 232 | del(self[temp]) 233 | return value 234 | 235 | def get(self, key, default=None): 236 | """Returns the value at ``key`` if present, otherwise returns 237 | ``default`` (``None`` by default.)""" 238 | try: 239 | return self[key] 240 | except KeyError: 241 | return default 242 | 243 | def expire(self, key, time): 244 | """设置key的过期时间 245 | Args: 246 | time: int or timedelta object 247 | key: string 248 | """ 249 | return self.api.expire(key, time) 250 | 251 | def expireat(self, key, when): 252 | """设置过期标记 253 | Args: 254 | key: string 255 | when: unixtime or datetime object 256 | """ 257 | return self.api.expireat(key, when) 258 | 259 | def mget(self, keys): 260 | """批量获取redis的key 261 | Args: 262 | keys: list 输入多个key 263 | Returns: 264 | 返回list,如果不存在则为None 265 | ['xxx', None, ...] 266 | """ 267 | return self.api.mget(keys) 268 | 269 | def mset(self, mapping): 270 | """批量set 271 | Args: 272 | key_value_list: list 多个key,value的list 273 | Returns: 274 | True or False 275 | """ 276 | return self.api.mset(dict((key, self.prepare_value(value)) 277 | for key, value in mapping.items())) 278 | 279 | def getset(self, key, value): 280 | """如果key存在返回key的值并设置value 281 | 如果key不存在返回空并设置value 282 | Args: 283 | key: string 284 | value: string 285 | """ 286 | return self.api.getset(key, value) 287 | 288 | def exists(self, key): 289 | """验证key是否存在 290 | """ 291 | return self.api.exists(key) 292 | 293 | def set(self, name, value): 294 | """x.set(name, value) <==> x[name] = value""" 295 | return self.api.set(mkey(name), self.prepare_value(value)) 296 | 297 | def __getitem__(self, name): 298 | """``x.__getitem__(name) <==> x[name]``""" 299 | name = mkey(name) 300 | value = self.api.get(name) 301 | if value is None: 302 | return None 303 | return self.value_to_python(value) 304 | 305 | def __setitem__(self, name, value): 306 | """``x.__setitem(name, value) <==> x[name] = value``""" 307 | return self.api.set(mkey(name), self.prepare_value(value)) 308 | 309 | def __delitem__(self, name): 310 | """``x.__delitem__(name) <==> del(x[name])`` 311 | Returns: 312 | True: delete success 313 | False: key is not exists 314 | """ 315 | name = mkey(name) 316 | if not self.api.delete(name): 317 | return False 318 | return True 319 | 320 | def __len__(self): 321 | """``x.__len__() <==> len(x)``""" 322 | if self.transaction: 323 | return self.api.dbsize() 324 | LOG.warning("dbsize is not supported by codis") 325 | 326 | def __contains__(self, name): 327 | """``x.__contains__(name) <==> name in x``""" 328 | return self.api.exists(mkey(name)) 329 | 330 | def __repr__(self): 331 | """``x.__repr__() <==> repr(x)``""" 332 | return "".format(self.host, 333 | self.port, 334 | self.db or "") 335 | 336 | 337 | def _wrap(self, method, *args, **kwargs): 338 | """对执行的命令进行验证 339 | Args: 340 | method: redis的方法 341 | """ 342 | LOG.debug("redis adapter execute method:%s, args=%s, kwargs=%s", method, args, kwargs) 343 | try: 344 | key = args[0] 345 | except IndexError: 346 | raise ValueError('method %s requires a key param as the first argument' % method) 347 | if method.upper() in NOT_SUPPORT_COMMANDS: 348 | LOG.error('%s is not supported by codis', method) 349 | raise NotSupportCommandError('method %s is not supported by codis, key=%s' % (method, key)) 350 | codis_func = getattr(self.api, method) 351 | return codis_func(*args, **kwargs) 352 | 353 | 354 | def __getattr__(self, method): 355 | """用于适配未定义的redis client的函数 356 | """ 357 | LOG.debug("in __getattr__ method") 358 | return functools.partial(self._wrap, method) 359 | 360 | def pipeline(self): 361 | '''转换为自定义pipeline 362 | ''' 363 | return Pipeline(self, self.transaction) 364 | 365 | 366 | class Pipeline(object): 367 | 368 | '''自定义pipeline 369 | ''' 370 | 371 | def __init__(self, client, transaction=False): 372 | self.codis_pipeline = client.api.pipeline(transaction=transaction) 373 | 374 | def _wrap(self, method, *args, **kwargs): 375 | '''打包pipeline的方法 376 | Args: 377 | method: pipeline要执行的方法 378 | args: pipeline执行方法的参数 379 | kwargs: pipeline字典参数 380 | ''' 381 | LOG.debug("pipeline execute method:%s, args=%s, kwargs=%s", method, args, kwargs) 382 | try: 383 | key = args[0] 384 | except: 385 | raise ValueError("'%s' requires a key param as the first argument" % method) 386 | if not self.transaction and method.upper() in NOT_SUPPORT_COMMANDS: 387 | LOG.error('%s is not supported by codis', method) 388 | raise NotSupportCommandError('method %s is not supported by codis, key=%s' % (method, key)) 389 | # 执行codis 390 | f = getattr(self.codis_pipeline, method) 391 | f(*args, **kwargs) 392 | 393 | def execute(self): 394 | '''提交pipeline执行 395 | ''' 396 | LOG.debug("pipeline execute flag") 397 | return self.codis_pipeline.execute() 398 | 399 | def __getattr__(self, method): 400 | return functools.partial(self._wrap, method) 401 | -------------------------------------------------------------------------------- /test/test_client.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | """ 5 | Copyright (c) 2015,掌阅科技 6 | All rights reserved. 7 | 8 | 摘 要: test_client.py 9 | 创 建 者: WangLichao 10 | 创建日期: 2015-08-19 11 | """ 12 | import sys 13 | import os 14 | import logging.config 15 | sys.path.append(os.path.dirname(os.path.split(os.path.realpath(__file__))[0])) 16 | import unittest 17 | import zyredis 18 | from zyredis.serialization import JSON 19 | 20 | LOGGING = { 21 | 'version': 1, 22 | 'disable_existing_loggers': False, 23 | 'handlers': { 24 | 'console': { 25 | 'class': 'logging.StreamHandler', 26 | }, 27 | }, 28 | 'loggers': { 29 | 'zyredis': { 30 | 'handlers': ['console'], 31 | 'level': 'DEBUG', 32 | }, 33 | }, 34 | } 35 | 36 | logging.config.dictConfig(LOGGING) 37 | 38 | CODIS_CLI = zyredis.Client('localhost', 6389, transaction=False) 39 | CLI = zyredis.Client('localhost', 6389, transaction=True) 40 | 41 | 42 | class TestClient(unittest.TestCase): 43 | 44 | """测试key生成逻辑 45 | """ 46 | def setUp(self): 47 | """setup 48 | """ 49 | self.db = zyredis.Client(host='localhost', port=6389, serializer=JSON(), transaction=True) 50 | self.codis_db = zyredis.Client(host='localhost', port=6389, serializer=JSON(), transaction=False) 51 | 52 | # string type 53 | def test_string(self): 54 | """test pyredis string type 55 | """ 56 | # set 57 | self.db["foo"] = 'bar' 58 | 59 | # get 60 | self.failUnlessEqual(self.db['foo'], 'bar') 61 | self.failUnlessEqual(self.db.get('foo'), 'bar') 62 | 63 | # update 64 | self.db.update({'company': 'ZhangYue', 'product': 'iReader'}) 65 | self.failUnlessEqual(self.db.get('company'), 'ZhangYue') 66 | self.failUnlessEqual(self.db.get('product'), 'iReader') 67 | 68 | # mset 69 | self.db.mset({'wanglichao': 1, 'wlc': 2}) 70 | 71 | # mget 72 | data = self.db.mget(['wanglichao', 'wlc']) 73 | self.failUnlessEqual(self.db.get('wanglichao'), 1) 74 | self.failUnlessEqual(self.db.get('wlc'), 2) 75 | 76 | def test_range_by_score(self): 77 | """测试zrangebyscore问题 78 | """ 79 | ss = self.db.SortedSet(name="test_sortedset_zrangebyscore", 80 | initial=(('wlc', 1), ('ls', 2), ('wyf', 3))) 81 | assert ss.range_by_score(0, 10, 0, 1)[0] == 'wlc' 82 | assert ss.range_by_score(0, 10, 1, 1)[0] == 'ls' 83 | assert ss.range_by_score(0, 10, 2, 1)[0] == 'wyf' 84 | 85 | # db level 86 | def test_db_level(self): 87 | """pyredis db 88 | """ 89 | self.db['color'] = 'white' 90 | self.db['animal'] = 'dog' 91 | 92 | # in 93 | self.failUnlessEqual('color' in self.db, True) 94 | 95 | # rename 96 | self.db.rename('color', 'a_color') 97 | self.failUnlessEqual('color' in self.db, False) 98 | self.failUnlessEqual('a_color' in self.db, True) 99 | 100 | # pop 101 | self.failUnlessEqual(self.db.pop('a_color'), 'white') 102 | self.failUnlessEqual('a_color' in self.db, False) 103 | 104 | # delete 105 | del(self.db['animal']) 106 | self.failUnlessEqual('animal' in self.db, False) 107 | 108 | 109 | # List type 110 | def test_List(self): 111 | """test pyredis List 112 | """ 113 | try: 114 | del self.db['a_list'] # 先删除数据库中名为'a_list'的键，避免干扰。 115 | except KeyError: 116 | pass 117 | l = self.db.List("a_list") 118 | 119 | # append and pop 120 | l.append("one") 121 | l.append("two") 122 | self.failUnlessEqual(l.pop(), 'two') 123 | l.appendleft('33') 124 | self.failUnlessEqual(l.popleft(), '33') 125 | 126 | # len 127 | self.failUnlessEqual(len(l), 1) 128 | 129 | # extend 130 | l.extend(["11"]) 131 | self.failUnlessEqual(l.pop(), '11') 132 | l.extendleft(["aa"]) 133 | self.failUnlessEqual(l.popleft(), 'aa') 134 | 135 | # copy 136 | l.extend(['two', 'three']) 137 | self.failUnlessEqual(l.copy(), ['one', 'two', 'three']) 138 | 139 | # get and set 140 | self.failUnlessEqual(l[0], 'one') 141 | self.failUnlessEqual(l[-1], 'three') 142 | l[0] = 'abc' 143 | self.failUnlessEqual(l[0], 'abc') 144 | 145 | # slice 146 | self.failUnlessEqual(l[0:2], ['abc', 'two']) 147 | self.failUnlessEqual(l[:], ['abc', 'two', 'three']) 148 | 149 | # trim 150 | l.trim(0, 2) 151 | self.failUnlessEqual(l[:], ['abc', 'two']) 152 | 153 | # remove 154 | l.remove('abc') 155 | self.failUnlessEqual(l[:], ['two']) 156 | 157 | # Queue type 158 | def test_queue(self): 159 | """test pyredis Queue 160 | """ 161 | try: 162 | del self.db['a_queue'] # 先删除数据库中名为'a_queue'的键，避免干扰。 163 | except KeyError: 164 | pass 165 | q = self.db.Queue('a_queue', maxsize=10) 166 | 167 | # empty 168 | self.failUnlessEqual(q.empty(), True) 169 | 170 | # full 171 | self.failUnlessEqual(q.full(), False) 172 | 173 | # put 174 | q.put('iReader') 175 | q.put('book') 176 | 177 | # get 178 | self.failUnlessEqual(q.get(block=False), 'iReader') 179 | 180 | # qsize 181 | self.failUnlessEqual(q.qsize(), 1) 182 | 183 | # LifoQueue type 184 | def test_lifoqueue(self): 185 | try: 186 | del self.db['a_lifoqueue'] # 先删除数据库中名为'a_lifoqueue'的键，避免干扰。 187 | except: 188 | pass 189 | lifoq = self.db.LifoQueue('a_lifoqueue', maxsize=10) 190 | 191 | # empty 192 | self.failUnlessEqual(lifoq.empty(), True) 193 | 194 | # full 195 | self.failUnlessEqual(lifoq.full(), False) 196 | 197 | # put 198 | lifoq.put('iReader') 199 | lifoq.put('book') 200 | 201 | # get 202 | self.failUnlessEqual(lifoq.get(block=False), 'book') 203 | 204 | # qsize 205 | self.failUnlessEqual(lifoq.qsize(), 1) 206 | 207 | # Set type 208 | def test_set_type(self): 209 | try: 210 | del self.db['a_set'] # 先删除数据库中名为'a_set'的键，避免干扰。 211 | except: 212 | pass 213 | s = self.db.Set('a_set', {'A', 'B'}) 214 | 215 | # contains 216 | self.failUnlessEqual('A' in s, True) 217 | 218 | # len 219 | self.failUnlessEqual(len(s), 2) 220 | 221 | # copy 222 | self.failUnlessEqual(s.copy(), {'A', 'B'}) 223 | 224 | # add 225 | s.add('C') 226 | self.failUnlessEqual('C' in s, True) 227 | 228 | # remove 229 | s.remove('A') 230 | self.failUnlessEqual('A' in s, False) 231 | 232 | # pop 233 | self.failUnlessEqual(s.pop() in {'B', 'C'}, True) 234 | 235 | # update 236 | new_s = self.db.Set('b_set', {'A', 'B', 'C'}) 237 | new_s.update({'D', 'E'}) 238 | self.failUnlessEqual('D' in new_s, True) 239 | self.failUnlessEqual('E' in new_s, True) 240 | 241 | # union 242 | s1 = self.db.Set('x_set', {'A', 'B', 'C'}) 243 | s2 = self.db.Set('y_set', {'B', 'C', 'D'}) 244 | self.failUnlessEqual(s1.union(s2), {'A', 'B', 'C', 'D'}) 245 | 246 | # intersection 247 | self.failUnlessEqual(s1.intersection(s2), {'B', 'C'}) 248 | 249 | # intersection_update 250 | s1.intersection_update(s2) 251 | self.failUnlessEqual(s1.copy(), {'B', 'C'}) 252 | 253 | # difference 254 | s1 = self.db.Set('z_set', {'A', 'B', 'C'}) 255 | self.failUnlessEqual(s1.difference(s2), {'A'}) 256 | self.failUnlessEqual(s2.difference(s1), {'D'}) 257 | 258 | # difference_update 259 | s1.difference_update(s2) 260 | self.failUnlessEqual(s1.copy(), {'A'}) 261 | s2.difference_update(s1) 262 | self.failUnlessEqual(s2.copy(), {'B', 'C', 'D'}) 263 | 264 | # SortedSet type 265 | def test_sortedset(self): 266 | try: 267 | del self.db['sortedset'] # 先删除数据库中名为'sortedset'的键，避免干扰。 268 | except: 269 | pass 270 | sts = self.db.SortedSet('sortedset') 271 | 272 | # add 273 | sts.add('A', 100) 274 | sts.add('B', 90) 275 | 276 | # get 277 | self.failUnlessEqual(sts[0], ['B']) 278 | self.failUnlessEqual(list(sts), ['B', 'A']) 279 | 280 | # len 281 | self.failUnlessEqual(len(sts), 2) 282 | 283 | # remove 284 | sts.remove('A') 285 | self.failUnlessEqual('A' in sts[:], False) 286 | 287 | # discard 288 | sts.discard('B') 289 | self.failUnlessEqual('B' in sts[:], False) 290 | 291 | # update 292 | sts.update([('C', 80), ('D', 70), ('E', 60)]) 293 | self.failUnlessEqual(list(sts), ['E', 'D', 'C']) 294 | 295 | # copy 296 | self.failUnlessEqual(sts.copy(), ['E', 'D', 'C']) 297 | 298 | # revrange 299 | self.failUnlessEqual(sts.revrange(), ['C', 'D', 'E']) 300 | 301 | # score 302 | self.failUnlessEqual(sts.score('C'), 80) 303 | 304 | # increment 305 | sts.increment('C', 5) 306 | new_score = sts.score('C') 307 | self.failUnlessEqual(new_score, 85) 308 | 309 | # rank 310 | self.failUnlessEqual(sts.rank('C'), 2) 311 | self.failUnlessEqual(sts.rank('E'), 0) 312 | 313 | # revrank 314 | self.failUnlessEqual(sts.revrank('C'), 0) 315 | self.failUnlessEqual(sts.revrank('E'), 2) 316 | 317 | # range_by_score 318 | self.failUnlessEqual(sts.range_by_score(60, 85), ['E', 'D', 'C']) 319 | self.failUnlessEqual(sts.range_by_score(60, 70), ['E', 'D']) 320 | 321 | # items 322 | sts_items = sts.items() 323 | self.failUnlessEqual(sts_items, ['E', 'D', 'C']) 324 | sts_items_withscores = sts.items(withscores=True) 325 | self.failUnlessEqual(sts_items_withscores, [('E', 60.0), ('D', 70.0), ('C', 85.0)]) 326 | 327 | # Dict type 328 | def test_dict(self): 329 | try: 330 | del self.db['a_dict'] # 先删除数据库中名为'a_dict'的键，避免干扰。 331 | except KeyError: 332 | pass 333 | d = self.db.Dict('a_dict') 334 | 335 | # set and get 336 | d['a'] = 'valueA' 337 | self.failUnlessEqual(d['a'], 'valueA') 338 | res = d.get('b', 'default') 339 | self.failUnlessEqual(res, 'default') 340 | 341 | # setdefault 342 | res_a = d.setdefault('a', 'valueB') 343 | self.failUnlessEqual(res_a, 'valueA') 344 | res_b = d.setdefault('b', 'valueB') 345 | self.failUnlessEqual(res_b, 'valueB') 346 | self.failUnlessEqual(d['b'], 'valueB') 347 | 348 | # len 349 | self.failUnlessEqual(len(d), 2) 350 | 351 | # in 352 | self.failUnlessEqual('a' in d, True) 353 | 354 | # has_key 355 | self.failUnlessEqual('b' in d, True) 356 | 357 | # copy 358 | self.failUnlessEqual(d.copy(), {'a': 'valueA', 'b': 'valueB'}) 359 | 360 | # keys 361 | self.failUnlessEqual(d.keys(), ['a', 'b']) 362 | 363 | # values 364 | self.failUnlessEqual(d.values(), ['valueA', 'valueB']) 365 | 366 | # items 367 | d_items = d.items() 368 | self.failUnlessEqual(('a', 'valueA') in d_items, True) 369 | self.failUnlessEqual(('b', 'valueB') in d_items, True) 370 | 371 | # iter 372 | self.failUnlessEqual(('a', 'valueA') in d.iteritems(), True) 373 | self.failUnlessEqual('a' in d.iterkeys(), True) 374 | self.failUnlessEqual('valueA' in d.itervalues(), True) 375 | 376 | # update 377 | d.update({'c': 'valueC', 'd': 'valueD'}) 378 | self.failUnlessEqual('c' in d, True) 379 | self.failUnlessEqual('d' in d, True) 380 | 381 | # pop 382 | self.failUnlessEqual(d.pop('c'), 'valueC') 383 | self.failUnlessEqual('c' in d, False) 384 | self.failUnlessEqual(d.pop('no_exist_key', 'no_exist'), 'no_exist') 385 | 386 | # del 387 | del d['b'] 388 | self.failUnlessEqual('b' in d, False) 389 | 390 | def test_basic(self): 391 | """测试基本操作 392 | """ 393 | CLI.set("wlc123", "www") 394 | assert CLI.get("wlc123") == "www" 395 | CLI.delete("wlc123") 396 | assert CLI.get("wlc123") is None 397 | assert CODIS_CLI.dbsize() is None 398 | assert CLI.dbsize() > 0 399 | pass 400 | 401 | def test_nodefine(self): 402 | """测试未实现的方法是否可用 403 | """ 404 | CLI.hset("wlchset", 'key', 123) 405 | assert CLI.hget("wlchset", 'key') == "123" # 注意数据类型发生变化 406 | 407 | def test_pipeline(self): 408 | """测试管道 409 | """ 410 | pipe = CLI.pipeline() 411 | pipe.set('wlc333', 333) 412 | pipe.set('wlc332', 332) 413 | pipe.execute() 414 | assert CLI.get("wlc333") == 333 415 | pipe = CLI.pipeline() 416 | pipe.delete("wlc333") 417 | pipe.delete("wlc332") 418 | assert CLI.get("wlc333") == 333 419 | pipe.execute() 420 | assert CLI.get("wlc333") is None 421 | 422 | def test_pythonic(self): 423 | """测试pythonionc方法 424 | """ 425 | dbsize = len(CLI) 426 | assert dbsize > 0 427 | del CLI["hello_world"] 428 | CLI["hello_world"] = "hello_world" 429 | assert "hello_world" in CLI 430 | 431 | 432 | def test_types_list(self): 433 | """测试列表类型List 434 | """ 435 | del CLI["wlc_list"] 436 | list_struct = CLI.List("wlc_list") 437 | list_struct.append(1) 438 | list_struct.append(2) 439 | list_struct.append(3) 440 | assert list_struct[0] == "1" # 注意数据类型不能保证,默认序列化方式为json导致 441 | assert list_struct[1] == "2" 442 | assert list_struct[2] == "3" 443 | assert len(list_struct) == 3 444 | del CLI["wlc_list"] 445 | assert "wlc_list" not in list_struct 446 | del(CLI['mylist']) # 先删除，避免存在key的情况会extend操作 447 | l = CLI.List("mylist", ["Jerry", "George"]) # 注意如果key已经存在的话，默认使用extend操作扩展 448 | # 添加另外一个列表到列表尾 449 | l.extend(["Elaine", "Kramer"]) 450 | assert l[0] == "Jerry" 451 | assert l[1] == "George" 452 | assert l[2] == "Elaine" 453 | assert l[3] == "Kramer" 454 | slice_test = l[2:4] 455 | assert slice_test[0] == "Elaine" 456 | assert slice_test[1] == "Kramer" 457 | assert l.popleft() == "Jerry" 458 | assert len(l) == 3 459 | # 验证迭代器部分逻辑 460 | it = iter(l) 461 | assert it.next() == "George" 462 | # 查找并删除 463 | l.remove("Elaine", count=1) 464 | assert "Elaine" not in l 465 | # 添加另外一个列表到列表头,遍历要添加的列表到l 466 | l.extendleft(["Args", "Soup-nazi", "Art"]) 467 | # now ``l = ['Art', 'Soup-nazi', 'Args', 'George', 'Kramer']`` 468 | assert l[0] == "Art" 469 | # 清理列表的某些字段 470 | l.trim(start=1, stop=2) 471 | # now ``l = ['Soup-nazi']`` 472 | assert len(l) == 1 473 | assert l[0] == "Soup-nazi" 474 | assert l[:] == ["Soup-nazi"] 475 | 476 | def test_types_dict(self): 477 | """测试数据类型Dict 478 | """ 479 | del CLI["test_dict"] 480 | d = CLI.Dict("test_dict", {'1': 1, '2': 2}) 481 | rd = CLI.Dict("test_dict") 482 | assert rd.get('1') == '1' # json序列化后类型变化 483 | assert rd['2'] == '2' 484 | rd['3'] = '3' 485 | assert len(rd) == 3 486 | assert rd['3'] == '3' 487 | del rd["2"] 488 | assert "2" not in rd 489 | for k, v in rd.iteritems(): 490 | assert k == v 491 | 492 | def test_types_set(self): 493 | """测试集合类型Set 494 | """ 495 | del CLI["test_set"] 496 | s = CLI.Set("test_set", {1, 3, 4, 4, 5}) 497 | rs = CLI.Set("test_set") 498 | assert len(rs) == 4 499 | assert 1 in rs 500 | assert 4 in rs 501 | 502 | if __name__ == '__main__': 503 | unittest.main() 504 | -------------------------------------------------------------------------------- /zyredis/types.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | """ 5 | Copyright (c) 2014,掌阅科技 6 | All rights reserved. 7 | 8 | 摘 要: types.py 9 | 创 建 者: WangLichao 10 | 创建日期: 2014-10-24 11 | """ 12 | import sys 13 | import bisect 14 | from Queue import Empty, Full 15 | 16 | from redis.exceptions import ResponseError 17 | from zyredis.utils import mkey 18 | from zyredis.utils import sublist 19 | 20 | MAX_INT = sys.maxint 21 | 22 | 23 | class Type(object): 24 | 25 | """Base-class for Redis datatypes.""" 26 | 27 | def __init__(self, name, client): 28 | self.name = mkey(name) 29 | self.client = client 30 | 31 | def __getindex__(self, i, j, step, length): 32 | if i is None: 33 | i = 0 if step > 0 else length - 1 34 | elif i < 0: 35 | i = length + i 36 | if j is None or j == MAX_INT: 37 | j = length if step > 0 else -1 38 | elif j < 0: 39 | j = length + j 40 | if step < 0: 41 | i, j = j + 1, i + 1 42 | return i, j 43 | 44 | 45 | class List(Type): 46 | 47 | """list""" 48 | 49 | def __init__(self, name, client, initial=None, auto_slice=True, max_slice_size=50): 50 | super(List, self).__init__(name, client) 51 | self.extend(initial or []) 52 | self.auto_slice = auto_slice 53 | self.max_slice_size = max_slice_size 54 | 55 | def __getitem__(self, index): 56 | """``x.__getitem__(index) <==> x[index]``""" 57 | if isinstance(index, slice): 58 | li_len = self.__len__() 59 | i, j = self.__getindex__(index.start, index.stop, index.step, li_len) 60 | return self.__getslice__(i, j, index.step) 61 | item = self.client.lindex(self.name, index) 62 | if item: 63 | return item 64 | raise IndexError("list index out of range") 65 | 66 | def __setitem__(self, index, value): 67 | """``x.__setitem__(index, value) <==> x[index] = value``""" 68 | try: 69 | self.client.lset(self.name, index, value) 70 | except ResponseError, exc: 71 | if "index out of range" in exc.args: 72 | raise IndexError("list assignment index out of range") 73 | raise 74 | 75 | def __len__(self): 76 | """``x.__len__() <==> len(x)``""" 77 | return self.client.llen(self.name) 78 | 79 | def __repr__(self): 80 | """``x.__repr__() <==> repr(x)``""" 81 | return repr(self._as_list()) 82 | 83 | def __iter__(self): 84 | """``x.__iter__() <==> iter(x)``""" 85 | return iter(self._as_list()) 86 | 87 | def __getslice__(self, i, j, step=1): 88 | """``x.__getslice__(start, stop) <==> x[start:stop]``""" 89 | # Redis indices are zero-based, while Python indices are 1-based. 90 | if i >= j: 91 | return [] 92 | result = self._slice(i, j) 93 | return result[::step] 94 | 95 | def _as_list(self): 96 | """as list 97 | """ 98 | slice_len = self.__len__() 99 | return self._slice(0, slice_len) 100 | 101 | def _slice(self, i, j): 102 | """执行List切片操作,i < j 103 | """ 104 | if self.auto_slice: 105 | result = [] 106 | slice_len = self.__len__() 107 | if j > slice_len: 108 | j = slice_len 109 | keys = range(i, j) 110 | slice_li = sublist(keys, self.max_slice_size) 111 | for sli in slice_li: 112 | start = sli[0] 113 | end = sli[-1] 114 | slice_li = self.client.lrange(self.name, start, end) 115 | result.extend(slice_li) 116 | return result 117 | else: 118 | return self.client.lrange(self.name, i, j - 1) 119 | 120 | def copy(self): 121 | """copy 122 | """ 123 | return self._as_list() 124 | 125 | def append(self, value): 126 | """Add ``value`` to the end of the list.""" 127 | return self.client.rpush(self.name, value) 128 | 129 | def appendleft(self, value): 130 | """Add ``value`` to the head of the list.""" 131 | return self.client.lpush(self.name, value) 132 | 133 | def trim(self, start, stop): 134 | """Trim the list to the specified range of elements.""" 135 | return self.client.ltrim(self.name, start, stop - 1) 136 | 137 | def pop(self): 138 | """Remove and return the last element of the list.""" 139 | return self.client.rpop(self.name) 140 | 141 | def popleft(self): 142 | """Remove and return the first element of the list.""" 143 | return self.client.lpop(self.name) 144 | 145 | def remove(self, value, count=1): 146 | """Remove occurences of ``value`` from the list. 147 | Args: 148 | count: Number of matching values to remove. 149 | Default is to remove a single value. 150 | """ 151 | count = self.client.lrem(self.name, value, num=count) 152 | if not count: 153 | raise ValueError("%s not in list" % value) 154 | return count 155 | 156 | def extend(self, iterable): 157 | """Append the values in ``iterable`` to this list.""" 158 | for value in iterable: 159 | self.append(value) 160 | 161 | def extendleft(self, iterable): 162 | """Add the values in ``iterable`` to the head of this list.""" 163 | for value in iterable: 164 | self.appendleft(value) 165 | 166 | 167 | class Set(Type): 168 | 169 | """set""" 170 | 171 | def __init__(self, name, client, initial=None): 172 | super(Set, self).__init__(name, client) 173 | if initial: 174 | self.update(initial) 175 | 176 | def __iter__(self): 177 | """``x.__iter__() <==> iter(x)``""" 178 | return iter(self._as_set()) 179 | 180 | def __repr__(self): 181 | """``x.__repr__() <==> repr(x)``""" 182 | return "" % (repr(list(self._as_set())), ) 183 | 184 | def __contains__(self, member): 185 | """``x.__contains__(member) <==> member in x``""" 186 | return self.client.sismember(self.name, member) 187 | 188 | def __len__(self): 189 | """``x.__len__() <==> len(x)``""" 190 | return self.client.scard(self.name) 191 | 192 | def _as_set(self): 193 | """as set 194 | """ 195 | return self.client.smembers(self.name) 196 | 197 | def copy(self): 198 | """copy 199 | """ 200 | return self._as_set() 201 | 202 | def add(self, member): 203 | """Add element to set. 204 | """ 205 | return self.client.sadd(self.name, member) 206 | 207 | def remove(self, member): 208 | """Remove element from set; it must be a member. 209 | Exceptions: 210 | KeyError: if the element is not a member. 211 | """ 212 | if not self.client.srem(self.name, member): 213 | raise KeyError(member) 214 | 215 | def pop(self): 216 | """Remove and return an arbitrary set element. 217 | Exceptions: 218 | KeyError: if the set is empty. 219 | """ 220 | member = self.client.spop(self.name) 221 | if member is not None: 222 | return member 223 | raise KeyError() 224 | 225 | def union(self, other): 226 | """Return the union of sets as a new set. 227 | """ 228 | if isinstance(other, self.__class__): 229 | return self.client.sunion([self.name, other.name]) 230 | else: 231 | return self._as_set().union(other) 232 | 233 | def update(self, other): 234 | """Update this set with the union of itself and others.""" 235 | if isinstance(other, self.__class__): 236 | return self.client.sunionstore(self.name, [self.name, other.name]) 237 | else: 238 | for member in other: 239 | self.add(member) 240 | # return map(self.add, other) # bad-buildin 241 | 242 | def intersection(self, other): 243 | """Return the intersection of two sets as a new set. 244 | """ 245 | if isinstance(other, self.__class__): 246 | return self.client.sinter([self.name, other.name]) 247 | else: 248 | return self._as_set().intersection(other) 249 | 250 | def intersection_update(self, other): 251 | """Update the set with the intersection of itself and another.""" 252 | return self.client.sinterstore(self.name, [self.name, other.name]) 253 | 254 | def difference(self, *others): 255 | """Return the difference of two or more sets as a new :class:`set`. 256 | """ 257 | if all([isinstance(a, self.__class__) for a in others]): 258 | return self.client.sdiff([self.name] + [other.name for other in others]) 259 | else: 260 | othersets = [x for x in others if isinstance(x, set)] 261 | # othersets = filter(lambda x: isinstance(x, set), others) # bad-buildin 262 | othertypes = [x for x in others if isinstance(x, self.__class__)] 263 | # othertypes = filter(lambda x: isinstance(x, self.__class__), others) # bad-buildin 264 | return self.client.sdiff([self.name] + [other.name for other in othertypes]).difference(*othersets) 265 | 266 | def difference_update(self, other): 267 | """Remove all elements of another set from this set.""" 268 | return self.client.sdiffstore(self.name, [self.name, other.name]) 269 | 270 | 271 | class SortedSet(Type): 272 | 273 | """A sorted set. 274 | Args: 275 | initial: Initial data to populate the set with, 276 | must be an iterable of ``(element, score)`` tuples. 277 | """ 278 | class _itemsview(object): 279 | 280 | """视图类 281 | """ 282 | 283 | def __init__(self, zset, start=0, end=-1, desc=False, 284 | withscores=False): 285 | self.zset = zset 286 | self.start = start 287 | self.end = end 288 | self.desc = desc 289 | self.withscores = withscores 290 | 291 | def _items(self, start, end, desc, withscores): 292 | """get all items 293 | """ 294 | return self.zset.items(start, end, desc=desc, 295 | withscores=withscores) 296 | 297 | def __iter__(self): 298 | """for iter 299 | """ 300 | return iter(self._items(self.start, self.end, self.desc, 301 | self.withscores)) 302 | 303 | def __reversed__(self): 304 | """for reverse 305 | """ 306 | return self._items(self.start, self.end, True, self.withscores) 307 | 308 | def __getitem__(self, key): 309 | if isinstance(key, slice): 310 | i = key.start or 0 311 | j = key.stop or -1 312 | j = j - 1 313 | return self._items(i, j, False, self.withscores) 314 | else: 315 | return self._items(key, key, False, self.withscores)[0] 316 | 317 | def __init__(self, name, client, initial=None, auto_slice=True, max_slice_size=50): 318 | super(SortedSet, self).__init__(name, client) 319 | if initial: 320 | self.update(initial) 321 | self.auto_slice = auto_slice 322 | self.max_slice_size = max_slice_size 323 | 324 | def __iter__(self): 325 | """``x.__iter__() <==> iter(x)``""" 326 | return iter(self._as_set()) 327 | 328 | def __getitem__(self, index): 329 | """``x.__getitem__(index) <==> x[index]``""" 330 | if isinstance(index, slice): 331 | set_len = self.__len__() 332 | i, j = self.__getindex__(index.start, index.stop, index.step, set_len) 333 | return self.__getslice__(i, j, index.step) 334 | item = self.client.zrange(self.name, index, index) 335 | if item: 336 | return item 337 | raise IndexError("sortedset index out of range") 338 | 339 | def __len__(self): 340 | """``x.__len__() <==> len(x)``""" 341 | return self.client.zcard(self.name) 342 | 343 | def __repr__(self): 344 | """``x.__repr__() <==> repr(x)``""" 345 | return "" % (repr(list(self._as_set())), ) 346 | 347 | def add(self, member, score): 348 | """Add the specified member to the sorted set, or update the score 349 | if it already exist. 350 | """ 351 | return self.client.zadd(self.name, member, score) 352 | 353 | def remove(self, member): 354 | """Remove member.""" 355 | if not self.client.zrem(self.name, member): 356 | raise KeyError(member) 357 | 358 | def revrange(self, start=0, stop=-1): 359 | """revrange 360 | """ 361 | stop = stop is None and -1 or stop 362 | return self.client.zrevrange(self.name, start, stop) 363 | 364 | def discard(self, member): 365 | """Discard member.""" 366 | self.client.zrem(self.name, member) 367 | 368 | def increment(self, member, amount=1): 369 | """Increment the score of ``member`` by ``amount``.""" 370 | return self.client.zincrby(self.name, member, amount) 371 | 372 | def rank(self, member): 373 | """Rank the set with scores being ordered from low to high.""" 374 | return self.client.zrank(self.name, member) 375 | 376 | def revrank(self, member): 377 | """Rank the set with scores being ordered from high to low.""" 378 | return self.client.zrevrank(self.name, member) 379 | 380 | def score(self, member): 381 | """Return the score associated with the specified member.""" 382 | return self.client.zscore(self.name, member) 383 | 384 | def range_by_score(self, min_, max_, start=None, num=None, withscores=False, score_cast_func=float): 385 | """Return all the elements with score >= min and score <= max 386 | (a range query) from the sorted set. 387 | """ 388 | return self.client.zrangebyscore(self.name, min_, max_, start=start, num=num, 389 | withscores=withscores, score_cast_func=score_cast_func) 390 | 391 | def update(self, iterable): 392 | """update 393 | """ 394 | # support dict type 395 | if isinstance(iterable, dict): 396 | iterable = iterable.iteritems() 397 | for member, score in iterable: 398 | self.add(member, score) 399 | 400 | def __getslice__(self, i, j, step=1): 401 | """``x.__getslice__(start, stop) <==> x[start:stop]``""" 402 | if i >= j: 403 | return [] 404 | result = self.zrange_slice(i, j) 405 | return result[::step] 406 | 407 | def _as_set(self): 408 | """使用zrange 409 | """ 410 | slice_len = self.__len__() 411 | return self.zrange_slice(0, slice_len) 412 | 413 | def items(self, start=0, end=-1, desc=False, withscores=False): 414 | """items 415 | """ 416 | return self.zrange_slice(start, end, desc=desc, withscores=withscores) 417 | 418 | def zrange_slice(self, start=0, end=-1, desc=False, withscores=False): 419 | """切片后使用zrange 420 | """ 421 | if self.auto_slice: 422 | result = [] 423 | set_len = self.__len__() 424 | if end < 0: 425 | end = set_len + end + 1 426 | if end > set_len: 427 | end = set_len 428 | keys = range(start, end) 429 | slice_li = sublist(keys, self.max_slice_size) 430 | for sli in slice_li: 431 | i = sli[0] 432 | j = sli[-1] 433 | slice_li = self.client.zrange(self.name, i, j, desc=desc, withscores=withscores) 434 | result.extend(slice_li) 435 | return result 436 | else: 437 | return self.client.zrange(self.name, start, end - 1, desc=desc, withscores=withscores) 438 | 439 | def itemsview(self, start=0, end=-1, desc=False): 440 | """itemsview 441 | """ 442 | return self._itemsview(self, start, end, desc, withscores=True) 443 | 444 | def keysview(self, start=0, end=-1, desc=False): 445 | """keysview 446 | """ 447 | return self._itemsview(self, start, end, desc, withscores=False) 448 | 449 | def copy(self): 450 | """复制 451 | """ 452 | return self._as_set() 453 | 454 | def count(self, minnum, maxnum): 455 | """获取指定分数范围内的key个数（包含界定分数） 456 | minnum可以指定为'-inf'表示最小分数 457 | maxnum可以指定为'+inf'表示最大分数 458 | """ 459 | return self.client.zcount(self.name, minnum, maxnum) 460 | 461 | 462 | class Dict(Type): 463 | 464 | """A dictionary.""" 465 | 466 | def __init__(self, name, client, initial=None, auto_slice=True, max_slice_size=50, **extra): 467 | super(Dict, self).__init__(name, client) 468 | initial = dict(initial or {}, **extra) 469 | if initial: 470 | self.update(initial) 471 | self.auto_slice = auto_slice 472 | self.max_slice_size = max_slice_size 473 | 474 | def __getitem__(self, key): 475 | """``x.__getitem__(key) <==> x[key]``""" 476 | value = self.client.hget(self.name, key) 477 | if value is not None: 478 | return value 479 | if hasattr(self.__class__, "__missing__"): 480 | return self.__class__.__missing__(self, key) 481 | raise KeyError(key) 482 | 483 | def __setitem__(self, key, value): 484 | """``x.__setitem__(key, value) <==> x[key] = value``""" 485 | return self.client.hset(self.name, key, value) 486 | 487 | def __delitem__(self, key): 488 | """``x.__delitem__(key) <==> del(x[key])``""" 489 | if not self.client.hdel(self.name, key): 490 | raise KeyError(key) 491 | 492 | def __contains__(self, key): 493 | """``x.__contains__(key) <==> key in x``""" 494 | return self.client.hexists(self.name, key) 495 | 496 | def __len__(self): 497 | """``x.__len__() <==> len(x)``""" 498 | return self.client.hlen(self.name) 499 | 500 | def __iter__(self): 501 | """``x.__iter__() <==> iter(x)``""" 502 | return self.iteritems() 503 | 504 | def __repr__(self): 505 | """``x.__repr__() <==> repr(x)``""" 506 | return repr(self._as_dict()) 507 | 508 | def keys(self): 509 | """Returns the list of keys present in this dictionary.""" 510 | return self.client.hkeys(self.name) 511 | 512 | def values(self): 513 | """Returns the list of values present in this dictionary.""" 514 | return self.client.hvals(self.name) 515 | 516 | def items(self): 517 | """This dictionary as a list of ``(key, value)`` pairs, as 518 | 2-tuples. 519 | """ 520 | return self._as_dict().items() 521 | 522 | def iteritems(self): 523 | """Returns an iterator over the ``(key, value)`` items present in this 524 | dictionary. 525 | """ 526 | return iter(self.items()) 527 | 528 | def iterkeys(self): 529 | """Returns an iterator over the keys present in this dictionary.""" 530 | return iter(self.keys()) 531 | 532 | def itervalues(self): 533 | """Returns an iterator over the values present in this dictionary.""" 534 | return iter(self.values()) 535 | 536 | def has_key(self, key): 537 | """Returns ``True`` if ``key`` is present in this dictionary, 538 | ``False`` otherwise. 539 | """ 540 | return key in self 541 | 542 | def get(self, key, default=None): 543 | """Returns the value at ``key`` if present, otherwise returns 544 | ``default`` (``None`` by default.) 545 | """ 546 | try: 547 | return self[key] 548 | except KeyError: 549 | return default 550 | 551 | def mget(self, keys): 552 | """返回一个或多个给定域的值 553 | Args: 554 | keys: list 需获取值的key列表 555 | Returns: 556 | 值的列表,顺序与传入参数顺序一致 557 | """ 558 | if self.auto_slice and isinstance(keys, list): 559 | result = [] 560 | slice_list = sublist(keys, self.max_slice_size) 561 | for sli in slice_list: 562 | slice_li = self.client.hmget(self.name, sli) 563 | result.extend(slice_li) 564 | return result 565 | else: 566 | return self.client.hmget(self.name, keys) 567 | 568 | def setdefault(self, key, default=None): 569 | """Returns the value at ``key`` if present, otherwise 570 | stores ``default`` value at ``key``. 571 | """ 572 | try: 573 | return self[key] 574 | except KeyError: 575 | self[key] = default 576 | return default 577 | 578 | def pop(self, key, *args, **kwargs): 579 | """Remove specified key and return the corresponding value. 580 | Args: 581 | default: If key is not found, ``default`` is returned if given, 582 | otherwise :exc:`KeyError` is raised. 583 | """ 584 | try: 585 | val = self[key] 586 | except KeyError: 587 | if len(args): 588 | return args[0] 589 | if "default" in kwargs: 590 | return kwargs["default"] 591 | raise 592 | 593 | try: 594 | del self[key] 595 | except KeyError: 596 | pass 597 | 598 | return val 599 | 600 | def update(self, other): 601 | """Update this dictionary with another.""" 602 | return self.client.hmset(self.name, other) 603 | 604 | def _as_dict(self): 605 | """使用hgetall 606 | """ 607 | return self.client.hgetall(self.name) 608 | 609 | def copy(self): 610 | """复制 611 | """ 612 | return self._as_dict() 613 | 614 | 615 | class Queue(Type): 616 | 617 | """FIFO Queue.""" 618 | 619 | Empty = Empty 620 | Full = Full 621 | 622 | def __init__(self, name, client, initial=None, maxsize=0): 623 | super(Queue, self).__init__(name, client) 624 | self.list = List(name, client, initial) 625 | self.maxsize = maxsize 626 | self._pop = self.list.pop 627 | self._bpop = self.client.brpop 628 | self._append = self.list.appendleft 629 | 630 | def empty(self): 631 | """Return ``True`` if the queue is empty, or ``False`` 632 | otherwise (not reliable!).""" 633 | return not len(self.list) 634 | 635 | def full(self): 636 | """Return ``True`` if the queue is full, ``False`` 637 | otherwise (not reliable!). 638 | 639 | Only applicable if :attr:`maxsize` is set. 640 | 641 | """ 642 | return self.maxsize and len(self.list) >= self.maxsize or False 643 | 644 | def get(self, block=True, timeout=None): 645 | """Remove and return an item from the queue. 646 | 647 | If optional args ``block`` is ``True`` and ``timeout`` is 648 | ``None`` (the default), block if necessary until an item is 649 | available. If ``timeout`` is a positive number, it blocks at 650 | most ``timeout`` seconds and raises the :exc:`Queue.Empty` exception 651 | if no item was available within that time. Otherwise (``block`` is 652 | ``False``), return an item if one is immediately available, else 653 | raise the :exc:`Queue.Empty` exception (``timeout`` is ignored 654 | in that case). 655 | """ 656 | if not block: 657 | return self.get_nowait() 658 | item = self._bpop([self.name], timeout=timeout) 659 | if item is not None: 660 | return item 661 | raise Empty 662 | 663 | def get_nowait(self): 664 | """Remove and return an item from the queue without blocking. 665 | Exceptions: 666 | Queue.Empty: if an item is not immediately available. 667 | """ 668 | item = self._pop() 669 | if item is not None: 670 | return item 671 | raise Empty() 672 | 673 | def put(self, item): 674 | """Put an item into the queue.""" 675 | if self.full(): 676 | raise Full() 677 | self._append(item) 678 | 679 | def qsize(self): 680 | """Returns the current size of the queue.""" 681 | return len(self.list) 682 | 683 | 684 | class LifoQueue(Queue): 685 | 686 | """Variant of :class:`Queue` that retrieves most recently added 687 | entries first.""" 688 | 689 | def __init__(self, name, client, initial=None, maxsize=0): 690 | super(LifoQueue, self).__init__(name, client, initial, maxsize) 691 | self._pop = self.list.popleft 692 | self._bpop = self.client.blpop 693 | 694 | 695 | class Int(Type): 696 | 697 | """实现对int类型的基础操作 698 | """ 699 | 700 | def __add__(self, other): 701 | return type(other).__radd__(other, self.__int__()) 702 | 703 | def __sub__(self, other): 704 | return type(other).__rsub__(other, self.__int__()) 705 | 706 | def __cmp__(self, other): 707 | return cmp(self.__int__(), other) 708 | 709 | def __mul__(self, other): 710 | return type(other).__rmul__(other, self.__int__()) 711 | 712 | def __div__(self, other): 713 | return type(other).__rdiv__(other, self.__int__()) 714 | 715 | def __truediv__(self, other): 716 | return type(other).__rtruediv__(other, self.__int__()) 717 | 718 | def __floordiv__(self, other): 719 | return type(other).__rfloordiv__(other, self.__int__()) 720 | 721 | def __mod__(self, other): 722 | return type(other).__rmod__(other, self.__int__()) 723 | 724 | def __divmod__(self, other): 725 | return type(other).__rdivmod__(other, self.__int__()) 726 | 727 | def __pow__(self, other): 728 | return type(other).__rpow__(other, self.__int__()) 729 | 730 | def __lshift__(self, other): 731 | return type(other).__rlshift__(other, self.__int__()) 732 | 733 | def __rshift__(self, other): 734 | return type(other).__rrshift__(other, self.__int__()) 735 | 736 | def __and__(self, other): 737 | return type(other).__rand__(other, self.__int__()) 738 | 739 | def __or__(self, other): 740 | return type(other).__ror__(other, self.__int__()) 741 | 742 | def __xor__(self, other): 743 | return type(other).__rxor__(other, self.__int__()) 744 | 745 | def __radd__(self, other): 746 | return type(other).__add__(other, self.__int__()) 747 | 748 | def __rsub__(self, other): 749 | return type(other).__sub__(other, self.__int__()) 750 | 751 | def __rmul__(self, other): 752 | return type(other).__mul__(other, self.__int__()) 753 | 754 | def __rdiv__(self, other): 755 | return type(other).__div__(other, self.__int__()) 756 | 757 | def __rtruediv__(self, other): 758 | return type(other).__truediv__(other, self.__int__()) 759 | 760 | def __rfloordiv__(self, other): 761 | return type(other).__floordiv__(other, self.__int__()) 762 | 763 | def __rmod__(self, other): 764 | return type(other).__mod__(other, self.__int__()) 765 | 766 | def __rdivmod__(self, other): 767 | return type(other).__divmod__(other, self.__int__()) 768 | 769 | def __rpow__(self, other): 770 | return type(other).__pow__(other, self.__int__()) 771 | 772 | def __rlshift__(self, other): 773 | return type(other).__lshift__(other, self.__int__()) 774 | 775 | def __rrshift__(self, other): 776 | return type(other).__shift__(other, self.__int__()) 777 | 778 | def __rand__(self, other): 779 | return type(other).__and__(other, self.__int__()) 780 | 781 | def __ror__(self, other): 782 | return type(other).__or__(other, self.__int__()) 783 | 784 | def __rxor__(self, other): 785 | return type(other).__xor__(other, self.__int__()) 786 | 787 | def __iadd__(self, other): 788 | self.client.incr(self.name, other) 789 | return self 790 | 791 | def __isub__(self, other): 792 | self.client.decr(self.name, other) 793 | return self 794 | 795 | def __imul__(self, other): 796 | self.client.set(self.name, int.__mul__(self.__int__(), other)) 797 | return self 798 | 799 | def __idiv__(self, other): 800 | self.client.set(self.name, int.__div__(self.__int__(), other)) 801 | return self 802 | 803 | def __itruediv__(self, other): 804 | self.client.set(self.name, int.__truediv__(self.__int__(), other)) 805 | return self 806 | 807 | def __ifloordiv__(self, other): 808 | self.client.set(self.name, int.__floordiv__(self.__int__(), other)) 809 | return self 810 | 811 | def __imod__(self, other): 812 | self.client.set(self.name, int.__mod__(self.__int__(), other)) 813 | return self 814 | 815 | def __ipow__(self, other): 816 | self.client.set(self.name, int.__pow__(self.__int__(), other)) 817 | return self 818 | 819 | def __iand__(self, other): 820 | self.client.set(self.name, int.__and__(self.__int__(), other)) 821 | return self 822 | 823 | def __ior__(self, other): 824 | self.client.set(self.name, int.__or__(self.__int__(), other)) 825 | return self 826 | 827 | def __ixor__(self, other): 828 | self.client.set(self.name, int.__xor__(self.__int__(), other)) 829 | return self 830 | 831 | def __ilshift__(self, other): 832 | self.client.set(self.name, int.__lshift__(self.__int__(), other)) 833 | return self 834 | 835 | def __irshift__(self, other): 836 | self.client.set(self.name, int.__rshift__(self.__int__(), other)) 837 | return self 838 | 839 | def __neg__(self): 840 | return int.__neg__(self.__int__()) 841 | 842 | def __pos__(self): 843 | return int.__pos__(self.__int__()) 844 | 845 | def __abs__(self): 846 | return int.__abs__(self.__int__()) 847 | 848 | def __invert__(self): 849 | return int.__invert__(self.__int__()) 850 | 851 | def __long__(self): 852 | return int.__long__(self.__int__()) 853 | 854 | def __float__(self): 855 | return int.__float__(self.__int__()) 856 | 857 | def __complex__(self): 858 | return int.__complex__(self.__int__()) 859 | 860 | def __int__(self): 861 | return int(self.client.get(self.name)) 862 | 863 | def __repr__(self): 864 | return repr(int(self)) 865 | 866 | def copy(self): 867 | """get real item 868 | """ 869 | return self.__int__() 870 | 871 | 872 | class ZSet(object): 873 | 874 | """The simplest local implementation to Redis's Sorted Set imaginable. 875 | Little thought given to performance, simply get the basic implementation 876 | right.""" 877 | 878 | def __init__(self, initial=None): 879 | if not self.is_zsettable(initial): 880 | raise ValueError(initial) 881 | self._dict = initial 882 | 883 | @staticmethod 884 | def is_zsettable(initial): 885 | """quick check that all values in a dict are reals""" 886 | return all([x for x in initial.values() if isinstance(x, (int, float, long))]) 887 | # return all(map(lambda x: isinstance(x, (int, float, long)), initial.values())) # bad-buildin 888 | 889 | def items(self): 890 | """items 891 | """ 892 | return sorted(self._dict.items(), key=lambda x: (x[1], x[0])) 893 | 894 | def __getitem__(self, key): 895 | """``x.__getitem__('key') <==> x['key'] `` 896 | """ 897 | return self._as_set()[key] 898 | 899 | def __len__(self): 900 | """``x.__len__() <==> len(x)``""" 901 | return len(self._dict) 902 | 903 | def __iter__(self): 904 | """``x.__iter__() <==> iter(x)``""" 905 | return iter(self._as_set()) 906 | 907 | def __repr__(self): 908 | """``x.__repr__() <==> repr(x)``""" 909 | return repr(self._as_set()) 910 | 911 | def add(self, member, score): 912 | """Add the specified member to the sorted set, or update the score 913 | if it already exist.""" 914 | self._dict[member] = score 915 | 916 | def remove(self, member): 917 | """Remove member.""" 918 | del self._dict[member] 919 | 920 | def discard(self, member): 921 | """discard 922 | """ 923 | if member in self._dict: 924 | del self._dict[member] 925 | 926 | def increment(self, member, amount=1): 927 | """Increment the score of ``member`` by ``amount``.""" 928 | self._dict[member] += amount 929 | return self._dict[member] 930 | 931 | def rank(self, member): 932 | """Rank the set with scores being ordered from low to high.""" 933 | return self._as_set().index(member) 934 | 935 | def revrank(self, member): 936 | """Rank the set with scores being ordered from high to low.""" 937 | return self.__len__() - self.rank(member) - 1 938 | 939 | def score(self, member): 940 | """Return the score associated with the specified member.""" 941 | return self._dict[member] 942 | 943 | def range_by_score(self, min_, max_): 944 | """Return all the elements with score >= min and score <= max 945 | (a range query) from the sorted set.""" 946 | data = self.items() 947 | keys = [r[1] for r in data] 948 | start = bisect.bisect_left(keys, min_) 949 | end = bisect.bisect_right(keys, max_, start) 950 | return self._as_set()[start:end] 951 | 952 | def _as_set(self): 953 | """_as_set 954 | """ 955 | return [x[0] for x in self.items()] 956 | # return map(lambda x: x[0], self.items()) # bad-buildin 957 | --------------------------------------------------------------------------------