├── .gitignore ├── AUTHORS ├── COPYING ├── ChangeLog ├── Makefile ├── README.md ├── benchmark.py ├── cmemcached.py ├── cmemcached_imp.pyx ├── patches ├── 0.40 │ ├── buffer_requests.patch │ ├── empty_string.patch │ └── fail_over.patch └── 1.0 │ ├── behavior.patch │ ├── empty_string.patch │ └── touch_expire-1.0.patch ├── requirements.txt ├── setup.cfg ├── setup.py ├── split_mc.c ├── split_mc.h ├── t ├── benchmark.py ├── node_failure_test.py ├── pickle_perf.py ├── set_get_pressure_test.py ├── test_key_mapping.py └── test_server_down.py └── tests ├── conftest.py ├── test_cmemcached.py ├── test_cmemcached_imp.py ├── test_logger.py ├── test_multi_return_failure.py ├── test_prefix.py └── test_set_long.py /.gitignore: -------------------------------------------------------------------------------- 1 | cmemcached_imp.c 2 | build/* 3 | cmemcached.pyc 4 | cmemcached_imp.so 5 | dist/* 6 | python_libmemcached.egg-info/* 7 | *.py[co] 8 | *.sw[po] 9 | venv/* 10 | *.eggs 11 | *.egg 12 | -------------------------------------------------------------------------------- /AUTHORS: -------------------------------------------------------------------------------- 1 | davies davies.liu#gmail.com -- current maintainer 2 | hongqn hongqn#gmail.com -- original wrapper for the libmemcache 3 | subdragon subdragon#gmail.com -- wrapper for the libmemcached 4 | -------------------------------------------------------------------------------- /COPYING: -------------------------------------------------------------------------------- 1 | Copyright (c) 2008, douban.com (hongqn,subdragon) 2 | All rights reserved. 3 | 4 | Redistribution and use in source and binary forms, with or without 5 | modification, are permitted provided that the following conditions are 6 | met: 7 | 8 | * Redistributions of source code must retain the above copyright 9 | notice, this list of conditions and the following disclaimer. 10 | 11 | * Redistributions in binary form must reproduce the above 12 | copyright notice, this list of conditions and the following disclaimer 13 | in the documentation and/or other materials provided with the 14 | distribution. 15 | 16 | * Neither the name of douban.com nor the names of its 17 | contributors may be used to endorse or promote products derived from 18 | this software without specific prior written permission. 19 | 20 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 21 | "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 22 | LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 23 | A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 24 | OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 25 | SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 26 | LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 27 | DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 28 | THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 29 | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 30 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 31 | -------------------------------------------------------------------------------- /ChangeLog: -------------------------------------------------------------------------------- 1 | 0.40 Mon Mar 15 13:36:00 CST 2011 2 | * update to libmemcached-0.40 3 | * support large item (>=1M), auto splitting and merging 4 | * set_multi, prepend_multi, delete_multi 5 | * nonblock by default 6 | * fail-over patch for libmemcached-0.40 7 | * empty string patch for libmemcached-0.40 8 | * configable item compression and decompression when needed 9 | * use marshal prior to cPickle when serializing. 10 | 11 | 0.23 Mon Aug 25 16:57:59 CST 2008 12 | * Support the libmemcached 0.23 version 13 | * server rehash when one server is down 14 | * weighted ketama 15 | * cas support and fix incr and decr bug 16 | 17 | 0.14 Sun Mar 16 22:03:10 CST 2008 18 | * Frist douban version 19 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | test: 2 | python setup.py test 3 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # python-libmemcached 2 | 3 | **THE PROJECT IS DEPRECATED**, please refer to https://github.com/douban/libmc instead 4 | 5 | 1.It needs libmemcached >=1.0.2 (touch support), patched with behavior.patch, empty_string.patch, touch_expire.patch 6 | 2.If you use the python whose version is < 2.5, you should modify the Py_ssize_t definition. 7 | -------------------------------------------------------------------------------- /benchmark.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | import time 4 | import random 5 | import sys 6 | 7 | 8 | options = None 9 | total_time = None 10 | 11 | def run_test(func, name): 12 | sys.stdout.write(name + ': ') 13 | sys.stdout.flush() 14 | start_time = time.time() 15 | try: 16 | func() 17 | except: 18 | print "failed or not supported" 19 | global options 20 | if options.verbose: 21 | import traceback; traceback.print_exc() 22 | else: 23 | end_time = time.time() 24 | global total_time 25 | total_time += end_time - start_time 26 | print "%f seconds" % (end_time - start_time) 27 | 28 | 29 | class BigObject(object): 30 | def __init__(self, letter='1', size=10000): 31 | self.object = letter * size 32 | 33 | def __eq__(self, other): 34 | return self.object == other.object 35 | 36 | 37 | class Benchmark(object): 38 | def __init__(self, module, options): 39 | self.module = module 40 | self.options = options 41 | self.init_server() 42 | self.test_set() 43 | self.test_set_get() 44 | self.test_random_get() 45 | self.test_set_same() 46 | self.test_set_big_object() 47 | self.test_set_get_big_object() 48 | self.test_set_big_string() 49 | self.test_set_get_big_string() 50 | self.test_get() 51 | self.test_get_big_object() 52 | self.test_get_multi() 53 | self.test_get_list() 54 | 55 | def init_server(self): 56 | #self.mc = self.module.Client([self.options.server_address]) 57 | self.mc = self.module.Client(["faramir:11217"]) 58 | self.mc.set_behavior(self.module.BEHAVIOR_BINARY_PROTOCOL, 1) 59 | self.mc.set('bench_key', "E" * 50) 60 | 61 | num_tests = self.options.num_tests 62 | self.keys = ['key%d' % i for i in xrange(num_tests)] 63 | self.values = ['value%d' % i for i in xrange(num_tests)] 64 | self.random_keys = ['key%d' % random.randint(0, num_tests) for i in xrange(num_tests * 3)] 65 | 66 | def test_set(self): 67 | set_ = self.mc.set 68 | pairs = zip(self.keys, self.values) 69 | 70 | def test(): 71 | for key, value in pairs: 72 | set_(key, value) 73 | def test_loop(): 74 | for i in range(10): 75 | for key, value in pairs: 76 | set_(key, value) 77 | run_test(test, 'test_set') 78 | 79 | for key, value in pairs: 80 | self.mc.delete(key) 81 | 82 | def test_set_get(self): 83 | set_ = self.mc.set 84 | get_ = self.mc.get 85 | pairs = zip(self.keys, self.values) 86 | 87 | def test(): 88 | for key, value in pairs: 89 | set_(key, value) 90 | result = get_(key) 91 | assert result == value 92 | run_test(test, 'test_set_get') 93 | 94 | #for key, value in pairs: 95 | # self.mc.delete(key) 96 | 97 | def test_random_get(self): 98 | get_ = self.mc.get 99 | set_ = self.mc.set 100 | 101 | value = "chenyin" 102 | 103 | def test(): 104 | index = 0 105 | for key in self.random_keys: 106 | result = get_(key) 107 | index += 1 108 | if(index % 5 == 0): 109 | set_(key, value) 110 | run_test(test, 'test_random_get') 111 | 112 | def test_set_same(self): 113 | set_ = self.mc.set 114 | 115 | def test(): 116 | for i in xrange(self.options.num_tests): 117 | set_('key', 'value') 118 | def test_loop(): 119 | for i in range(10): 120 | for i in xrange(self.options.num_tests): 121 | set_('key', 'value') 122 | run_test(test, 'test_set_same') 123 | 124 | self.mc.delete('key') 125 | 126 | def test_set_big_object(self): 127 | set_ = self.mc.set 128 | # libmemcached is slow to store large object, so limit the 129 | # number of objects here to make tests not stall. 130 | pairs = [('key%d' % i, BigObject()) for i in xrange(100)] 131 | 132 | def test(): 133 | for key, value in pairs: 134 | set_(key, value) 135 | 136 | run_test(test, 'test_set_big_object (100 objects)') 137 | 138 | for key, value in pairs: 139 | self.mc.delete(key) 140 | 141 | def test_set_get_big_object(self): 142 | set_ = self.mc.set 143 | get_ = self.mc.get 144 | # libmemcached is slow to store large object, so limit the 145 | # number of objects here to make tests not stall. 146 | pairs = [('key%d' % i, BigObject()) for i in xrange(100)] 147 | 148 | def test(): 149 | for key, value in pairs: 150 | set_(key, value) 151 | result = get_(key) 152 | assert result == value 153 | 154 | run_test(test, 'test_set_get_big_object (100 objects)') 155 | 156 | #for key, value in pairs: 157 | # self.mc.delete(key) 158 | 159 | def test_set_get_big_string(self): 160 | set_ = self.mc.set 161 | get_ = self.mc.get 162 | 163 | # libmemcached is slow to store large object, so limit the 164 | # number of objects here to make tests not stall. 165 | pairs = [('key%d' % i, 'x' * 10000) for i in xrange(100)] 166 | 167 | def test(): 168 | for key, value in pairs: 169 | set_(key, value) 170 | result = get_(key) 171 | assert result == value 172 | run_test(test, 'test_set_get_big_string (100 objects)') 173 | 174 | 175 | def test_set_big_string(self): 176 | set_ = self.mc.set 177 | 178 | # libmemcached is slow to store large object, so limit the 179 | # number of objects here to make tests not stall. 180 | pairs = [('key%d' % i, 'x' * 10000) for i in xrange(100)] 181 | 182 | def test(): 183 | for key, value in pairs: 184 | set_(key, value) 185 | run_test(test, 'test_set_big_string (100 objects)') 186 | 187 | for key, value in pairs: 188 | self.mc.delete(key) 189 | 190 | 191 | def test_get(self): 192 | pairs = zip(self.keys, self.values) 193 | for key, value in pairs: 194 | self.mc.set(key, value) 195 | 196 | get = self.mc.get 197 | 198 | def test(): 199 | for key, value in pairs: 200 | result = get(key) 201 | assert result == value 202 | run_test(test, 'test_get') 203 | 204 | for key, value in pairs: 205 | self.mc.delete(key) 206 | 207 | def test_get_big_object(self): 208 | pairs = [('bkey%d' % i, BigObject('x')) for i in xrange(100)] 209 | for key, value in pairs: 210 | self.mc.set(key, value) 211 | 212 | get = self.mc.get 213 | expected_values = [BigObject('x') for i in xrange(100)] 214 | 215 | def test(): 216 | for i in xrange(100): 217 | result = get('bkey%d' % i) 218 | assert result == expected_values[i] 219 | run_test(test, 'test_get_big_object (100 objects)') 220 | 221 | for key, value in pairs: 222 | self.mc.delete(key) 223 | 224 | def test_get_multi(self): 225 | pairs = zip(self.keys, self.values) 226 | for key, value in pairs: 227 | self.mc.set(key, value) 228 | 229 | keys = self.keys 230 | expected_result = dict(pairs) 231 | 232 | def test(): 233 | result = self.mc.get_multi(keys) 234 | assert result == expected_result 235 | run_test(test, 'test_get_multi') 236 | 237 | for key, value in pairs: 238 | self.mc.delete(key) 239 | 240 | def test_get_list(self): 241 | pairs = zip(self.keys, self.values) 242 | for key, value in pairs: 243 | self.mc.set(key, value) 244 | 245 | keys = self.keys 246 | expected_result = self.values 247 | 248 | def test(): 249 | result = self.mc.get_list(keys) 250 | assert result == expected_result 251 | run_test(test, 'test_get_list') 252 | 253 | for key in self.keys: 254 | self.mc.delete(key) 255 | 256 | 257 | def main(): 258 | from optparse import OptionParser 259 | parser = OptionParser() 260 | parser.add_option('-a', '--server-address', dest='server_address', 261 | default='127.0.0.1:11211', 262 | help="address:port of memcached [default: 127.0.0.1:11211]") 263 | parser.add_option('-n', '--num-tests', dest='num_tests', type='int', 264 | default=1000, 265 | help="repeat counts of each test [default: 1000]") 266 | parser.add_option('-v', '--verbose', dest='verbose', 267 | action='store_true', default=False, 268 | help="show traceback infomation if a test fails") 269 | global options 270 | options, args = parser.parse_args() 271 | 272 | global total_time 273 | total_time = 0 274 | 275 | print "Benchmarking cmemcached..." 276 | import cmemcached 277 | Benchmark(cmemcached, options) 278 | 279 | 280 | if __name__ == '__main__': 281 | main() 282 | global total_time 283 | print "total_time is %f" % total_time 284 | -------------------------------------------------------------------------------- /cmemcached.py: -------------------------------------------------------------------------------- 1 | import os 2 | import sys 3 | import traceback 4 | from zlib import compress, decompress, error as zlib_error 5 | from cmemcached_imp import * 6 | import cmemcached_imp 7 | import threading 8 | 9 | _FLAG_PICKLE = 1 << 0 10 | _FLAG_INTEGER = 1 << 1 11 | _FLAG_LONG = 1 << 2 12 | _FLAG_BOOL = 1 << 3 13 | _FLAG_COMPRESS = 1 << 4 14 | _FLAG_MARSHAL = 1 << 5 15 | 16 | VERSION = "0.41-greenify" 17 | 18 | 19 | def prepare(val, comp_threshold): 20 | val, flag = cmemcached_imp.prepare(val) 21 | if comp_threshold > 0 and val and len(val) > comp_threshold: 22 | val = compress(val) 23 | flag |= _FLAG_COMPRESS 24 | return val, flag 25 | 26 | 27 | def restore(val, flag): 28 | if val is None: 29 | return val 30 | 31 | if flag & _FLAG_COMPRESS: 32 | try: 33 | val = decompress(val) 34 | except zlib_error: 35 | return None 36 | flag &= ~_FLAG_COMPRESS 37 | 38 | return cmemcached_imp.restore(val, flag) 39 | 40 | 41 | class ThreadUnsafe(Exception): 42 | pass 43 | 44 | 45 | class Client(cmemcached_imp.Client): 46 | 47 | "a wraper around cmemcached_imp" 48 | 49 | def __init__(self, servers, do_split=1, comp_threshold=0, behaviors={}, logger=None, cas_support=False, *a, **kw): 50 | cmemcached_imp.Client.__init__(self) 51 | self.servers = servers 52 | self.do_split = do_split 53 | self.comp_threshold = comp_threshold 54 | self.behaviors = dict(behaviors.items()) 55 | self.add_server(servers) 56 | 57 | self.set_behavior(BEHAVIOR_NO_BLOCK, 1) # nonblock 58 | self.set_behavior(BEHAVIOR_TCP_NODELAY, 1) # nonblock 59 | self.set_behavior(BEHAVIOR_TCP_KEEPALIVE, 1) 60 | self.set_behavior(BEHAVIOR_CACHE_LOOKUPS, 1) 61 | # self.set_behavior(BEHAVIOR_BUFFER_REQUESTS, 0) # no request buffer 62 | 63 | #self.set_behavior(BEHAVIOR_KETAMA, 1) 64 | self.set_behavior(BEHAVIOR_HASH, HASH_MD5) 65 | self.set_behavior(BEHAVIOR_KETAMA_HASH, HASH_MD5) 66 | self.set_behavior(BEHAVIOR_DISTRIBUTION, DIST_CONSISTENT_KETAMA) 67 | if cas_support: 68 | self.set_behavior(BEHAVIOR_SUPPORT_CAS, 1) 69 | 70 | for k, v in behaviors.items(): 71 | self.set_behavior(k, v) 72 | 73 | self._thread_ident = None 74 | self._created_stack = traceback.extract_stack() 75 | 76 | def __reduce__(self): 77 | return (Client, (self.servers, self.do_split, self.comp_threshold, self.behaviors)) 78 | 79 | def set_behavior(self, k, v): 80 | self.behaviors[k] = v 81 | return cmemcached_imp.Client.set_behavior(self, k, v) 82 | 83 | def set(self, key, val, time=0, compress=True): 84 | comp = compress and self.comp_threshold or 0 85 | val, flag = prepare(val, comp) 86 | if val is not None: 87 | return self.set_raw(key, val, time, flag) 88 | else: 89 | print >>sys.stderr, '[cmemcached]', 'serialize %s failed' % key 90 | 91 | def set_raw(self, key, val, time, flag): 92 | self._record_thread_ident() 93 | self._check_thread_ident() 94 | return cmemcached_imp.Client.set_raw(self, key, val, time, flag) 95 | 96 | def set_multi(self, values, time=0, compress=True, return_failure=False): 97 | self._record_thread_ident() 98 | self._check_thread_ident() 99 | comp = compress and self.comp_threshold or 0 100 | raw_values = dict((k, prepare(v, comp)) for k, v in values.iteritems()) 101 | return self.set_multi_raw(raw_values, time, return_failure=return_failure) 102 | 103 | def get(self, key): 104 | self._record_thread_ident() 105 | val, flag = cmemcached_imp.Client.get_raw(self, key) 106 | return restore(val, flag) 107 | 108 | def get_multi(self, keys): 109 | self._record_thread_ident() 110 | result = cmemcached_imp.Client.get_multi_raw(self, keys) 111 | return dict((k, restore(v, flag)) 112 | for k, (v, flag) in result.iteritems()) 113 | 114 | def gets(self, key): 115 | self._record_thread_ident() 116 | val, flag, cas = cmemcached_imp.Client.gets_raw(self, key) 117 | return restore(val, flag), cas 118 | 119 | def get_list(self, keys): 120 | self._record_thread_ident() 121 | result = self.get_multi(keys) 122 | return [result.get(key) for key in keys] 123 | 124 | def expire(self, key): 125 | self._record_thread_ident() 126 | return self.touch(key, -1) 127 | 128 | def reset(self): 129 | self.clear_thread_ident() 130 | 131 | def clear_thread_ident(self): 132 | self._thread_ident = None 133 | self._thread_ident_stack = None 134 | 135 | def _record_thread_ident(self): 136 | if self._thread_ident is None: 137 | self._thread_ident = self._get_current_thread_ident() 138 | 139 | def _check_thread_ident(self): 140 | if self._get_current_thread_ident() != self._thread_ident: 141 | raise ThreadUnsafe("mc client created in %s\n%s, called in %s" % 142 | (self._thread_ident, 143 | self._created_stack, 144 | self._get_current_thread_ident())) 145 | 146 | def _get_current_thread_ident(self): 147 | return (os.getpid(), threading.current_thread().name) 148 | -------------------------------------------------------------------------------- /cmemcached_imp.pyx: -------------------------------------------------------------------------------- 1 | __author__ = "davies hongqn subdragon hurricane1026@gmail.com" 2 | __version__ = "0.40" 3 | __copyright__ = "Copyright (C) 2010 douban.com" 4 | __license__ = "Apache License 2.0" 5 | 6 | cdef extern from "Python.h": 7 | ctypedef int Py_ssize_t 8 | int PyString_AsStringAndSize(object obj, char **s, Py_ssize_t *len) except -1 9 | object PyString_FromStringAndSize(char * v, Py_ssize_t len) 10 | object PyInt_FromLong(long v) 11 | char *PyString_AsString(object obj) except NULL 12 | 13 | ctypedef struct PyThreadState: 14 | pass 15 | PyThreadState *PyEval_SaveThread() 16 | void PyEval_RestoreThread(PyThreadState *_save) 17 | int PyObject_CheckReadBuffer(object o) 18 | 19 | cdef extern from "stdlib.h": 20 | ctypedef unsigned int size_t 21 | ctypedef unsigned int time_t 22 | void *malloc(size_t size) 23 | void free(void *ptr) 24 | int atoi (char *STRING) 25 | int strlen(char *STRING) 26 | 27 | cdef extern from "stdint.h": 28 | ctypedef unsigned short int uint16_t 29 | ctypedef unsigned int uint32_t 30 | ctypedef unsigned long long int uint64_t 31 | 32 | cdef extern from "pthread.h": 33 | int pthread_atfork(void (*prepare)(), void (*parent)(), 34 | void (*child)()) 35 | 36 | cdef extern from "libmemcached/memcached.h": 37 | ctypedef enum memcached_return: 38 | MEMCACHED_SUCCESS 39 | MEMCACHED_FAILURE 40 | MEMCACHED_HOST_LOOKUP_FAILURE 41 | MEMCACHED_CONNECTION_FAILURE 42 | MEMCACHED_CONNECTION_BIND_FAILURE 43 | MEMCACHED_WRITE_FAILURE 44 | MEMCACHED_READ_FAILURE 45 | MEMCACHED_UNKNOWN_READ_FAILURE 46 | MEMCACHED_PROTOCOL_ERROR 47 | MEMCACHED_CLIENT_ERROR 48 | MEMCACHED_SERVER_ERROR 49 | MEMCACHED_CONNECTION_SOCKET_CREATE_FAILURE 50 | MEMCACHED_DATA_EXISTS 51 | MEMCACHED_DATA_DOES_NOT_EXIST 52 | MEMCACHED_NOTSTORED 53 | MEMCACHED_STORED 54 | MEMCACHED_NOTFOUND 55 | MEMCACHED_MEMORY_ALLOCATION_FAILURE 56 | MEMCACHED_PARTIAL_READ 57 | MEMCACHED_SOME_ERRORS 58 | MEMCACHED_NO_SERVERS 59 | MEMCACHED_END 60 | MEMCACHED_DELETED 61 | MEMCACHED_VALUE 62 | MEMCACHED_STAT 63 | MEMCACHED_ERRNO 64 | MEMCACHED_FAIL_UNIX_SOCKET 65 | MEMCACHED_NOT_SUPPORTED 66 | MEMCACHED_NO_KEY_PROVIDED 67 | MEMCACHED_FETCH_NOTFINISHED 68 | MEMCACHED_TIMEOUT 69 | MEMCACHED_BUFFERED 70 | MEMCACHED_BAD_KEY_PROVIDED 71 | MEMCACHED_INVALID_HOST_PROTOCOL 72 | MEMCACHED_SERVER_MARKED_DEAD 73 | MEMCACHED_UNKNOWN_STAT_KEY 74 | MEMCACHED_E2BIG 75 | MEMCACHED_INVALID_ARGUMENTS 76 | MEMCACHED_KEY_TOO_BIG 77 | MEMCACHED_AUTH_PROBLEM 78 | MEMCACHED_AUTH_FAILURE 79 | MEMCACHED_AUTH_CONTINUE 80 | MEMCACHED_PARSE_ERROR 81 | MEMCACHED_PARSE_USER_ERROR 82 | MEMCACHED_DEPRECATED 83 | MEMCACHED_IN_PROGRESS 84 | MEMCACHED_SERVER_TEMPORARILY_DISABLED 85 | MEMCACHED_SERVER_MEMORY_ALLOCATION_FAILURE 86 | MEMCACHED_MAXIMUM_RETURN # Always add new error code before 87 | 88 | ctypedef enum memcached_behavior: 89 | MEMCACHED_BEHAVIOR_NO_BLOCK 90 | MEMCACHED_BEHAVIOR_TCP_NODELAY 91 | MEMCACHED_BEHAVIOR_HASH 92 | MEMCACHED_BEHAVIOR_KETAMA 93 | MEMCACHED_BEHAVIOR_SOCKET_SEND_SIZE 94 | MEMCACHED_BEHAVIOR_SOCKET_RECV_SIZE 95 | MEMCACHED_BEHAVIOR_CACHE_LOOKUPS 96 | MEMCACHED_BEHAVIOR_SUPPORT_CAS 97 | MEMCACHED_BEHAVIOR_POLL_TIMEOUT 98 | MEMCACHED_BEHAVIOR_DISTRIBUTION 99 | MEMCACHED_BEHAVIOR_BUFFER_REQUESTS 100 | MEMCACHED_BEHAVIOR_USER_DATA 101 | MEMCACHED_BEHAVIOR_SORT_HOSTS 102 | MEMCACHED_BEHAVIOR_VERIFY_KEY 103 | MEMCACHED_BEHAVIOR_CONNECT_TIMEOUT 104 | MEMCACHED_BEHAVIOR_RETRY_TIMEOUT 105 | MEMCACHED_BEHAVIOR_KETAMA_WEIGHTED 106 | MEMCACHED_BEHAVIOR_KETAMA_HASH 107 | MEMCACHED_BEHAVIOR_BINARY_PROTOCOL 108 | MEMCACHED_BEHAVIOR_SND_TIMEOUT 109 | MEMCACHED_BEHAVIOR_RCV_TIMEOUT 110 | MEMCACHED_BEHAVIOR_SERVER_FAILURE_LIMIT 111 | MEMCACHED_BEHAVIOR_IO_MSG_WATERMARK 112 | MEMCACHED_BEHAVIOR_IO_BYTES_WATERMARK 113 | MEMCACHED_BEHAVIOR_IO_KEY_PREFETCH 114 | MEMCACHED_BEHAVIOR_HASH_WITH_PREFIX_KEY 115 | MEMCACHED_BEHAVIOR_NOREPLY 116 | MEMCACHED_BEHAVIOR_USE_UDP 117 | MEMCACHED_BEHAVIOR_AUTO_EJECT_HOSTS 118 | MEMCACHED_BEHAVIOR_NUMBER_OF_REPLICAS 119 | MEMCACHED_BEHAVIOR_RANDOMIZE_REPLICA_READ 120 | MEMCACHED_BEHAVIOR_CORK 121 | MEMCACHED_BEHAVIOR_TCP_KEEPALIVE 122 | MEMCACHED_BEHAVIOR_TCP_KEEPIDLE 123 | MEMCACHED_BEHAVIOR_LOAD_FROM_FILE 124 | MEMCACHED_BEHAVIOR_REMOVE_FAILED_SERVERS 125 | MEMCACHED_BEHAVIOR_DEAD_TIMEOUT 126 | MEMCACHED_BEHAVIOR_MAX 127 | 128 | ctypedef enum memcached_server_distribution: 129 | MEMCACHED_DISTRIBUTION_MODULA 130 | MEMCACHED_DISTRIBUTION_CONSISTENT 131 | MEMCACHED_DISTRIBUTION_CONSISTENT_KETAMA 132 | MEMCACHED_DISTRIBUTION_RANDOM 133 | MEMCACHED_DISTRIBUTION_CONSISTENT_KETAMA_SPY 134 | MEMCACHED_DISTRIBUTION_CONSISTENT_WEIGHTED 135 | MEMCACHED_DISTRIBUTION_VIRTUAL_BUCKET 136 | MEMCACHED_DISTRIBUTION_CONSISTENT_MAX 137 | 138 | ctypedef enum memcached_hash: 139 | MEMCACHED_HASH_DEFAULT= 0 140 | MEMCACHED_HASH_MD5 141 | MEMCACHED_HASH_CRC 142 | MEMCACHED_HASH_FNV1_64 143 | MEMCACHED_HASH_FNV1A_64 144 | MEMCACHED_HASH_FNV1_32 145 | MEMCACHED_HASH_FNV1A_32 146 | MEMCACHED_HASH_HSIEH 147 | MEMCACHED_HASH_MURMUR 148 | MEMCACHED_HASH_JENKINS 149 | MEMCACHED_HASH_CUSTOM 150 | MEMCACHED_HASH_MAX 151 | 152 | ctypedef enum memcached_connection: 153 | MEMCACHED_CONNECTION_TCP 154 | MEMCACHED_CONNECTION_UDP 155 | MEMCACHED_CONNECTION_UNIX_SOCKET 156 | MEMCACHED_CONNECTION_MAX 157 | 158 | cdef enum: 159 | MEMCACHED_MAX_KEY 160 | 161 | struct memcached_st: 162 | pass 163 | 164 | struct memcached_server_st: 165 | pass 166 | 167 | struct memcached_allocated: 168 | pass 169 | 170 | struct memcached_string_st: 171 | pass 172 | 173 | struct memcached_result_st: 174 | pass 175 | 176 | struct memcached_stat_st: 177 | uint32_t connection_structures 178 | uint32_t curr_connections 179 | uint32_t curr_items 180 | uint32_t pid 181 | uint32_t pointer_size 182 | uint32_t rusage_system_microseconds 183 | uint32_t rusage_system_seconds 184 | uint32_t rusage_user_microseconds 185 | uint32_t rusage_user_seconds 186 | uint32_t threads 187 | uint32_t time 188 | uint32_t total_connections 189 | uint32_t total_items 190 | uint32_t uptime 191 | uint64_t bytes 192 | uint64_t bytes_read 193 | uint64_t bytes_written 194 | uint64_t cmd_get 195 | uint64_t cmd_set 196 | uint64_t evictions 197 | uint64_t get_hits 198 | uint64_t get_misses 199 | uint64_t limit_maxbytes 200 | char version[24] 201 | 202 | size_t memcached_string_length(memcached_string_st* str) 203 | char* memcached_string_value(memcached_string_st* str) 204 | char* memcached_string_c_copy(memcached_string_st* str) 205 | 206 | uint32_t memcached_result_flags(memcached_result_st* result) 207 | uint64_t memcached_result_cas(memcached_result_st* result) 208 | uint32_t memcached_result_key_length(memcached_result_st* result) 209 | void memcached_result_free(memcached_result_st* result) 210 | const char *memcached_result_value(memcached_result_st *ptr) 211 | size_t memcached_result_length(memcached_result_st *ptr) 212 | 213 | char* memcached_result_key_value(memcached_result_st* result) 214 | memcached_string_st memcached_result_string_st(memcached_result_st* result) 215 | 216 | memcached_st *memcached_create(memcached_st *ptr) 217 | void memcached_free(memcached_st *ptr) 218 | char* memcached_get(memcached_st *ptr, char *key, size_t key_length, size_t *value_length, uint32_t *flags, memcached_return *error) 219 | memcached_return memcached_set(memcached_st *ptr, char *key, size_t key_length, 220 | char *value, size_t value_length, time_t expiration, uint32_t flags) 221 | memcached_server_st *memcached_servers_parse(char *server_strings) 222 | memcached_return memcached_server_push(memcached_st *ptr, memcached_server_st *list) 223 | void memcached_server_list_free(memcached_server_st *ptr) 224 | memcached_return memcached_server_add_udp(memcached_st *ptr, char *hostname, int port) 225 | memcached_return memcached_server_add_unix_socket(memcached_st *ptr, char *filename) 226 | memcached_return memcached_server_add(memcached_st *ptr, char *hostname, int port) 227 | memcached_return memcached_server_add_udp_with_weight(memcached_st *ptr, char *hostname, int port, uint32_t weight) 228 | memcached_return memcached_server_add_unix_socket_with_weight(memcached_st *ptr, char *filename, uint32_t weight) 229 | memcached_return memcached_server_add_with_weight(memcached_st *ptr, char *hostname, int port, uint32_t weight) 230 | 231 | memcached_return memcached_increment(memcached_st *ptr, 232 | char *key, size_t key_length, 233 | uint32_t offset, 234 | uint64_t *value) 235 | memcached_return memcached_decrement(memcached_st *ptr, 236 | char *key, size_t key_length, 237 | uint32_t offset, 238 | uint64_t *value) 239 | memcached_return memcached_delete(memcached_st *ptr, char *key, size_t key_length, 240 | time_t expiration) 241 | memcached_return memcached_mget(memcached_st *ptr, 242 | const char * const *keys, 243 | const size_t *key_length, 244 | size_t number_of_keys) 245 | char *memcached_fetch(memcached_st *ptr, char *key, size_t *key_length, 246 | size_t *value_length, uint32_t *flags, 247 | memcached_return *error) 248 | memcached_return memcached_behavior_set(memcached_st *ptr, memcached_behavior flag, uint64_t data) 249 | uint64_t memcached_behavior_get(memcached_st *ptr, unsigned int flag) 250 | memcached_return memcached_append(memcached_st *ptr, 251 | char *key, size_t key_length, 252 | char *value, size_t value_length, 253 | time_t expiration, 254 | uint32_t flags) 255 | memcached_return memcached_prepend(memcached_st *ptr, 256 | char *key, size_t key_length, 257 | char *value, size_t value_length, 258 | time_t expiration, 259 | uint32_t flags) 260 | memcached_result_st *memcached_result_create(memcached_st *memc, 261 | memcached_result_st *ptr) 262 | memcached_result_st *memcached_fetch_result(memcached_st *ptr, 263 | memcached_result_st *result, memcached_return *error) 264 | memcached_return memcached_cas(memcached_st *ptr, 265 | char *key, size_t key_length, 266 | char *value, size_t value_length, 267 | time_t expiration, 268 | uint32_t flags, 269 | uint64_t cas) 270 | memcached_return memcached_add(memcached_st *ptr, 271 | char *key, size_t key_length, 272 | char *value, size_t value_length, 273 | time_t expiration, 274 | uint32_t flags) 275 | memcached_return memcached_replace(memcached_st *ptr, 276 | char *key, size_t key_length, 277 | char *value, size_t value_length, 278 | time_t expiration, 279 | uint32_t flags) 280 | memcached_return memcached_touch(memcached_st *ptr, char *key, 281 | size_t key_length, int expiration) 282 | memcached_stat_st *memcached_stat(memcached_st *ptr, char *args, memcached_return *error) 283 | void memcached_stat_free(memcached_st *ptr, memcached_stat_st *stat) 284 | uint32_t memcached_generate_hash(memcached_st *ptr, char *key, size_t key_length) 285 | const char *memcached_strerror(memcached_st *ptr, memcached_return rc) 286 | memcached_return memcached_flush_buffers(memcached_st *mem) 287 | void memcached_quit(memcached_st *ptr) 288 | 289 | cdef extern from "split_mc.h": 290 | cdef enum: 291 | CHUNK_SIZE 292 | _FLAG_CHUNKED "FLAG_CHUNKED" 293 | 294 | memcached_return split_mc_set(memcached_st *mc, char *key, size_t key_len, void *val, 295 | size_t bytes, time_t expire, uint32_t flags) 296 | char* split_mc_get(memcached_st *mc, char *key, size_t key_len, 297 | int count, size_t *bytes) 298 | 299 | #----------------------------------------- 300 | 301 | import sys 302 | from cPickle import dumps, loads 303 | import marshal 304 | from string import join 305 | from time import strftime 306 | import weakref 307 | 308 | class Error(Exception): 309 | pass 310 | 311 | cdef int _FLAG_PICKLE, _FLAG_INTEGER, _FLAG_LONG, _FLAG_BOOL, _FLAG_MARSHAL 312 | 313 | _FLAG_PICKLE = 1<<0 314 | _FLAG_INTEGER = 1<<1 315 | _FLAG_LONG = 1<<2 316 | _FLAG_BOOL = 1<<3 317 | _FLAG_MARSHAL = 1<<5 318 | 319 | cdef object _prepare(object val, uint32_t *flags): 320 | cdef uint32_t f 321 | f = 0 322 | 323 | if isinstance(val, str): 324 | pass 325 | elif isinstance(val, bool): 326 | f = _FLAG_BOOL 327 | val = str(int(val)) 328 | elif isinstance(val, int): 329 | f = _FLAG_INTEGER 330 | val = str(val) 331 | elif isinstance(val, long): 332 | f = _FLAG_LONG 333 | val = str(val) 334 | elif type(val) is unicode: 335 | val = marshal.dumps(val, 2) 336 | f = _FLAG_MARSHAL 337 | else: 338 | # marshal treats buffers as strings and cause objects e.g. 339 | # numpy.array unrestorable 340 | if not PyObject_CheckReadBuffer(val): 341 | try: 342 | val = marshal.dumps(val, 2) 343 | f = _FLAG_MARSHAL 344 | except ValueError, e: 345 | pass 346 | 347 | if f != _FLAG_MARSHAL: 348 | # val is buffer or marshal failed 349 | try: 350 | val = dumps(val, -1) 351 | f = _FLAG_PICKLE 352 | except Exception, e: 353 | sys.stderr.write('pickle failed: %s\n' % e) 354 | val = None 355 | 356 | flags[0] = f 357 | return val 358 | 359 | def prepare(val): 360 | cdef uint32_t flag 361 | val = _prepare(val, &flag) 362 | return val, flag 363 | 364 | cdef object _restore(object val, uint32_t flags): 365 | 366 | if flags == 0: 367 | pass 368 | elif flags & _FLAG_BOOL: 369 | val = bool(int(val)) 370 | elif flags & _FLAG_INTEGER: 371 | val = int(val) 372 | elif flags & _FLAG_LONG: 373 | val = long(val) 374 | elif flags & _FLAG_MARSHAL: 375 | try: 376 | val = marshal.loads(val) 377 | except Exception, e: 378 | val = None 379 | elif flags & _FLAG_PICKLE: 380 | try: 381 | val = loads(val) 382 | except Exception, e: 383 | val = None 384 | return val 385 | 386 | def restore(val, flag): 387 | return _restore(val, flag) 388 | 389 | # count is the size of the whole chunk 390 | cdef object _restore_splitted(memcached_st *mc, object key, int count): 391 | cdef char *c_key 392 | cdef Py_ssize_t key_len 393 | cdef char *c_val 394 | cdef size_t bytes 395 | 396 | PyString_AsStringAndSize(key, &c_key, &key_len) 397 | c_val = split_mc_get(mc, c_key, key_len, count, &bytes) 398 | 399 | if c_val != NULL: 400 | val = PyString_FromStringAndSize(c_val, bytes) 401 | free(c_val) 402 | else: 403 | val = None 404 | return val 405 | 406 | HASH_DEFAULT = PyInt_FromLong(MEMCACHED_HASH_DEFAULT) 407 | HASH_MD5 = PyInt_FromLong(MEMCACHED_HASH_MD5) 408 | HASH_CRC = PyInt_FromLong(MEMCACHED_HASH_CRC) 409 | HASH_FNV1_64 = PyInt_FromLong(MEMCACHED_HASH_FNV1_64) 410 | HASH_FNV1A_64 = PyInt_FromLong(MEMCACHED_HASH_FNV1A_64) 411 | HASH_FNV1_32 = PyInt_FromLong(MEMCACHED_HASH_FNV1_32) 412 | HASH_FNV1A_32 = PyInt_FromLong(MEMCACHED_HASH_FNV1A_32) 413 | HASH_HSIEH = PyInt_FromLong(MEMCACHED_HASH_HSIEH) 414 | HASH_MURMUR = PyInt_FromLong(MEMCACHED_HASH_MURMUR) 415 | HASH_JENKINS = PyInt_FromLong(MEMCACHED_HASH_JENKINS) 416 | 417 | DIST_MODULA = PyInt_FromLong(MEMCACHED_DISTRIBUTION_MODULA) 418 | DIST_CONSISTENT = PyInt_FromLong(MEMCACHED_DISTRIBUTION_CONSISTENT) 419 | DIST_CONSISTENT_KETAMA = PyInt_FromLong(MEMCACHED_DISTRIBUTION_CONSISTENT_KETAMA) 420 | DIST_RANDOM = PyInt_FromLong(MEMCACHED_DISTRIBUTION_RANDOM) 421 | DIST_CONSISTENT_KETAMA_SPY = PyInt_FromLong(MEMCACHED_DISTRIBUTION_CONSISTENT_KETAMA_SPY) 422 | DIST_CONSISTENT_WEIGHTED = PyInt_FromLong(MEMCACHED_DISTRIBUTION_CONSISTENT_WEIGHTED) 423 | 424 | BEHAVIOR_NO_BLOCK = PyInt_FromLong(MEMCACHED_BEHAVIOR_NO_BLOCK) 425 | BEHAVIOR_TCP_NODELAY = PyInt_FromLong(MEMCACHED_BEHAVIOR_TCP_NODELAY) 426 | BEHAVIOR_HASH = PyInt_FromLong(MEMCACHED_BEHAVIOR_HASH) 427 | BEHAVIOR_KETAMA = PyInt_FromLong(MEMCACHED_BEHAVIOR_KETAMA) 428 | BEHAVIOR_SOCKET_SEND_SIZE = PyInt_FromLong(MEMCACHED_BEHAVIOR_SOCKET_SEND_SIZE) 429 | BEHAVIOR_SOCKET_RECV_SIZE = PyInt_FromLong(MEMCACHED_BEHAVIOR_SOCKET_RECV_SIZE) 430 | BEHAVIOR_CACHE_LOOKUPS = PyInt_FromLong(MEMCACHED_BEHAVIOR_CACHE_LOOKUPS) 431 | BEHAVIOR_SUPPORT_CAS = PyInt_FromLong(MEMCACHED_BEHAVIOR_SUPPORT_CAS) 432 | BEHAVIOR_POLL_TIMEOUT = PyInt_FromLong(MEMCACHED_BEHAVIOR_POLL_TIMEOUT) 433 | BEHAVIOR_DISTRIBUTION = PyInt_FromLong(MEMCACHED_BEHAVIOR_DISTRIBUTION) 434 | BEHAVIOR_BUFFER_REQUESTS = PyInt_FromLong(MEMCACHED_BEHAVIOR_BUFFER_REQUESTS) 435 | BEHAVIOR_USER_DATA = PyInt_FromLong(MEMCACHED_BEHAVIOR_USER_DATA) 436 | BEHAVIOR_SORT_HOSTS = PyInt_FromLong(MEMCACHED_BEHAVIOR_SORT_HOSTS) 437 | BEHAVIOR_VERIFY_KEY = PyInt_FromLong(MEMCACHED_BEHAVIOR_VERIFY_KEY) 438 | BEHAVIOR_CONNECT_TIMEOUT = PyInt_FromLong(MEMCACHED_BEHAVIOR_CONNECT_TIMEOUT) 439 | BEHAVIOR_RETRY_TIMEOUT = PyInt_FromLong(MEMCACHED_BEHAVIOR_RETRY_TIMEOUT) 440 | BEHAVIOR_KETAMA_WEIGHTED = PyInt_FromLong(MEMCACHED_BEHAVIOR_KETAMA_WEIGHTED) 441 | BEHAVIOR_KETAMA_HASH = PyInt_FromLong(MEMCACHED_BEHAVIOR_KETAMA_HASH) 442 | BEHAVIOR_BINARY_PROTOCOL = PyInt_FromLong(MEMCACHED_BEHAVIOR_BINARY_PROTOCOL) 443 | BEHAVIOR_SND_TIMEOUT = PyInt_FromLong(MEMCACHED_BEHAVIOR_SND_TIMEOUT) 444 | BEHAVIOR_RCV_TIMEOUT = PyInt_FromLong(MEMCACHED_BEHAVIOR_RCV_TIMEOUT) 445 | BEHAVIOR_SERVER_FAILURE_LIMIT = PyInt_FromLong(MEMCACHED_BEHAVIOR_SERVER_FAILURE_LIMIT) 446 | BEHAVIOR_IO_MSG_WATERMARK = PyInt_FromLong(MEMCACHED_BEHAVIOR_IO_MSG_WATERMARK) 447 | BEHAVIOR_IO_BYTES_WATERMARK = PyInt_FromLong(MEMCACHED_BEHAVIOR_IO_BYTES_WATERMARK) 448 | BEHAVIOR_IO_KEY_PREFETCH = PyInt_FromLong(MEMCACHED_BEHAVIOR_IO_KEY_PREFETCH) 449 | BEHAVIOR_HASH_WITH_PREFIX_KEY = PyInt_FromLong(MEMCACHED_BEHAVIOR_HASH_WITH_PREFIX_KEY) 450 | BEHAVIOR_NOREPLY = PyInt_FromLong(MEMCACHED_BEHAVIOR_NOREPLY) 451 | BEHAVIOR_USE_UDP = PyInt_FromLong(MEMCACHED_BEHAVIOR_USE_UDP) 452 | BEHAVIOR_AUTO_EJECT_HOSTS = PyInt_FromLong(MEMCACHED_BEHAVIOR_AUTO_EJECT_HOSTS) 453 | BEHAVIOR_NUMBER_OF_REPLICAS = PyInt_FromLong(MEMCACHED_BEHAVIOR_NUMBER_OF_REPLICAS) 454 | BEHAVIOR_RANDOMIZE_REPLICA_READ = PyInt_FromLong(MEMCACHED_BEHAVIOR_RANDOMIZE_REPLICA_READ) 455 | BEHAVIOR_CORK = PyInt_FromLong(MEMCACHED_BEHAVIOR_CORK) 456 | BEHAVIOR_TCP_KEEPALIVE = PyInt_FromLong(MEMCACHED_BEHAVIOR_TCP_KEEPALIVE) 457 | BEHAVIOR_TCP_KEEPIDLE = PyInt_FromLong(MEMCACHED_BEHAVIOR_TCP_KEEPIDLE) 458 | 459 | RETURN_MEMCACHED_SUCCESS = PyInt_FromLong(MEMCACHED_SUCCESS) 460 | RETURN_MEMCACHED_FAILURE = PyInt_FromLong(MEMCACHED_FAILURE) 461 | RETURN_MEMCACHED_HOST_LOOKUP_FAILURE = PyInt_FromLong(MEMCACHED_HOST_LOOKUP_FAILURE) 462 | RETURN_MEMCACHED_CONNECTION_FAILURE = PyInt_FromLong(MEMCACHED_CONNECTION_FAILURE) 463 | RETURN_MEMCACHED_CONNECTION_BIND_FAILURE = PyInt_FromLong(MEMCACHED_CONNECTION_BIND_FAILURE) 464 | RETURN_MEMCACHED_WRITE_FAILURE = PyInt_FromLong(MEMCACHED_WRITE_FAILURE) 465 | RETURN_MEMCACHED_READ_FAILURE = PyInt_FromLong(MEMCACHED_READ_FAILURE) 466 | RETURN_MEMCACHED_UNKNOWN_READ_FAILURE = PyInt_FromLong(MEMCACHED_UNKNOWN_READ_FAILURE) 467 | RETURN_MEMCACHED_PROTOCOL_ERROR = PyInt_FromLong(MEMCACHED_PROTOCOL_ERROR) 468 | RETURN_MEMCACHED_CLIENT_ERROR = PyInt_FromLong(MEMCACHED_CLIENT_ERROR) 469 | RETURN_MEMCACHED_SERVER_ERROR = PyInt_FromLong(MEMCACHED_SERVER_ERROR) 470 | RETURN_MEMCACHED_CONNECTION_SOCKET_CREATE_FAILURE = PyInt_FromLong(MEMCACHED_CONNECTION_SOCKET_CREATE_FAILURE) 471 | RETURN_MEMCACHED_DATA_EXISTS = PyInt_FromLong(MEMCACHED_DATA_EXISTS) 472 | RETURN_MEMCACHED_DATA_DOES_NOT_EXIST = PyInt_FromLong(MEMCACHED_DATA_DOES_NOT_EXIST) 473 | RETURN_MEMCACHED_NOTSTORED = PyInt_FromLong(MEMCACHED_NOTSTORED) 474 | RETURN_MEMCACHED_STORED = PyInt_FromLong(MEMCACHED_STORED) 475 | RETURN_MEMCACHED_NOTFOUND = PyInt_FromLong(MEMCACHED_NOTFOUND) 476 | RETURN_MEMCACHED_MEMORY_ALLOCATION_FAILURE = PyInt_FromLong(MEMCACHED_MEMORY_ALLOCATION_FAILURE) 477 | RETURN_MEMCACHED_PARTIAL_READ = PyInt_FromLong(MEMCACHED_PARTIAL_READ) 478 | RETURN_MEMCACHED_SOME_ERRORS = PyInt_FromLong(MEMCACHED_SOME_ERRORS) 479 | RETURN_MEMCACHED_NO_SERVERS = PyInt_FromLong(MEMCACHED_NO_SERVERS) 480 | RETURN_MEMCACHED_END = PyInt_FromLong(MEMCACHED_END) 481 | RETURN_MEMCACHED_DELETED = PyInt_FromLong(MEMCACHED_DELETED) 482 | RETURN_MEMCACHED_VALUE = PyInt_FromLong(MEMCACHED_VALUE) 483 | RETURN_MEMCACHED_STAT = PyInt_FromLong(MEMCACHED_STAT) 484 | RETURN_MEMCACHED_ERRNO = PyInt_FromLong(MEMCACHED_ERRNO) 485 | RETURN_MEMCACHED_FAIL_UNIX_SOCKET = PyInt_FromLong(MEMCACHED_FAIL_UNIX_SOCKET) 486 | RETURN_MEMCACHED_NOT_SUPPORTED = PyInt_FromLong(MEMCACHED_NOT_SUPPORTED) 487 | RETURN_MEMCACHED_NO_KEY_PROVIDED = PyInt_FromLong(MEMCACHED_NO_KEY_PROVIDED) 488 | RETURN_MEMCACHED_FETCH_NOTFINISHED = PyInt_FromLong(MEMCACHED_FETCH_NOTFINISHED) 489 | RETURN_MEMCACHED_TIMEOUT = PyInt_FromLong(MEMCACHED_TIMEOUT) 490 | RETURN_MEMCACHED_BUFFERED = PyInt_FromLong(MEMCACHED_BUFFERED) 491 | RETURN_MEMCACHED_BAD_KEY_PROVIDED = PyInt_FromLong(MEMCACHED_BAD_KEY_PROVIDED) 492 | RETURN_MEMCACHED_INVALID_HOST_PROTOCOL = PyInt_FromLong(MEMCACHED_INVALID_HOST_PROTOCOL) 493 | RETURN_MEMCACHED_SERVER_MARKED_DEAD = PyInt_FromLong(MEMCACHED_SERVER_MARKED_DEAD) 494 | RETURN_MEMCACHED_UNKNOWN_STAT_KEY = PyInt_FromLong(MEMCACHED_UNKNOWN_STAT_KEY) 495 | RETURN_MEMCACHED_E2BIG = PyInt_FromLong(MEMCACHED_E2BIG) 496 | RETURN_MEMCACHED_INVALID_ARGUMENTS = PyInt_FromLong(MEMCACHED_INVALID_ARGUMENTS) 497 | RETURN_MEMCACHED_KEY_TOO_BIG = PyInt_FromLong(MEMCACHED_KEY_TOO_BIG) 498 | RETURN_MEMCACHED_AUTH_PROBLEM = PyInt_FromLong(MEMCACHED_AUTH_PROBLEM) 499 | RETURN_MEMCACHED_AUTH_FAILURE = PyInt_FromLong(MEMCACHED_AUTH_FAILURE) 500 | RETURN_MEMCACHED_AUTH_CONTINUE = PyInt_FromLong(MEMCACHED_AUTH_CONTINUE) 501 | RETURN_MEMCACHED_PARSE_ERROR = PyInt_FromLong(MEMCACHED_PARSE_ERROR) 502 | RETURN_MEMCACHED_PARSE_USER_ERROR = PyInt_FromLong(MEMCACHED_PARSE_USER_ERROR) 503 | RETURN_MEMCACHED_DEPRECATED = PyInt_FromLong(MEMCACHED_DEPRECATED) 504 | RETURN_MEMCACHED_IN_PROGRESS = PyInt_FromLong(MEMCACHED_IN_PROGRESS) 505 | RETURN_MEMCACHED_SERVER_TEMPORARILY_DISABLED = PyInt_FromLong(MEMCACHED_SERVER_TEMPORARILY_DISABLED) 506 | RETURN_MEMCACHED_SERVER_MEMORY_ALLOCATION_FAILURE = PyInt_FromLong(MEMCACHED_SERVER_MEMORY_ALLOCATION_FAILURE) 507 | RETURN_MEMCACHED_MAXIMUM_RETURN = PyInt_FromLong(MEMCACHED_MAXIMUM_RETURN) 508 | 509 | 510 | __mc_instances = [] 511 | 512 | cdef void close_all_mc(): 513 | for r in __mc_instances: 514 | mc = r() 515 | if mc is not None: 516 | mc.close() 517 | 518 | cdef class Client: 519 | cdef memcached_st *mc 520 | cdef object servers 521 | cdef memcached_return last_error 522 | cdef object __prefix 523 | cdef char* prefix 524 | cdef object log 525 | cdef object __weakref__ 526 | 527 | def __cinit__(self, *a, **kw): 528 | """ 529 | Create a new Client object with the given list of servers. 530 | """ 531 | self.mc = memcached_create(NULL) 532 | self.__prefix = kw.pop('prefix', '') 533 | self.prefix = self.__prefix 534 | if not self.mc: 535 | raise MemoryError 536 | self.servers = [] 537 | self.log = kw.pop('logger', lambda x: None) 538 | 539 | __mc_instances.append(weakref.ref(self)) 540 | 541 | def add_server(self, addrs): 542 | """ 543 | Add new server list 544 | """ 545 | cdef int port 546 | for addr in addrs: 547 | ps = addr.split(':') 548 | if addr.startswith('/'): 549 | path = ps[0] 550 | if len(ps) > 1: 551 | memcached_server_add_unix_socket_with_weight(self.mc, path, int(ps[1])) 552 | else: 553 | memcached_server_add_unix_socket(self.mc, path) 554 | else: 555 | host = ps[0] 556 | port = 11211 557 | if len(ps) > 1: 558 | port = int(ps[1]) 559 | if len(ps) > 2: 560 | memcached_server_add_with_weight(self.mc, host, port, int(ps[2])) 561 | else: 562 | memcached_server_add(self.mc, host, port) 563 | self.servers += addrs 564 | 565 | def get_host_by_key(self, key): 566 | key = self._preprocess_key(key) 567 | if not self.check_key(key, prefixed=1): 568 | return None 569 | cdef char *c_key 570 | cdef Py_ssize_t key_len 571 | cdef unsigned int hash 572 | PyString_AsStringAndSize(key, &c_key, &key_len) 573 | hash = memcached_generate_hash(self.mc, c_key, key_len) 574 | if hash < len(self.servers): 575 | return self.servers[hash] 576 | 577 | def get_last_error(self): 578 | return self.last_error 579 | 580 | def get_last_strerror(self): 581 | cdef const char *c_str = memcached_strerror(self.mc, self.last_error) 582 | return PyString_FromStringAndSize(c_str, strlen(c_str)) 583 | 584 | def __dealloc__(self): 585 | self.close() 586 | memcached_free(self.mc) 587 | 588 | def set_behavior(self, int flag, uint64_t behavior): 589 | return memcached_behavior_set(self.mc, flag, behavior) 590 | 591 | def get_behavior(self, unsigned int behavior): 592 | return memcached_behavior_get(self.mc, behavior) 593 | 594 | def _preprocess_key(self, key): 595 | return self._normalize_to_bytes(self._use_prefix(key)) 596 | 597 | def _use_prefix(self, key): 598 | if self.prefix: 599 | ask_exists = key.startswith('?') 600 | key = self.prefix + (key[1:] if ask_exists else key) 601 | if ask_exists: 602 | key = '?' + key 603 | return key 604 | 605 | def _normalize_to_bytes(self, key, encoding="utf-8"): 606 | if isinstance(key, unicode): 607 | key = key.encode(encoding) 608 | return key 609 | 610 | def _store(self, cmd, key, val, time_t time=0, cas=0, expected=(MEMCACHED_SUCCESS,)): 611 | cdef Py_ssize_t key_len, bytes 612 | cdef char *c_key, *c_val 613 | cdef uint32_t flags 614 | cdef memcached_return retval 615 | cdef int retval_int 616 | cdef PyThreadState *_save 617 | 618 | # set prefix 619 | key = self._preprocess_key(key) 620 | PyString_AsStringAndSize(key, &c_key, &key_len) 621 | 622 | # memcached do not support the key whose length is bigger than MEMCACHED_MAX_KEY 623 | if key_len >= MEMCACHED_MAX_KEY: 624 | return 0 625 | 626 | #validate key 627 | for i from 0 <= i < key_len: 628 | if c_key[i] <= 32 and c_key[i] >=0: 629 | sys.stderr.write("[cmemcached]%s: invalid key(%s)(%d)\n" % (cmd, key, i,)) 630 | return 0 631 | 632 | if cmd in ('append', 'prepend') and type(val) != type(''): 633 | sys.stderr.write("[cmemcached]%s only support string: %s" % (cmd, key)) 634 | return 0 635 | 636 | val = _prepare(val, &flags) 637 | PyString_AsStringAndSize(val, &c_val, &bytes) 638 | 639 | #_save = PyEval_SaveThread() 640 | if cmd == 'add': 641 | retval = memcached_add(self.mc, c_key, key_len, c_val, bytes, time, flags) 642 | elif cmd == 'replace': 643 | retval = memcached_replace(self.mc, c_key, key_len, c_val, bytes, time, flags) 644 | elif cmd == 'cas': 645 | if not self.get_behavior(BEHAVIOR_SUPPORT_CAS): 646 | raise Exception('cas operator need cas_support flag setup') 647 | retval = memcached_cas(self.mc, c_key, key_len, c_val, bytes, time, flags, cas) 648 | elif cmd == 'append': 649 | retval = memcached_append(self.mc, c_key, key_len, c_val, bytes, time, flags) 650 | elif cmd == 'prepend': 651 | retval = memcached_prepend(self.mc, c_key, key_len, c_val, bytes, time, flags) 652 | else: 653 | #PyEval_RestoreThread(_save) 654 | raise Exception, "invalid cmd %s" % cmd 655 | #PyEval_RestoreThread(_save) 656 | 657 | return retval in expected 658 | 659 | def add(self, key, val, time_t time=0): 660 | return self._store('add', key, val, time) 661 | 662 | def replace(self, key, val, time_t time=0): 663 | return self._store('replace', key, val, time) 664 | 665 | def cas(self, key, val, time_t time=0, cas=0): 666 | return self._store('cas', key, val, time, cas) 667 | 668 | def append(self, key, val): 669 | return self._store('append', key, val) 670 | 671 | def prepend(self, key, val): 672 | return self._store('prepend', key, val) 673 | 674 | def _store_multi(self, cmd, keys, val, time_t time=0): 675 | cdef Py_ssize_t key_len, bytes 676 | cdef char *c_key, *c_val 677 | cdef memcached_return retval 678 | 679 | if type(val) != type(''): 680 | sys.stderr.write("[cmemcached]%s only support string: %s" % (cmd, keys)) 681 | return 0 682 | PyString_AsStringAndSize(val, &c_val, &bytes) 683 | 684 | self.set_behavior(BEHAVIOR_NOREPLY, 1) 685 | for key in keys: 686 | key = self._preprocess_key(key) 687 | PyString_AsStringAndSize(key, &c_key, &key_len) 688 | if key_len >= MEMCACHED_MAX_KEY: 689 | continue 690 | if cmd == 'append': 691 | retval = memcached_append(self.mc, c_key, key_len, c_val, bytes, 0, 0) 692 | elif cmd == 'prepend': 693 | retval = memcached_prepend(self.mc, c_key, key_len, c_val, bytes, 0, 0) 694 | self.set_behavior(BEHAVIOR_NOREPLY, 0) 695 | 696 | return retval == MEMCACHED_SUCCESS 697 | 698 | def append_multi(self, keys, val): 699 | return self._store_multi('append', keys, val) 700 | 701 | def prepend_multi(self, keys, val): 702 | return self._store_multi('prepend', keys, val) 703 | 704 | def check_key(self, key, prefixed=0): 705 | cdef Py_ssize_t key_len 706 | cdef char *c_key 707 | cdef int i 708 | 709 | if self.prefix and not prefixed: 710 | key = self._use_prefix(key) 711 | key = self._normalize_to_bytes(key) 712 | PyString_AsStringAndSize(key, &c_key, &key_len) 713 | 714 | if key_len >= MEMCACHED_MAX_KEY: 715 | return 0 716 | 717 | #validate key 718 | for i from 0 <= i < key_len: 719 | if c_key[i] <= 32 and c_key[i] >=0: 720 | sys.stderr.write("[cmemcached]set_raw: invalid key(%s)(%d)\n" % (key, i,)) 721 | return 0 722 | return 1 723 | 724 | def set_raw(self, key, val, time_t time=0, flags_py=0): 725 | cdef Py_ssize_t key_len, bytes 726 | cdef char *c_key, *c_val 727 | cdef uint32_t flags 728 | cdef memcached_return retval 729 | cdef int i 730 | cdef PyThreadState *_save 731 | 732 | key = self._preprocess_key(key) 733 | if self.check_key(key, prefixed=1) == 0: 734 | return False 735 | 736 | flags=flags_py 737 | PyString_AsStringAndSize(key, &c_key, &key_len) 738 | PyString_AsStringAndSize(val, &c_val, &bytes) 739 | 740 | if bytes > CHUNK_SIZE and self.do_split != 0: 741 | _save = PyEval_SaveThread() 742 | retval = split_mc_set(self.mc, c_key, key_len, c_val, bytes, time, flags) 743 | PyEval_RestoreThread(_save) 744 | self.last_error = retval 745 | return (retval == MEMCACHED_SUCCESS) 746 | 747 | _save = PyEval_SaveThread() 748 | retval = memcached_set(self.mc, c_key, key_len, c_val, bytes, time, flags) 749 | PyEval_RestoreThread(_save) 750 | 751 | if retval not in (MEMCACHED_SUCCESS, MEMCACHED_NOTSTORED, MEMCACHED_STORED, 752 | MEMCACHED_SERVER_TEMPORARILY_DISABLED, 753 | MEMCACHED_SERVER_MARKED_DEAD, MEMCACHED_BUFFERED): 754 | #temporary disable this error code above for smoothly upgrading 755 | self.last_error = retval 756 | self.log('[cmemcached]memcached_set: server %s error: %s\n' 757 | % (self.get_host_by_key(key), memcached_strerror(self.mc, retval))) 758 | 759 | return retval in (MEMCACHED_SUCCESS, MEMCACHED_NOTSTORED, MEMCACHED_STORED) 760 | 761 | def set_multi_raw(self, values, time_t time=0, return_failure=False): 762 | cdef Py_ssize_t key_len, bytes 763 | cdef char *c_key, *c_val 764 | cdef uint32_t flags 765 | cdef memcached_return retval 766 | failed_keys = [] 767 | 768 | #self.set_behavior(BEHAVIOR_NOREPLY, 1) 769 | for key, (val, flags_py) in values.iteritems(): 770 | key = self._preprocess_key(key) 771 | if not self.check_key(key, prefixed=1): 772 | continue 773 | flags=flags_py 774 | PyString_AsStringAndSize(key, &c_key, &key_len) 775 | PyString_AsStringAndSize(val, &c_val, &bytes) 776 | if bytes > CHUNK_SIZE and self.do_split != 0: 777 | retval = split_mc_set(self.mc, c_key, key_len, c_val, bytes, time, flags) 778 | else: 779 | retval = memcached_set(self.mc, c_key, key_len, c_val, bytes, time, flags) 780 | if retval != MEMCACHED_SUCCESS: 781 | failed_keys.append(key) 782 | #self.set_behavior(BEHAVIOR_NOREPLY, 0) 783 | return (len(failed_keys) == 0, failed_keys) if return_failure else len(failed_keys) == 0 784 | 785 | def delete(self, key, time_t time=0): 786 | cdef Py_ssize_t key_len 787 | cdef char *c_key 788 | cdef memcached_return retval 789 | cdef PyThreadState *_save 790 | 791 | key = self._preprocess_key(key) 792 | if not self.check_key(key, prefixed=1): 793 | return 0 794 | 795 | PyString_AsStringAndSize(key, &c_key, &key_len) 796 | _save = PyEval_SaveThread() 797 | retval = memcached_delete(self.mc, c_key, key_len, time) 798 | PyEval_RestoreThread(_save) 799 | self.last_error = retval 800 | return retval in (MEMCACHED_SUCCESS, MEMCACHED_NOTFOUND) 801 | 802 | def delete_multi(self, keys, time_t time=0, return_failure=False): 803 | "delete multi key with noreply" 804 | cdef Py_ssize_t key_len 805 | cdef char *c_key 806 | cdef memcached_return retval 807 | failed_keys = [] 808 | 809 | #self.set_behavior(BEHAVIOR_NOREPLY, 1) 810 | for key in keys: 811 | key = self._preprocess_key(key) 812 | PyString_AsStringAndSize(key, &c_key, &key_len) 813 | if key_len >= MEMCACHED_MAX_KEY: 814 | continue 815 | retval = memcached_delete(self.mc, c_key, key_len, time) 816 | if retval not in (MEMCACHED_SUCCESS, MEMCACHED_NOTFOUND): 817 | failed_keys.append(key) 818 | #self.set_behavior(BEHAVIOR_NOREPLY, 0) 819 | 820 | return (len(failed_keys) == 0, failed_keys) if return_failure else len(failed_keys) == 0 821 | 822 | def touch(self, key, int exptime): 823 | cdef Py_ssize_t key_len 824 | cdef char *c_key 825 | cdef memcached_return retval 826 | cdef PyThreadState *_save 827 | 828 | key = self._preprocess_key(key) 829 | if not self.check_key(key, prefixed=1): 830 | return False 831 | 832 | PyString_AsStringAndSize(key, &c_key, &key_len) 833 | _save = PyEval_SaveThread() 834 | retval = memcached_touch(self.mc, c_key, key_len, exptime) 835 | PyEval_RestoreThread(_save) 836 | self.last_error = retval 837 | 838 | return retval == MEMCACHED_SUCCESS 839 | 840 | def get_raw(self, key): 841 | cdef char *c_key 842 | cdef Py_ssize_t key_len 843 | cdef uint32_t flags 844 | cdef size_t bytes 845 | cdef memcached_return rc 846 | cdef char * c_val 847 | cdef PyThreadState *_save 848 | 849 | self.last_error = MEMCACHED_SUCCESS 850 | 851 | key = self._preprocess_key(key) 852 | if not self.check_key(key, prefixed=1): 853 | return None, 0 854 | 855 | flags = 0 856 | PyString_AsStringAndSize(key, &c_key, &key_len) 857 | 858 | _save = PyEval_SaveThread() 859 | c_val = memcached_get(self.mc, c_key, key_len, &bytes, &flags, &rc) 860 | PyEval_RestoreThread(_save) 861 | if NULL == c_val and rc not in (MEMCACHED_SUCCESS, MEMCACHED_NOTFOUND, 862 | MEMCACHED_SERVER_MARKED_DEAD, MEMCACHED_BUFFERED, 863 | MEMCACHED_SERVER_TEMPORARILY_DISABLED): 864 | self.log('[cmemcached]memcached_get: server %s error: %s\n' 865 | % (self.get_host_by_key(key), memcached_strerror(self.mc, rc))) 866 | 867 | #temporary disable MEMCACHED_NOTFOUND for smoothly upgrading 868 | if NULL == c_val and rc not in (MEMCACHED_SUCCESS, MEMCACHED_NOTFOUND): 869 | self.last_error = rc 870 | 871 | if c_val: 872 | if flags & _FLAG_CHUNKED: 873 | val = _restore_splitted(self.mc, key, atoi(c_val)) 874 | flags = flags & (~_FLAG_CHUNKED) 875 | else: 876 | val = PyString_FromStringAndSize(c_val, bytes) 877 | free(c_val) 878 | else: 879 | val = None 880 | 881 | return val, flags 882 | 883 | def gets_raw(self, key): 884 | """ get value and version for following cas operator 885 | if the value is chunked in some keys, and return 886 | the origin key's version. 887 | """ 888 | cdef char *c_key 889 | cdef Py_ssize_t key_len 890 | cdef uint32_t flags 891 | cdef size_t bytes 892 | cdef memcached_return rc 893 | cdef long long cas 894 | 895 | cdef const char * c_val 896 | cdef PyThreadState *_save 897 | cdef memcached_result_st mc_result 898 | cdef memcached_result_st *mc_result_ptr 899 | 900 | if not self.get_behavior(BEHAVIOR_SUPPORT_CAS): 901 | raise Exception('gets operator need cas_support flag setup') 902 | 903 | self.last_error = MEMCACHED_SUCCESS 904 | key = self._preprocess_key(key) 905 | if not self.check_key(key, prefixed=1): 906 | return None, 0, 0 907 | 908 | flags = 0 909 | PyString_AsStringAndSize(key, &c_key, &key_len) 910 | 911 | _save = PyEval_SaveThread() 912 | rc = memcached_mget(self.mc, &c_key, (&key_len), 1) 913 | PyEval_RestoreThread(_save) 914 | if rc != MEMCACHED_SUCCESS: 915 | self.last_error = rc 916 | return None, 0, 0 917 | 918 | self.last_error = MEMCACHED_SUCCESS 919 | mc_result_ptr = memcached_result_create(self.mc, &mc_result) 920 | mc_result_ptr = memcached_fetch_result(self.mc, mc_result_ptr, &rc) 921 | if mc_result_ptr == NULL: 922 | #can not create mc_result 923 | self.last_error = rc 924 | return None, 0, 0 925 | c_val = memcached_result_value(mc_result_ptr) 926 | flags = memcached_result_flags(mc_result_ptr) 927 | cas = memcached_result_cas(mc_result_ptr) 928 | memcached_result_free(mc_result_ptr) 929 | mc_result_ptr = memcached_fetch_result(self.mc, mc_result_ptr, &rc) 930 | self.last_error = rc 931 | if mc_result_ptr== NULL: 932 | if flags & _FLAG_CHUNKED: 933 | val = _restore_splitted(self.mc, key, atoi(c_val)) 934 | flags = flags & (~_FLAG_CHUNKED) 935 | else: 936 | val = PyString_FromStringAndSize(c_val, 937 | memcached_result_length(mc_result_ptr) 938 | ) 939 | return val, flags, cas 940 | else: 941 | memcached_result_free(mc_result_ptr) 942 | memcached_quit(self.mc) 943 | return None, 0, 0 944 | 945 | def get_multi_raw(self, keys): 946 | cdef char **ckeys 947 | cdef Py_ssize_t *ckey_lens 948 | 949 | cdef memcached_return rc 950 | cdef uint32_t flags 951 | 952 | cdef Py_ssize_t key_len 953 | cdef int i, nkeys, valid_nkeys, index 954 | cdef char return_key[MEMCACHED_MAX_KEY] 955 | cdef size_t return_key_length 956 | cdef char *return_value 957 | cdef size_t bytes 958 | cdef PyThreadState *_save 959 | 960 | # do not modify input parameter 961 | keys = list(keys) 962 | 963 | nkeys = len(keys) 964 | ckeys = malloc(sizeof(char *) * nkeys) 965 | ckey_lens = malloc(sizeof(Py_ssize_t) * nkeys) 966 | 967 | index = 0 968 | for i from 0 <= i < nkeys: 969 | keys[i] = self._preprocess_key(keys[i]) 970 | if not self.check_key(keys[i], prefixed=1): 971 | continue 972 | PyString_AsStringAndSize(keys[i], &(ckeys[index]), &(ckey_lens[index])) 973 | if ckey_lens[index] > 0 and ckey_lens[index] < MEMCACHED_MAX_KEY: 974 | index = index + 1 975 | 976 | valid_nkeys = index 977 | 978 | _save = PyEval_SaveThread() 979 | rc = memcached_mget(self.mc, ckeys, ckey_lens, valid_nkeys) 980 | PyEval_RestoreThread(_save) 981 | if rc != MEMCACHED_SUCCESS: 982 | self.last_error = rc 983 | return {} 984 | 985 | self.last_error = MEMCACHED_SUCCESS 986 | result = {} 987 | chunks_record = [] 988 | 989 | flags = 0 990 | while 1: 991 | flags = 0 992 | _save = PyEval_SaveThread() 993 | return_value= memcached_fetch(self.mc, return_key, &return_key_length, 994 | &bytes, &flags, &rc) 995 | PyEval_RestoreThread(_save) 996 | if return_value == NULL: 997 | if rc not in (MEMCACHED_SUCCESS, MEMCACHED_NOTFOUND, MEMCACHED_END): 998 | self.last_error = rc 999 | break 1000 | key = PyString_FromStringAndSize(return_key, return_key_length) 1001 | if flags & _FLAG_CHUNKED: 1002 | chunks_record.append((key, atoi(return_value), flags)) 1003 | else: 1004 | val = PyString_FromStringAndSize(return_value, bytes) 1005 | result[key] = (val, flags) 1006 | free(return_value) 1007 | for key, count, flags in chunks_record: 1008 | val = _restore_splitted(self.mc, key, count) 1009 | flags = flags & (~_FLAG_CHUNKED) 1010 | if val is not None: 1011 | result[key] = (val, flags) 1012 | 1013 | free(ckeys) 1014 | free(ckey_lens) 1015 | 1016 | if self.prefix: 1017 | result = dict([(key.replace(self.prefix, '', 1), value) 1018 | for key, value in result.iteritems()]) 1019 | return result 1020 | 1021 | def incr(self, key, int val=1): 1022 | cdef char *c_key 1023 | cdef Py_ssize_t key_len 1024 | cdef uint64_t new_value 1025 | cdef memcached_return rc 1026 | cdef PyThreadState *_save 1027 | 1028 | key = self._preprocess_key(key) 1029 | PyString_AsStringAndSize(key, &c_key, &key_len) 1030 | if key_len >= MEMCACHED_MAX_KEY: 1031 | return 1032 | 1033 | _save = PyEval_SaveThread() 1034 | if val > 0: 1035 | rc=memcached_increment(self.mc, c_key, key_len, val, &new_value) 1036 | else: 1037 | rc=memcached_decrement(self.mc, c_key, key_len, -val, &new_value) 1038 | PyEval_RestoreThread(_save) 1039 | self.last_error = rc 1040 | 1041 | if rc != MEMCACHED_SUCCESS: 1042 | return 1043 | return new_value 1044 | 1045 | def decr(self, key, int val=1): 1046 | return self.incr(key, -val) 1047 | 1048 | def stats(self): 1049 | cdef memcached_stat_st *stat 1050 | cdef memcached_return rc 1051 | 1052 | stat = memcached_stat(self.mc, NULL, &rc) 1053 | if stat == NULL: 1054 | return {} 1055 | 1056 | stats = {} 1057 | for i in range(len(self.servers)): 1058 | st = {} 1059 | st['pid'] = stat[i].pid 1060 | st['uptime'] = stat[i].uptime 1061 | st['time'] = stat[i].time 1062 | st['pointer_size'] = stat[i].pointer_size 1063 | st['threads'] = stat[i].threads 1064 | st['version'] = stat[i].version 1065 | 1066 | st['rusage_user'] = stat[i].rusage_system_seconds + stat[i].rusage_system_microseconds / 1e6 1067 | st['rusage_system'] = stat[i].rusage_user_seconds + stat[i].rusage_user_microseconds / 1e6 1068 | 1069 | st['curr_items'] = stat[i].curr_items 1070 | st['total_items'] = stat[i].total_items 1071 | 1072 | st['curr_connections'] = stat[i].curr_connections 1073 | st['total_connections'] = stat[i].total_connections 1074 | st['connection_structures'] = stat[i].connection_structures 1075 | 1076 | st['cmd_get'] = stat[i].cmd_get 1077 | st['cmd_set'] = stat[i].cmd_set 1078 | st['get_hits'] = stat[i].get_hits 1079 | st['get_misses'] = stat[i].get_misses 1080 | st['evictions'] = stat[i].evictions 1081 | 1082 | st['bytes'] = stat[i].bytes 1083 | st['bytes_read'] = stat[i].bytes_read 1084 | st['bytes_written'] = stat[i].bytes_written 1085 | st['limit_maxbytes'] = stat[i].limit_maxbytes 1086 | 1087 | stats[self.servers[i]] = st 1088 | 1089 | memcached_stat_free(self.mc, stat) 1090 | 1091 | return stats 1092 | 1093 | def quit(self): 1094 | memcached_quit(self.mc) 1095 | 1096 | def close(self): 1097 | self.quit() 1098 | -------------------------------------------------------------------------------- /patches/0.40/buffer_requests.patch: -------------------------------------------------------------------------------- 1 | --- libmemcached-0.40/libmemcached/behavior.c 2010-04-07 01:58:47.000000000 +0800 2 | +++ libmemcached-0.40-r1/libmemcached/behavior.c 2010-12-05 23:56:39.000000000 +0800 3 | @@ -70,7 +70,7 @@ 4 | break; 5 | case MEMCACHED_BEHAVIOR_BUFFER_REQUESTS: 6 | ptr->flags.buffer_requests= set_flag(data); 7 | - memcached_quit(ptr); 8 | + memcached_flush_buffers(ptr); 9 | break; 10 | case MEMCACHED_BEHAVIOR_USE_UDP: 11 | if (memcached_server_count(ptr)) 12 | --- libmemcached-0.40/libmemcached/storage.c 2010-04-22 07:38:40.000000000 +0800 13 | +++ libmemcached-0.40-r1/libmemcached/storage.c 2010-12-06 00:05:20.000000000 +0800 14 | @@ -162,7 +162,7 @@ 15 | { .length= 2, .buffer= "\r\n" } 16 | }; 17 | 18 | - if (ptr->flags.buffer_requests && verb == SET_OP) 19 | + if (ptr->flags.buffer_requests) 20 | { 21 | to_write= false; 22 | } 23 | @@ -476,7 +476,7 @@ 24 | if (cas) 25 | request.message.header.request.cas= htonll(cas); 26 | 27 | - flush= (bool) ((server->root->flags.buffer_requests && verb == SET_OP) ? 0 : 1); 28 | + flush= (bool) ((server->root->flags.buffer_requests) ? 0 : 1); 29 | 30 | if (server->root->flags.use_udp && ! flush) 31 | { 32 | -------------------------------------------------------------------------------- /patches/0.40/empty_string.patch: -------------------------------------------------------------------------------- 1 | diff -rNu libmemcached-0.40/libmemcached/string.c libmemcached-0.40-r1/libmemcached/string.c 2 | --- libmemcached-0.40/libmemcached/string.c 2010-04-06 00:42:18.000000000 +0800 3 | +++ libmemcached-0.40-r1/libmemcached/string.c 2010-06-09 17:58:09.000000000 +0800 4 | @@ -131,9 +131,6 @@ 5 | { 6 | char *c_ptr; 7 | 8 | - if (memcached_string_length(string) == 0) 9 | - return NULL; 10 | - 11 | c_ptr= libmemcached_malloc(string->root, (memcached_string_length(string)+1) * sizeof(char)); 12 | 13 | if (c_ptr == NULL) 14 | -------------------------------------------------------------------------------- /patches/0.40/fail_over.patch: -------------------------------------------------------------------------------- 1 | diff -rNu libmemcached-0.40/libmemcached/connect.c libmemcached-0.40-r1/libmemcached/connect.c 2 | --- libmemcached-0.40/libmemcached/connect.c 2010-04-23 07:42:03.000000000 +0800 3 | +++ libmemcached-0.40-r1/libmemcached/connect.c 2010-06-11 10:43:23.000000000 +0800 4 | @@ -319,15 +319,15 @@ 5 | if (ptr->root->flags.no_block == false) 6 | timeout= -1; 7 | 8 | - size_t loop_max= 5; 9 | - while (--loop_max) 10 | + int loop_max= 5; 11 | + while (--loop_max > 0) 12 | { 13 | int error= poll(fds, 1, timeout); 14 | 15 | switch (error) 16 | { 17 | case 1: 18 | - loop_max= 1; 19 | + loop_max = 0; 20 | break; 21 | case 0: 22 | continue; 23 | @@ -352,11 +352,16 @@ 24 | 25 | (void)close(ptr->fd); 26 | ptr->fd= -1; 27 | + loop_max = 0; 28 | 29 | break; 30 | } 31 | } 32 | } 33 | + // timeout 34 | + if (loop_max == 0){ 35 | + ptr->fd = -1; 36 | + } 37 | } 38 | else if (errno == EISCONN) /* we are connected :-) */ 39 | { 40 | @@ -397,15 +402,6 @@ 41 | { 42 | WATCHPOINT_STRING("Never got a good file descriptor"); 43 | 44 | - /* Failed to connect. schedule next retry */ 45 | - if (ptr->root->retry_timeout) 46 | - { 47 | - struct timeval next_time; 48 | - 49 | - if (gettimeofday(&next_time, NULL) == 0) 50 | - ptr->next_retry= next_time.tv_sec + ptr->root->retry_timeout; 51 | - } 52 | - 53 | if (ptr->cached_errno == 0) 54 | return MEMCACHED_TIMEOUT; 55 | 56 | @@ -453,22 +449,6 @@ 57 | } 58 | } 59 | 60 | - // If we are over the counter failure, we just fail. Reject host only 61 | - // works if you have a set number of failures. 62 | - if (ptr->root->server_failure_limit && ptr->server_failure_counter >= ptr->root->server_failure_limit) 63 | - { 64 | - set_last_disconnected_host(ptr); 65 | - 66 | - // @todo fix this by fixing behavior to no longer make use of 67 | - // memcached_st 68 | - if (_is_auto_eject_host(ptr->root)) 69 | - { 70 | - run_distribution((memcached_st *)ptr->root); 71 | - } 72 | - 73 | - return MEMCACHED_SERVER_MARKED_DEAD; 74 | - } 75 | - 76 | /* We need to clean up the multi startup piece */ 77 | switch (ptr->type) 78 | { 79 | @@ -488,14 +468,26 @@ 80 | WATCHPOINT_ASSERT(0); 81 | } 82 | 83 | - if (rc == MEMCACHED_SUCCESS) 84 | - { 85 | - ptr->server_failure_counter= 0; 86 | - ptr->next_retry= 0; 87 | - } 88 | - else 89 | + if (rc != MEMCACHED_SUCCESS) 90 | { 91 | ptr->server_failure_counter++; 92 | + 93 | + /* Failed to connect. schedule next retry */ 94 | + if (ptr->root->retry_timeout && ptr->next_retry || ptr->root->server_failure_limit 95 | + && ptr->server_failure_counter >= ptr->root->server_failure_limit) 96 | + { 97 | + struct timeval next_time; 98 | + 99 | + if (gettimeofday(&next_time, NULL) == 0) 100 | + ptr->next_retry= next_time.tv_sec + ptr->root->retry_timeout; 101 | + 102 | + // @todo fix this by fixing behavior to no longer make use of 103 | + // memcached_st 104 | + if (_is_auto_eject_host(ptr->root)) 105 | + { 106 | + run_distribution((memcached_st *)ptr->root); 107 | + } 108 | + } 109 | 110 | set_last_disconnected_host(ptr); 111 | } 112 | diff -rNu libmemcached-0.40/libmemcached/io.c libmemcached-0.40-r1/libmemcached/io.c 113 | --- libmemcached-0.40/libmemcached/io.c 2010-04-23 07:19:48.000000000 +0800 114 | +++ libmemcached-0.40-r1/libmemcached/io.c 2010-06-11 10:44:13.000000000 +0800 115 | @@ -345,6 +345,7 @@ 116 | } 117 | 118 | ptr->server_failure_counter= 0; 119 | + ptr->next_retry = 0; 120 | *nread = (ssize_t)(buffer_ptr - (char*)buffer); 121 | return MEMCACHED_SUCCESS; 122 | } 123 | @@ -627,7 +628,7 @@ 124 | memcached_return_t rc; 125 | rc= io_wait(ptr, MEM_WRITE); 126 | 127 | - if (rc == MEMCACHED_SUCCESS || rc == MEMCACHED_TIMEOUT) 128 | + if (rc == MEMCACHED_SUCCESS) 129 | continue; 130 | 131 | memcached_quit_server(ptr, true); 132 | diff -rNu libmemcached-0.40/libmemcached/quit.c libmemcached-0.40-r1/libmemcached/quit.c 133 | --- libmemcached-0.40/libmemcached/quit.c 2010-04-22 07:31:31.000000000 +0800 134 | +++ libmemcached-0.40-r1/libmemcached/quit.c 2010-06-11 10:42:35.000000000 +0800 135 | @@ -60,6 +60,7 @@ 136 | * sent to the server. 137 | */ 138 | ptr->server_failure_counter= 0; 139 | + ptr->next_retry = 0; 140 | } 141 | memcached_io_close(ptr); 142 | } 143 | @@ -74,6 +75,23 @@ 144 | if (io_death) 145 | { 146 | ptr->server_failure_counter++; 147 | + 148 | + /* Failed to connect. schedule next retry */ 149 | + if (ptr->root->retry_timeout && ptr->next_retry || ptr->root->server_failure_limit 150 | + && ptr->server_failure_counter >= ptr->root->server_failure_limit) 151 | + { 152 | + struct timeval next_time; 153 | + 154 | + if (gettimeofday(&next_time, NULL) == 0) 155 | + ptr->next_retry= next_time.tv_sec + ptr->root->retry_timeout; 156 | + 157 | + // @todo fix this by fixing behavior to no longer make use of 158 | + // memcached_st 159 | + if (_is_auto_eject_host(ptr->root)) 160 | + { 161 | + run_distribution((memcached_st *)ptr->root); 162 | + } 163 | + } 164 | set_last_disconnected_host(ptr); 165 | } 166 | } 167 | -------------------------------------------------------------------------------- /patches/1.0/behavior.patch: -------------------------------------------------------------------------------- 1 | --- libmemcached-1.0.8/libmemcached/behavior.cc 2012-05-13 07:52:47.000000000 +0800 2 | +++ libmemcached-1.0.8-r1/libmemcached/behavior.cc 2012-05-31 15:43:08.000000000 +0800 3 | @@ -489,7 +489,7 @@ 4 | { 5 | if (type < MEMCACHED_DISTRIBUTION_CONSISTENT_MAX) 6 | { 7 | - if (MEMCACHED_DISTRIBUTION_CONSISTENT_WEIGHTED) 8 | + if (type == MEMCACHED_DISTRIBUTION_CONSISTENT_WEIGHTED) 9 | { 10 | ptr->ketama.weighted= true; 11 | } 12 | -------------------------------------------------------------------------------- /patches/1.0/empty_string.patch: -------------------------------------------------------------------------------- 1 | --- libmemcached-1.0.8/libmemcached/string.cc 2012-05-13 07:52:47.000000000 +0800 2 | +++ libmemcached-1.0.8-r1/libmemcached/string.cc 2012-05-30 11:23:10.000000000 +0800 3 | @@ -168,11 +168,6 @@ 4 | 5 | char *memcached_string_c_copy(memcached_string_st *string) 6 | { 7 | - if (memcached_string_length(string) == 0) 8 | - { 9 | - return NULL; 10 | - } 11 | - 12 | char *c_ptr= static_cast(libmemcached_malloc(string->root, (memcached_string_length(string)+1) * sizeof(char))); 13 | 14 | if (c_ptr == NULL) 15 | -------------------------------------------------------------------------------- /patches/1.0/touch_expire-1.0.patch: -------------------------------------------------------------------------------- 1 | --- libmemcached-1.0.8/libmemcached/touch.cc 2012-05-31 17:59:36.000000000 +0800 2 | +++ libmemcached-1.0.8-r1/libmemcached/touch.cc 2012-05-31 18:01:09.000000000 +0800 3 | @@ -40,10 +40,10 @@ 4 | 5 | static memcached_return_t ascii_touch(memcached_server_write_instance_st instance, 6 | const char *key, size_t key_length, 7 | - time_t expiration) 8 | + int expiration) 9 | { 10 | char expiration_buffer[MEMCACHED_MAXIMUM_INTEGER_DISPLAY_LENGTH +1]; 11 | - int expiration_buffer_length= snprintf(expiration_buffer, sizeof(expiration_buffer), " %llu", (unsigned long long)expiration); 12 | + int expiration_buffer_length= snprintf(expiration_buffer, sizeof(expiration_buffer), " %d", expiration); 13 | if (size_t(expiration_buffer_length) >= sizeof(expiration_buffer) or expiration_buffer_length < 0) 14 | { 15 | return memcached_set_error(*instance, MEMCACHED_MEMORY_ALLOCATION_FAILURE, MEMCACHED_AT, 16 | @@ -72,7 +72,7 @@ 17 | 18 | static memcached_return_t binary_touch(memcached_server_write_instance_st instance, 19 | const char *key, size_t key_length, 20 | - time_t expiration) 21 | + int expiration) 22 | { 23 | protocol_binary_request_touch request= {}; //{.bytes= {0}}; 24 | request.message.header.request.magic= PROTOCOL_BINARY_REQ; 25 | @@ -103,7 +103,7 @@ 26 | 27 | memcached_return_t memcached_touch(memcached_st *ptr, 28 | const char *key, size_t key_length, 29 | - time_t expiration) 30 | + int expiration) 31 | { 32 | return memcached_touch_by_key(ptr, key, key_length, key, key_length, expiration); 33 | } 34 | @@ -111,7 +111,7 @@ 35 | memcached_return_t memcached_touch_by_key(memcached_st *ptr, 36 | const char *group_key, size_t group_key_length, 37 | const char *key, size_t key_length, 38 | - time_t expiration) 39 | + int expiration) 40 | { 41 | LIBMEMCACHED_MEMCACHED_TOUCH_START(); 42 | 43 | --- libmemcached-1.0.8/libmemcached-1.0/touch.h 2012-05-13 07:52:47.000000000 +0800 44 | +++ libmemcached-1.0.8-r1/libmemcached-1.0/touch.h 2012-05-31 18:02:18.000000000 +0800 45 | @@ -46,13 +46,13 @@ 46 | LIBMEMCACHED_API 47 | memcached_return_t memcached_touch(memcached_st *ptr, 48 | const char *key, size_t key_length, 49 | - time_t expiration); 50 | + int expiration); 51 | 52 | LIBMEMCACHED_API 53 | memcached_return_t memcached_touch_by_key(memcached_st *ptr, 54 | const char *group_key, size_t group_key_length, 55 | const char *key, size_t key_length, 56 | - time_t expiration); 57 | + int expiration); 58 | 59 | #ifdef __cplusplus 60 | } 61 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | Cython==0.20.2 2 | -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [egg_info] 2 | tag_build = dev 3 | tag_svn_revision = true 4 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | import os 4 | import sys 5 | from setuptools import setup, Extension 6 | from setuptools.command.test import test as TestCommand 7 | 8 | # setuptools DWIM monkey-patch madness 9 | # http://mail.python.org/pipermail/distutils-sig/2007-September/thread.html#8204 10 | if 'setuptools.extension' in sys.modules: 11 | m = sys.modules['setuptools.extension'] 12 | m.Extension.__dict__ = m._Extension.__dict__ 13 | 14 | options = {} 15 | if os.environ.get('LIBRARY_DIRS'): 16 | options['library_dirs'] = [os.environ['LIBRARY_DIRS']] 17 | if os.environ.get('INCLUDE_DIRS'): 18 | options['include_dirs'] = [os.environ['INCLUDE_DIRS']] 19 | 20 | 21 | # support python setup.py test 22 | # http://pytest.org/latest/goodpractises.html#integration-with-setuptools-test-commands 23 | class PyTest(TestCommand): 24 | user_options = [('pytest-args=', 'a', "Arguments to pass to py.test")] 25 | 26 | def initialize_options(self): 27 | TestCommand.initialize_options(self) 28 | self.pytest_args = ['tests'] 29 | 30 | def finalize_options(self): 31 | TestCommand.finalize_options(self) 32 | self.test_args = [] 33 | self.test_suite = True 34 | 35 | def run_tests(self): 36 | # import here, cause outside the eggs aren't loaded 37 | import pytest 38 | errno = pytest.main(self.pytest_args or []) 39 | sys.exit(errno) 40 | 41 | setup(name="python-libmemcached", 42 | version="1.0", 43 | description="python memcached client wrapped on libmemcached", 44 | maintainer="Qiangning Hong", 45 | maintainer_email="hongqn@douban.com", 46 | setup_requires=['setuptools_cython'], 47 | install_requires=['Cython>=0.18'], 48 | ext_modules=[Extension('cmemcached_imp', 49 | ['cmemcached_imp.pyx', 'split_mc.c'], 50 | libraries=['memcached'], 51 | **options)], 52 | py_modules=['cmemcached'], 53 | cmdclass={'test': PyTest}, 54 | tests_require=['pytest', 'mock'], 55 | ) 56 | -------------------------------------------------------------------------------- /split_mc.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include "split_mc.h" 6 | 7 | memcached_return split_mc_set(struct memcached_st *mc, char *key, size_t key_len, void *val, 8 | size_t bytes, time_t expire, uint32_t flags) 9 | { 10 | char chunk_key[250]; 11 | int i; 12 | memcached_return retval; 13 | int r; 14 | size_t chunk_bytes; 15 | 16 | /* assert bytes > CHUNK_SIZE */ 17 | if (key_len > 200 || bytes > CHUNK_SIZE * 10) return MEMCACHED_KEY_TOO_BIG; 18 | 19 | for (i=0; bytes>0; i++) { 20 | r = snprintf(chunk_key, 250, "~%zu%s/%d", key_len, key, i); 21 | if (r < 0) { 22 | return MEMCACHED_BAD_KEY_PROVIDED; 23 | } else if (r >= 250) { 24 | return MEMCACHED_KEY_TOO_BIG; 25 | } 26 | chunk_bytes = bytes > CHUNK_SIZE ? CHUNK_SIZE : bytes; 27 | retval = memcached_set(mc, chunk_key, r, val, chunk_bytes, 28 | expire, flags); 29 | if (retval != MEMCACHED_SUCCESS && retval != MEMCACHED_TIMEOUT) { 30 | return retval; 31 | } 32 | val += CHUNK_SIZE; 33 | bytes -= chunk_bytes; 34 | } 35 | 36 | sprintf(chunk_key, "%d", i); /* re-use chunk_key as value buffer */ 37 | return memcached_set(mc, key, key_len, chunk_key, strlen(chunk_key)+1, 38 | expire, flags|FLAG_CHUNKED); 39 | } 40 | 41 | /* It's up to the caller to free returned *val !!! */ 42 | char* split_mc_get(struct memcached_st *mc, char *key, size_t key_len, 43 | int nchunks, size_t *bytes) 44 | { 45 | int i; 46 | char *c_val, *r, *v; 47 | memcached_return rc; 48 | uint32_t flags; 49 | size_t length; 50 | char chunk_key[250]; 51 | 52 | /* assert res && res.val && (res.flags & FLAG_CHUNKED) */ 53 | if (nchunks > 10 || key_len > 200) 54 | return NULL; 55 | 56 | r = (char*)malloc(sizeof(char) * CHUNK_SIZE * nchunks); 57 | if (!r) return NULL; 58 | 59 | for (i=0, v=r; i CHUNK_SIZE) { 64 | if (c_val) free(c_val); 65 | goto error; 66 | } 67 | 68 | memcpy(v, c_val, length); 69 | free(c_val); 70 | v += length; 71 | } 72 | 73 | *bytes = v - r; 74 | return r; 75 | 76 | error: 77 | memcached_delete(mc, key, key_len, 0); 78 | free(r); 79 | return NULL; 80 | } 81 | -------------------------------------------------------------------------------- /split_mc.h: -------------------------------------------------------------------------------- 1 | #ifndef SPLIT_MC_H 2 | #define SPLIT_MC_H 3 | 4 | #include "libmemcached/memcached.h" 5 | 6 | #ifdef __cplusplus 7 | extern "C" { 8 | #endif 9 | 10 | #define CHUNK_SIZE 1000000 11 | #define FLAG_CHUNKED (1<<12) 12 | 13 | memcached_return split_mc_set(struct memcached_st *mc, char *key, size_t key_len, void *val, 14 | size_t bytes, time_t expire, uint32_t flags); 15 | 16 | char* split_mc_get(struct memcached_st *mc, char *key, size_t key_len, 17 | int count, size_t *bytes); 18 | 19 | 20 | #ifdef __cplusplus 21 | } 22 | #endif 23 | 24 | #endif 25 | -------------------------------------------------------------------------------- /t/benchmark.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | import time 4 | import random 5 | import string 6 | import sys 7 | import traceback 8 | 9 | 10 | global total_time 11 | 12 | def run_test(func, name): 13 | sys.stdout.write(name + ': ') 14 | sys.stdout.flush() 15 | start_time = time.time() 16 | try: 17 | func() 18 | except: 19 | print "failed or not supported" 20 | global options 21 | if options.verbose: 22 | traceback.print_exc() 23 | else: 24 | end_time = time.time() 25 | global total_time 26 | total_time += end_time - start_time 27 | print "%f seconds" % (end_time - start_time) 28 | 29 | 30 | class BigObject(object): 31 | def __init__(self, letter='1', size=10000): 32 | self.object = letter * size 33 | 34 | def __eq__(self, other): 35 | return self.object == other.object 36 | 37 | 38 | class Benchmark(object): 39 | def __init__(self, module, options): 40 | self.module = module 41 | self.options = options 42 | self.init_server() 43 | self.test_set() 44 | self.test_set_get() 45 | self.test_random_get() 46 | self.test_set_same() 47 | self.test_set_big_object() 48 | self.test_set_get_big_object() 49 | self.test_set_big_string() 50 | self.test_set_get_big_string() 51 | self.test_get() 52 | self.test_get_big_object() 53 | self.test_get_multi() 54 | self.test_get_list() 55 | 56 | def init_server(self): 57 | #self.mc = self.module.Client([self.options.server_address]) 58 | self.mc = self.module.Client(["127.0.0.1:11211","127.0.0.1:11212", "127.0.0.1:11213", "127.0.0.1:11214", "127.0.0.1:11215"]) 59 | self.mc.set('bench_key', "E" * 50) 60 | 61 | num_tests = self.options.num_tests 62 | self.keys = ['key%d' % i for i in xrange(num_tests)] 63 | self.values = ['value%d' % i for i in xrange(num_tests)] 64 | import random 65 | self.random_keys = ['key%d' % random.randint(0, num_tests) for i in xrange(num_tests * 3)] 66 | 67 | def test_set(self): 68 | set_ = self.mc.set 69 | pairs = zip(self.keys, self.values) 70 | 71 | def test(): 72 | for key, value in pairs: 73 | set_(key, value) 74 | def test_loop(): 75 | for i in range(10): 76 | for key, value in pairs: 77 | set_(key, value) 78 | run_test(test, 'test_set') 79 | 80 | for key, value in pairs: 81 | self.mc.delete(key) 82 | 83 | def test_set_get(self): 84 | set_ = self.mc.set 85 | get_ = self.mc.get 86 | pairs = zip(self.keys, self.values) 87 | 88 | def test(): 89 | for key, value in pairs: 90 | set_(key, value) 91 | result = get_(key) 92 | assert result == value 93 | run_test(test, 'test_set_get') 94 | 95 | #for key, value in pairs: 96 | # self.mc.delete(key) 97 | 98 | def test_random_get(self): 99 | get_ = self.mc.get 100 | set_ = self.mc.set 101 | 102 | value = "chenyin" 103 | 104 | def test(): 105 | index = 0 106 | for key in self.random_keys: 107 | result = get_(key) 108 | index += 1 109 | if(index % 5 == 0): 110 | set_(key, value) 111 | run_test(test, 'test_random_get') 112 | 113 | def test_set_same(self): 114 | set_ = self.mc.set 115 | 116 | def test(): 117 | for i in xrange(self.options.num_tests): 118 | set_('key', 'value') 119 | def test_loop(): 120 | for i in range(10): 121 | for i in xrange(self.options.num_tests): 122 | set_('key', 'value') 123 | run_test(test, 'test_set_same') 124 | 125 | self.mc.delete('key') 126 | 127 | def test_set_big_object(self): 128 | set_ = self.mc.set 129 | # libmemcached is slow to store large object, so limit the 130 | # number of objects here to make tests not stall. 131 | pairs = [('key%d' % i, BigObject()) for i in xrange(100)] 132 | 133 | def test(): 134 | for key, value in pairs: 135 | set_(key, value) 136 | 137 | run_test(test, 'test_set_big_object (100 objects)') 138 | 139 | for key, value in pairs: 140 | self.mc.delete(key) 141 | 142 | def test_set_get_big_object(self): 143 | set_ = self.mc.set 144 | get_ = self.mc.get 145 | # libmemcached is slow to store large object, so limit the 146 | # number of objects here to make tests not stall. 147 | pairs = [('key%d' % i, BigObject()) for i in xrange(100)] 148 | 149 | def test(): 150 | for key, value in pairs: 151 | set_(key, value) 152 | result = get_(key) 153 | assert result == value 154 | 155 | run_test(test, 'test_set_get_big_object (100 objects)') 156 | 157 | #for key, value in pairs: 158 | # self.mc.delete(key) 159 | 160 | def test_set_get_big_string(self): 161 | set_ = self.mc.set 162 | get_ = self.mc.get 163 | 164 | # libmemcached is slow to store large object, so limit the 165 | # number of objects here to make tests not stall. 166 | pairs = [('key%d' % i, 'x' * 10000) for i in xrange(100)] 167 | 168 | def test(): 169 | for key, value in pairs: 170 | set_(key, value) 171 | result = get_(key) 172 | assert result == value 173 | run_test(test, 'test_set_get_big_string (100 objects)') 174 | 175 | 176 | def test_set_big_string(self): 177 | set_ = self.mc.set 178 | 179 | # libmemcached is slow to store large object, so limit the 180 | # number of objects here to make tests not stall. 181 | pairs = [('key%d' % i, 'x' * 10000) for i in xrange(100)] 182 | 183 | def test(): 184 | for key, value in pairs: 185 | set_(key, value) 186 | run_test(test, 'test_set_big_string (100 objects)') 187 | 188 | for key, value in pairs: 189 | self.mc.delete(key) 190 | 191 | 192 | def test_get(self): 193 | pairs = zip(self.keys, self.values) 194 | for key, value in pairs: 195 | self.mc.set(key, value) 196 | 197 | get = self.mc.get 198 | 199 | def test(): 200 | for key, value in pairs: 201 | result = get(key) 202 | assert result == value 203 | run_test(test, 'test_get') 204 | 205 | for key, value in pairs: 206 | self.mc.delete(key) 207 | 208 | def test_get_big_object(self): 209 | pairs = [('bkey%d' % i, BigObject('x')) for i in xrange(100)] 210 | for key, value in pairs: 211 | self.mc.set(key, value) 212 | 213 | get = self.mc.get 214 | expected_values = [BigObject('x') for i in xrange(100)] 215 | 216 | def test(): 217 | for i in xrange(100): 218 | result = get('bkey%d' % i) 219 | assert result == expected_values[i] 220 | run_test(test, 'test_get_big_object (100 objects)') 221 | 222 | for key, value in pairs: 223 | self.mc.delete(key) 224 | 225 | def test_get_multi(self): 226 | pairs = zip(self.keys, self.values) 227 | for key, value in pairs: 228 | self.mc.set(key, value) 229 | 230 | keys = self.keys 231 | expected_result = dict(pairs) 232 | 233 | def test(): 234 | result = self.mc.get_multi(keys) 235 | assert result == expected_result 236 | run_test(test, 'test_get_multi') 237 | 238 | for key, value in pairs: 239 | self.mc.delete(key) 240 | 241 | def test_get_list(self): 242 | pairs = zip(self.keys, self.values) 243 | for key, value in pairs: 244 | self.mc.set(key, value) 245 | 246 | keys = self.keys 247 | expected_result = self.values 248 | 249 | def test(): 250 | result = self.mc.get_list(keys) 251 | assert result == expected_result 252 | run_test(test, 'test_get_list') 253 | 254 | for key in self.keys: 255 | self.mc.delete(key) 256 | 257 | 258 | def main(): 259 | from optparse import OptionParser 260 | parser = OptionParser() 261 | parser.add_option('-a', '--server-address', dest='server_address', 262 | default='127.0.0.1:11211', 263 | help="address:port of memcached [default: 127.0.0.1:11211]") 264 | parser.add_option('-n', '--num-tests', dest='num_tests', type='int', 265 | default=1000, 266 | help="repeat counts of each test [default: 1000]") 267 | parser.add_option('-v', '--verbose', dest='verbose', 268 | action='store_true', default=False, 269 | help="show traceback infomation if a test fails") 270 | global options 271 | options, args = parser.parse_args() 272 | 273 | 274 | options.verbose=1 275 | print "verbose", options.verbose 276 | 277 | global total_time 278 | total_time = 0 279 | 280 | print "Benchmarking cmemcached..." 281 | import cmemcached 282 | Benchmark(cmemcached, options) 283 | 284 | 285 | if __name__ == '__main__': 286 | main() 287 | global total_time 288 | print "total_time is %f" % total_time 289 | -------------------------------------------------------------------------------- /t/node_failure_test.py: -------------------------------------------------------------------------------- 1 | from cmemcached import * 2 | import time 3 | import cmemcached 4 | print cmemcached 5 | 6 | mc = Client([ 7 | # '192.163.1.222:11211', 8 | # '192.163.1.2:11211', 9 | # '192.163.1.5:11211', 10 | # 'localhost:7902' 11 | 'theoden:11400', 12 | ], do_split=0) 13 | print mc.set_behavior(BEHAVIOR_NO_BLOCK, 0) 14 | #print mc.set_behavior(BEHAVIOR_TCP_NODELAY, 1) 15 | mc.set_behavior(BEHAVIOR_CONNECT_TIMEOUT, 50) 16 | mc.set_behavior(BEHAVIOR_SND_TIMEOUT, 500*1000) 17 | mc.set_behavior(BEHAVIOR_RCV_TIMEOUT, 500*1000) 18 | mc.set_behavior(BEHAVIOR_POLL_TIMEOUT, 5000) 19 | mc.set_behavior(BEHAVIOR_SERVER_FAILURE_LIMIT, 2) 20 | mc.set_behavior(BEHAVIOR_RETRY_TIMEOUT, 10) 21 | 22 | print mc.get_behavior(BEHAVIOR_NO_BLOCK) 23 | print mc.get_behavior(BEHAVIOR_TCP_NODELAY) 24 | 25 | for i in xrange(1): 26 | print time.time(), mc.set('test', 'test'*1024*1024) 27 | # print time.time(), len(mc.get('test') or '') 28 | -------------------------------------------------------------------------------- /t/pickle_perf.py: -------------------------------------------------------------------------------- 1 | from cPickle import dumps, loads 2 | from timeit import Timer 3 | 4 | num = 300 5 | class BigObject(object): 6 | def __init__(self, letter='1', size=10000): 7 | self.object = letter * size 8 | 9 | def __eq__(self, other): 10 | return self.object == other.object 11 | 12 | sillything = [] 13 | 14 | for i in (1000, 10000, 100000, 200000): 15 | value = [BigObject(str(j%10), i) for j in xrange(num)] 16 | sillything.append(value) 17 | 18 | 19 | value_t = None 20 | 21 | def test_pickle(): 22 | for tset in sillything: 23 | for t in tset: 24 | value = dumps(t, -1) 25 | value_t = loads(value) 26 | #assert value_t == t 27 | 28 | from time import time 29 | t=time() 30 | for i in xrange(10): 31 | test_pickle() 32 | print time()-t 33 | #t=Timer(setup='from __main__ import test_pickle', stmt='test_pickle()') 34 | #print t.timeit() 35 | 36 | -------------------------------------------------------------------------------- /t/set_get_pressure_test.py: -------------------------------------------------------------------------------- 1 | import cmemcached 2 | class BigObject(object): 3 | def __init__(self, letter='1', size=10000): 4 | self.object = letter * size 5 | 6 | def __eq__(self, other): 7 | return self.object == other.object 8 | 9 | def main(): 10 | mc=cmemcached.Client(["127.0.0.1:11211","127.0.0.1:11212","127.0.0.1:11213", "127.0.0.1:11214"]) 11 | 12 | num=200 13 | keyss = [] 14 | for i in xrange(4): 15 | k=i 16 | keyss.append(['key%d_%d' % (k,j) for j in xrange(num)]) 17 | 18 | 19 | valuess = [] 20 | for i in (1000, 10000, 100000, 200000): 21 | values = [BigObject(str(j%10), i) for j in xrange(num)] 22 | valuess.append(values) 23 | 24 | def test_set_get(mc, pairs): 25 | counter = 0 26 | for key, value in pairs: 27 | mc.set(key, value) 28 | if(counter%4 == 0): 29 | #assert mc.get(key) == value 30 | pass 31 | counter+=1 32 | 33 | from time import time 34 | t=time() 35 | for i in xrange(100): 36 | for k in xrange(4): 37 | pairs = zip(keyss[k],valuess[k]) 38 | test_set_get(mc, pairs) 39 | print time()-t 40 | 41 | 42 | if __name__ == '__main__': 43 | main() 44 | -------------------------------------------------------------------------------- /t/test_key_mapping.py: -------------------------------------------------------------------------------- 1 | import cmemcached 2 | import optparse 3 | 4 | class BigObject(object): 5 | def __init__(self, letter='1', size=10000): 6 | self.object = letter * size 7 | 8 | def __eq__(self, other): 9 | return self.object == other.object 10 | 11 | 12 | def main(): 13 | parameters = ['set', 'add_server', 'remove_server'] 14 | parameters_string = '|'.join(parameters) 15 | p = optparse.OptionParser(usage=("%prog " + parameters_string)) 16 | 17 | options, arguments = p.parse_args() 18 | if len(arguments) != 1 or arguments[0] not in parameters: 19 | p.error('Only one parameter(in %s) are supported!' % parameters_string) 20 | print arguments[0] 21 | 22 | if arguments[0] == 'remove_server': 23 | mc = cmemcached.Client(["127.0.0.1:11211","127.0.0.1:11212","127.0.0.1:11213"]) 24 | else: 25 | mc = cmemcached.Client(["127.0.0.1:11211","127.0.0.1:11212","127.0.0.1:11213", "127.0.0.1:11214"]) 26 | 27 | num_tests = 10000 28 | count = 0 29 | keys = ['helloworld%d' % i for i in xrange(num_tests)] 30 | if arguments[0] == 'set': 31 | for key in keys: 32 | mc.set(key , 'aa') 33 | for key in keys: 34 | assert mc.get(key) == 'aa' 35 | 36 | if arguments[0] == 'add_server': 37 | mc.add_server(["127.0.0.1:11215"]) 38 | for key in keys: 39 | if mc.get(key) == 'aa': 40 | count += 1 41 | print "hit rate:", float(count)/num_tests 42 | 43 | if arguments[0] == 'remove_server': 44 | for key in keys: 45 | if mc.get(key) == 'aa': 46 | count += 1 47 | print "hit rate:", float(count)/num_tests 48 | 49 | 50 | if __name__ == '__main__': 51 | main() 52 | -------------------------------------------------------------------------------- /t/test_server_down.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | import cmemcached 3 | 4 | def main(): 5 | mc=cmemcached.Client(["127.0.0.1:11211","127.0.0.1:11212","127.0.0.1:11213", "127.0.0.1:11214"]) 6 | 7 | num = 10000 8 | keys = ["key%d" % k for k in xrange(num)] 9 | 10 | success_counter = 0 11 | failure_counter = 0 12 | 13 | def print_counter(): 14 | print "success_counter is ", success_counter 15 | print "failure_counter is ", failure_counter 16 | assert failure_counter+success_counter == num 17 | print "mis rate is ", float(failure_counter)/num 18 | 19 | 20 | while True: 21 | for key in keys: 22 | mc.set(key, "aa") 23 | for key in keys: 24 | if mc.get(key) == "aa": 25 | success_counter += 1 26 | else: 27 | failure_counter += 1 28 | print_counter() 29 | success_counter = 0 30 | failure_counter = 0 31 | 32 | 33 | if __name__ == '__main__': 34 | main() 35 | -------------------------------------------------------------------------------- /tests/conftest.py: -------------------------------------------------------------------------------- 1 | from subprocess import Popen 2 | import time 3 | import socket 4 | import pytest 5 | import cmemcached 6 | 7 | 8 | @pytest.fixture(scope='session') 9 | def memcached(request): 10 | # find a usable port in ephemeral ports 11 | for port in xrange(49152, 65536): 12 | sck = socket.socket() 13 | try: 14 | sck.bind(('127.0.0.1', port)) 15 | except socket.error: 16 | continue 17 | sck.close() 18 | 19 | process = Popen(['memcached', '-l', '127.0.0.1', '-p', str(port)]) 20 | time.sleep(1) 21 | if process.poll() is None: 22 | # we found a usable port! 23 | print "memcached listening on port %s" % port 24 | 25 | def fin(): 26 | print "shutting down memcached" 27 | process.terminate() 28 | process.wait() 29 | request.addfinalizer(fin) 30 | return '127.0.0.1:%d' % port 31 | 32 | raise Exception("Cannot find usable port for memcached to listen on") 33 | 34 | 35 | @pytest.fixture 36 | def mc(memcached): 37 | return cmemcached.Client([memcached], comp_threshold=1024) 38 | -------------------------------------------------------------------------------- /tests/test_cmemcached.py: -------------------------------------------------------------------------------- 1 | # -*- encoding:utf-8 -*- 2 | 3 | import cmemcached 4 | import unittest 5 | import cPickle as pickle 6 | import marshal 7 | import time 8 | import threading 9 | import platform 10 | import pytest 11 | 12 | INVALID_SERVER_ADDR = '127.0.0.1:1' 13 | 14 | skip_if_libmemcached_not_patched = pytest.mark.skipif( 15 | "gentoo" not in platform.platform(), # FIXME: better way to detect if patched? 16 | reason="need to link with libmemcached with douban's patches") 17 | 18 | 19 | class BigObject(object): 20 | 21 | def __init__(self, letter='1', size=2000000): 22 | self.object = letter * size 23 | 24 | def __eq__(self, other): 25 | return self.object == other.object 26 | 27 | 28 | class NoPickle(object): 29 | 30 | def __getattr__(self, name): 31 | pass 32 | 33 | 34 | class TestCmemcached(unittest.TestCase): 35 | 36 | @pytest.fixture(autouse=True) 37 | def setup(self, memcached): 38 | self.server_addr = memcached 39 | self.mc = cmemcached.Client([memcached], comp_threshold=1024) 40 | 41 | def test_set_get(self): 42 | self.mc.set("key", "value") 43 | self.assertEqual(self.mc.get("key"), "value") 44 | 45 | self.mc.set("key_int", 1) 46 | self.assertEqual(self.mc.get("key_int"), 1) 47 | 48 | self.mc.set("key_long", 1234567890L) 49 | self.assertEqual(self.mc.get("key_long"), 1234567890L) 50 | 51 | self.mc.set("key_object", BigObject()) 52 | self.assertEqual(self.mc.get("key_object"), BigObject()) 53 | 54 | big_object = BigObject('x', 1000001) 55 | self.mc.set("key_big_object", big_object) 56 | self.assertEqual(self.mc.get("key_big_object"), big_object) 57 | 58 | def test_set_get_none(self): 59 | self.assertEqual(self.mc.set('key', None), True) 60 | self.assertEqual(self.mc.get('key'), None) 61 | 62 | def test_chinese_set_get(self): 63 | key = '豆瓣' 64 | value = '在炎热的夏天我们无法停止上豆瓣' 65 | self.assertEqual(self.mc.set(key, value), 1) 66 | 67 | self.assertEqual(self.mc.get(key), value) 68 | 69 | def test_unicode_set_get(self): 70 | key = "test_unicode_set_get" 71 | value = u"中文" 72 | self.assertEqual(self.mc.set(key, value), 1) 73 | self.assertEqual(self.mc.get(key), value) 74 | 75 | def test_special_key(self): 76 | key = 'keke a kid' 77 | value = 1024 78 | self.assertEqual(self.mc.set(key, value), 0) 79 | self.assertEqual(self.mc.get(key), None) 80 | key = 'u:keke a kid' 81 | self.assertEqual(self.mc.set(key, value), 0) 82 | self.assertEqual(self.mc.get(key), None) 83 | 84 | def test_unicode_key(self): 85 | key1 = u"answer" 86 | key2 = u"答案" 87 | bytes_key1 = "answer" 88 | bytes_key2 = "答案" 89 | value = 42 90 | 91 | self.assertEqual(self.mc.set(key1, value), 1) 92 | self.assertEqual(self.mc.get(key1), value) 93 | 94 | self.assertEqual(self.mc.set(key2, value), 1) 95 | self.assertEqual(self.mc.get(key2), value) 96 | 97 | self.assertEqual(self.mc.incr(key2), value + 1) 98 | self.assertEqual(self.mc.get(key2), value + 1) 99 | 100 | self.assertEqual(self.mc.delete(key1), 1) 101 | self.assertEqual(self.mc.get(key1), None) 102 | 103 | self.assertEqual(self.mc.add(key1, value), 1) 104 | self.assertEqual(self.mc.get(key1), value) 105 | self.assertEqual(self.mc.add(key1, value), False) 106 | self.assertEqual(self.mc.set(key1, value), 1) 107 | 108 | self.assertEqual(self.mc.get(bytes_key1), self.mc.get(key1)) 109 | self.assertEqual(self.mc.get(bytes_key2), self.mc.get(key2)) 110 | 111 | def test_add(self): 112 | key = 'test_add' 113 | self.mc.delete(key) 114 | self.assertEqual(self.mc.add(key, 'tt'), 1) 115 | self.assertEqual(self.mc.get(key), 'tt') 116 | self.assertEqual(self.mc.add(key, 'tt'), 0) 117 | self.mc.delete(key + '2') 118 | self.assertEqual(self.mc.add(key + '2', range(10)), 1) 119 | 120 | def test_replace(self): 121 | key = 'test_replace' 122 | self.mc.delete(key) 123 | self.assertEqual(self.mc.replace(key, ''), 0) 124 | self.assertEqual(self.mc.set(key, 'b'), 1) 125 | self.assertEqual(self.mc.replace(key, 'a'), 1) 126 | self.assertEqual(self.mc.get(key), 'a') 127 | 128 | def test_append(self): 129 | key = "test_append" 130 | value = "append\n" 131 | self.mc.delete(key) 132 | self.assertEqual(self.mc.append(key, value), 0) 133 | self.mc.set(key, "") 134 | self.assertEqual(self.mc.append(key, value), 1) 135 | self.assertEqual(self.mc.append(key, value), 1) 136 | self.assertEqual(self.mc.prepend(key, 'before\n'), 1) 137 | self.assertEqual(self.mc.get(key), 'before\n' + value * 2) 138 | 139 | def test_append_multi(self): 140 | N = 10 141 | K = "test_append_multi_%d" 142 | data = "after\n" 143 | for i in range(N): 144 | self.assertEqual(self.mc.set(K % i, "before\n"), 1) 145 | keys = [K % i for i in range(N)] 146 | # append 147 | self.assertEqual(self.mc.append_multi(keys, data), 1) 148 | self.assertEqual(self.mc.get_multi(keys), 149 | dict(zip(keys, ["before\n" + data] * N))) 150 | # prepend 151 | self.assertEqual(self.mc.prepend_multi(keys, data), 1) 152 | self.assertEqual(self.mc.get_multi(keys), 153 | dict(zip(keys, [data + "before\n" + data] * N))) 154 | # delete 155 | self.assertEqual(self.mc.delete_multi(keys), 1) 156 | self.assertEqual(self.mc.get_multi(keys), {}) 157 | 158 | def test_append_multi_performance(self): 159 | N = 70000 160 | K = "test_append_multi_%d" 161 | data = "after\n" 162 | keys = [K % i for i in range(N)] 163 | t = time.time() 164 | self.mc.append_multi(keys, data) 165 | t = time.time() - t 166 | assert t < 1.5, 'should append 7w key in 1.5 secs, actual val: %f' % t 167 | 168 | def test_set_multi(self): 169 | values = dict(('key%s' % k, ('value%s' % k) * 100) 170 | for k in range(1000)) 171 | values.update({' ': ''}) 172 | self.assertEqual(self.mc.set_multi(values), 1) 173 | del values[' '] 174 | self.assertEqual(self.mc.get_multi(values.keys()), values) 175 | # mc=cmemcached.Client(["localhost:11999"], comp_threshold=1024) 176 | # self.assertEqual(mc.set_multi(values), 0) 177 | 178 | def test_append_large(self): 179 | k = 'test_append_large' 180 | self.mc.set(k, 'a' * 2048) 181 | self.mc.append(k, 'bbbb') 182 | assert 'bbbb' not in self.mc.get(k) 183 | self.mc.set(k, 'a' * 2048, compress=False) 184 | self.mc.append(k, 'bbbb') 185 | assert 'bbbb' in self.mc.get(k) 186 | 187 | def test_incr(self): 188 | key = "Not_Exist" 189 | self.assertEqual(self.mc.incr(key), None) 190 | # key="incr:key1" 191 | # self.mc.set(key, "not_numerical") 192 | # self.assertEqual(self.mc.incr(key), 0) 193 | key = "incr:key2" 194 | self.mc.set(key, 2007) 195 | self.assertEqual(self.mc.incr(key), 2008) 196 | 197 | def test_decr(self): 198 | key = "Not_Exist" 199 | self.assertEqual(self.mc.decr(key), None) 200 | # key="decr:key1" 201 | # self.mc.set(key, "not_numerical") 202 | # self.assertEqual(self.mc.decr(key),0) 203 | key = "decr:key2" 204 | self.mc.set(key, 2009) 205 | self.assertEqual(self.mc.decr(key), 2008) 206 | 207 | def test_get_multi(self): 208 | keys = ["hello1", "hello2", "hello3"] 209 | values = ["vhello1", "vhello2", "vhello3"] 210 | for x in xrange(3): 211 | self.mc.set(keys[x], values[x]) 212 | self.assertEqual(self.mc.get(keys[x]), values[x]) 213 | result = self.mc.get_multi(keys) 214 | for x in xrange(3): 215 | self.assertEqual(result[keys[x]], values[x]) 216 | 217 | def test_get_multi_invalid(self): 218 | keys = ["hello1", "hello2", "hello3"] 219 | values = ["vhello1", "vhello2", "vhello3"] 220 | for x in xrange(3): 221 | self.mc.set(keys[x], values[x]) 222 | self.assertEqual(self.mc.get(keys[x]), values[x]) 223 | invalid_keys = keys + ['hoho\r\n'] 224 | result = self.mc.get_multi(invalid_keys) 225 | for x in xrange(3): 226 | self.assertEqual(result[keys[x]], values[x]) 227 | result_new = self.mc.get_multi(keys) 228 | for x in xrange(3): 229 | self.assertEqual(result_new[keys[x]], values[x]) 230 | 231 | def test_get_multi_big(self): 232 | keys = ["hello1", "hello2", "hello3"] 233 | values = [BigObject(str(i), 1000001) for i in xrange(3)] 234 | for x in xrange(3): 235 | self.mc.set(keys[x], values[x]) 236 | self.assertEqual(self.mc.get(keys[x]), values[x]) 237 | result = self.mc.get_multi(keys) 238 | for x in xrange(3): 239 | self.assertEqual(result[keys[x]], values[x]) 240 | 241 | @skip_if_libmemcached_not_patched 242 | def test_get_multi_with_empty_string(self): 243 | keys = ["hello1", "hello2", "hello3"] 244 | for k in keys: 245 | self.mc.set(k, '') 246 | self.assertEqual(self.mc.get_multi(keys), dict(zip(keys, [""] * 3))) 247 | 248 | def testBool(self): 249 | self.mc.set("bool", True) 250 | value = self.mc.get("bool") 251 | self.assertEqual(value, True) 252 | self.mc.set("bool_", False) 253 | value = self.mc.get("bool_") 254 | self.assertEqual(value, False) 255 | 256 | @skip_if_libmemcached_not_patched 257 | def testEmptyString(self): 258 | self.assertTrue(self.mc.set("str", '')) 259 | value = self.mc.get("str") 260 | self.assertEqual(value, '') 261 | 262 | def testGetHost(self): 263 | host = self.mc.get_host_by_key("str") 264 | self.assertEqual(host, self.server_addr) 265 | 266 | def test_get_list(self): 267 | self.mc.set("a", 'a') 268 | self.mc.delete('b') 269 | v = self.mc.get_list(['a', 'b']) 270 | self.assertEqual(v, ['a', None]) 271 | 272 | def test_marshal(self): 273 | v = [{2: {"a": 337}}] 274 | self.mc.set("a", v) 275 | self.assertEqual(self.mc.get("a"), v) 276 | raw, flags = self.mc.get_raw("a") 277 | self.assertEqual(raw, marshal.dumps(v, 2)) 278 | 279 | def test_pickle(self): 280 | v = [{"v": BigObject('a', 10)}] 281 | self.mc.set("a", v) 282 | self.assertEqual(self.mc.get("a"), v) 283 | raw, flags = self.mc.get_raw("a") 284 | self.assertEqual(raw, pickle.dumps(v, -1)) 285 | 286 | def test_no_pickle(self): 287 | v = NoPickle() 288 | self.assertEqual(self.mc.set("nopickle", v), None) 289 | self.assertEqual(self.mc.get("nopickle"), None) 290 | 291 | def test_big_list(self): 292 | v = range(1024 * 1024) 293 | r = self.mc.set('big_list', v) 294 | 295 | self.assertEqual(r, True) 296 | self.assertEqual(self.mc.get('big_list'), v) 297 | 298 | def test_last_error(self): 299 | from cmemcached import RETURN_MEMCACHED_SUCCESS 300 | self.assertEqual(self.mc.set('testkey', 'hh'), True) 301 | self.assertEqual(self.mc.get('testkey'), 'hh') 302 | self.assertEqual(self.mc.get_last_error(), RETURN_MEMCACHED_SUCCESS) 303 | self.assertEqual(self.mc.get('testkey1'), None) 304 | self.assertEqual(self.mc.get_last_error(), RETURN_MEMCACHED_SUCCESS) 305 | self.assertEqual(self.mc.get_multi(['testkey']), {'testkey': 'hh'}) 306 | self.assertEqual(self.mc.get_last_error(), RETURN_MEMCACHED_SUCCESS) 307 | self.assertEqual(self.mc.get_multi(['testkey1']), {}) 308 | self.assertEqual(self.mc.get_last_error(), RETURN_MEMCACHED_SUCCESS) 309 | 310 | mc = cmemcached.Client([INVALID_SERVER_ADDR], comp_threshold=1024) 311 | self.assertEqual(mc.set('testkey', 'hh'), False) 312 | self.assertEqual(mc.get('testkey'), None) 313 | self.assertNotEqual(mc.get_last_error(), RETURN_MEMCACHED_SUCCESS) 314 | self.assertEqual(mc.get_multi(['testkey']), {}) 315 | self.assertNotEqual(mc.get_last_error(), RETURN_MEMCACHED_SUCCESS) 316 | 317 | def test_stats(self): 318 | s = self.mc.stats() 319 | self.assertEqual(self.server_addr in s, True) 320 | st = s[self.server_addr] 321 | st_keys = sorted([ 322 | "pid", 323 | "uptime", 324 | "time", 325 | "version", 326 | "pointer_size", 327 | "rusage_user", 328 | "rusage_system", 329 | "curr_items", 330 | "total_items", 331 | "bytes", 332 | "curr_connections", 333 | "total_connections", 334 | "connection_structures", 335 | "cmd_get", 336 | "cmd_set", 337 | "get_hits", 338 | "get_misses", 339 | "evictions", 340 | "bytes_read", 341 | "bytes_written", 342 | "limit_maxbytes", 343 | "threads", 344 | ]) 345 | self.assertEqual(sorted(st.keys()), st_keys) 346 | mc = cmemcached.Client([INVALID_SERVER_ADDR, self.server_addr]) 347 | s = mc.stats() 348 | self.assertEqual(len(s), 2) 349 | 350 | # def test_gets_multi(self): 351 | # keys=["hello1", "hello2", "hello3"] 352 | # values=["vhello1", "vhello2", "vhello3"] 353 | # for x in xrange(3): 354 | # self.mc.set(keys[x], values[x]) 355 | # self.assertEqual(self.mc.get(keys[x]) , values[x]) 356 | # result=self.mc.gets_multi(keys) 357 | # for x in xrange(3): 358 | # print result[keys[x]][0],result[keys[x]][1] 359 | # self.assertEqual(result[keys[x]][0] , values[x]) 360 | 361 | # def test_cas(self): 362 | # keys=["hello1", "hello2", "hello3"] 363 | # values=["vhello1", "vhello2", "vhello3"] 364 | # for x in xrange(3): 365 | # self.mc.set(keys[x], values[x]) 366 | # self.assertEqual(self.mc.get(keys[x]) , values[x]) 367 | # result=self.mc.gets_multi(keys) 368 | # for x in xrange(3): 369 | # self.assertEqual(result[keys[x]][0] , values[x]) 370 | # self.assertEqual(self.mc.cas(keys[x],'cas',cas=result[keys[x]][1]) , 1) 371 | # self.assertEqual(self.mc.cas(keys[x],'cas2',cas=result[keys[x]][1]) , 0) 372 | # self.assertEqual(self.mc.get(keys[x]) , 'cas') 373 | 374 | def test_client_pickable(self): 375 | import pickle 376 | d = pickle.dumps(self.mc) 377 | self.mc = pickle.loads(d) 378 | self.test_stats() 379 | 380 | @skip_if_libmemcached_not_patched 381 | def test_touch(self): 382 | self.mc.set('test', 1) 383 | self.assertEqual(self.mc.get('test'), 1) 384 | self.assertEqual(self.mc.touch('test', -1), 1) 385 | self.assertEqual(self.mc.get('test'), None) 386 | 387 | self.mc.set('test', 1) 388 | self.assertEqual(self.mc.get('test'), 1) 389 | self.assertEqual(self.mc.touch('test', 1), 1) 390 | time.sleep(1) 391 | self.assertEqual(self.mc.get('test'), None) 392 | 393 | def test_ketama(self): 394 | mc = cmemcached.Client( 395 | ['localhost', 'myhost:11211', '127.0.0.1:11212', 'myhost:11213']) 396 | rs = { 397 | 'test:10000': 'localhost', 398 | 'test:20000': '127.0.0.1:11212', 399 | 'test:30000': '127.0.0.1:11212', 400 | 'test:40000': '127.0.0.1:11212', 401 | 'test:50000': '127.0.0.1:11212', 402 | 'test:60000': 'myhost:11213', 403 | 'test:70000': '127.0.0.1:11212', 404 | 'test:80000': '127.0.0.1:11212', 405 | 'test:90000': '127.0.0.1:11212', 406 | } 407 | for k in rs: 408 | self.assertEqual(mc.get_host_by_key(k), rs[k]) 409 | 410 | def test_should_raise_exception_if_called_in_different_thread(self): 411 | catched = [False] 412 | 413 | def f(): 414 | try: 415 | self.mc.set('key_thread', 1) 416 | except cmemcached.ThreadUnsafe: 417 | catched[0] = True 418 | 419 | # make connection in main thread 420 | self.mc.get('key_thread') 421 | 422 | # use it in another thread (should be forbidden) 423 | t = threading.Thread(target=f) 424 | t.start() 425 | t.join() 426 | self.assertEqual(catched, [True]) 427 | 428 | 429 | # class TestUnixSocketCmemcached(TestCmemcached): 430 | # 431 | # def setUp(self): 432 | # os.system('memcached -d -s %s' % TEST_UNIX_SOCKET) 433 | # self.mc=cmemcached.Client([TEST_UNIX_SOCKET], comp_threshold=1024) 434 | # 435 | # def testGetHost(self): 436 | # host = self.mc.get_host_by_key("str") 437 | # self.assertEqual(host, TEST_UNIX_SOCKET) 438 | # 439 | # def test_stats(self): 440 | # "not need" 441 | 442 | class TestBinaryCmemcached(TestCmemcached): 443 | 444 | @pytest.fixture(autouse=True) 445 | def setup(self, memcached): 446 | super(TestBinaryCmemcached, self).setup(memcached) 447 | self.mc = cmemcached.Client([self.server_addr], comp_threshold=1024) 448 | self.mc.set_behavior(cmemcached.BEHAVIOR_BINARY_PROTOCOL, 1) 449 | 450 | def test_append_multi_performance(self): 451 | "binary is slow, bug ?" 452 | 453 | def test_stats(self): 454 | "not yet support" 455 | 456 | def test_touch(self): 457 | "not yet support" 458 | 459 | 460 | class TestPrefix(TestCmemcached): 461 | 462 | @pytest.fixture(autouse=True) 463 | def setup(self, memcached): 464 | super(TestPrefix, self).setup(memcached) 465 | self.prefix = '/prefix' 466 | self.mc = cmemcached.Client([self.server_addr], comp_threshold=1024, 467 | prefix=self.prefix) 468 | 469 | def test_duplicate_prefix_text(self): 470 | for case in ['%sforever/young', 'forever%s/young', 'forever/young/%s']: 471 | nasty_key = case % self.prefix 472 | self.mc.set(nasty_key, 1) 473 | self.assertEqual(self.mc.get(nasty_key), 1) 474 | self.assertEqual(self.mc.get_multi([nasty_key]), {nasty_key: 1}) 475 | 476 | def test_get_host_by_key(self): 477 | mc = cmemcached.Client( 478 | ['localhost', 'myhost:11211', '127.0.0.1:11212', 'myhost:11213'], 479 | prefix=self.prefix 480 | ) 481 | rs = { 482 | 'test:10000': '127.0.0.1:11212', 483 | 'test:20000': 'localhost', 484 | 'test:30000': 'myhost:11213', 485 | 'test:40000': 'myhost:11211', 486 | 'test:50000': 'myhost:11213', 487 | 'test:60000': 'myhost:11213', 488 | 'test:70000': 'localhost', 489 | 'test:80000': 'myhost:11213', 490 | 'test:90000': 'myhost:11213', 491 | } 492 | for k in rs: 493 | print k 494 | self.assertEqual(mc.get_host_by_key(k), rs[k]) 495 | 496 | 497 | if __name__ == '__main__': 498 | unittest.main() 499 | -------------------------------------------------------------------------------- /tests/test_cmemcached_imp.py: -------------------------------------------------------------------------------- 1 | import cmemcached_imp 2 | 3 | INVALID_SERVER_ADDR = '127.0.0.1:1' 4 | 5 | 6 | def test_client_instantiation(): 7 | cmemcached_imp.Client([INVALID_SERVER_ADDR]) 8 | -------------------------------------------------------------------------------- /tests/test_logger.py: -------------------------------------------------------------------------------- 1 | from mock import Mock, patch 2 | import cmemcached 3 | 4 | INVALID_SERVER_ADDR = '127.0.0.1:1' 5 | 6 | 7 | def test_sys_stderr_should_have_no_output_when_no_logger_is_set(memcached): 8 | mc = cmemcached.Client([memcached]) 9 | with patch('sys.stderr') as mock_stderr: 10 | mc.get('test_key_with_no_logger') 11 | mc.set('test_key_with_no_logger', 'test_value_with_no_logger') 12 | assert not mock_stderr.write.called 13 | 14 | 15 | def test_logger_should_be_called_when_set(): 16 | logger = Mock() 17 | client = cmemcached.Client([INVALID_SERVER_ADDR], logger=logger) 18 | client.get('test_key_with_logger') 19 | client.set('test_key_with_logger', 'test_value_with_logger') 20 | logger.assert_called_with( 21 | "[cmemcached]memcached_get: server %s error: CONNECTION FAILURE\n" % 22 | INVALID_SERVER_ADDR) 23 | -------------------------------------------------------------------------------- /tests/test_multi_return_failure.py: -------------------------------------------------------------------------------- 1 | import cmemcached 2 | 3 | INVALID_SERVER_ADDR = '127.0.0.1:1' 4 | 5 | 6 | def test_set_multi_return_failure(): 7 | client = cmemcached.Client([INVALID_SERVER_ADDR]) 8 | v = dict([(str(i), "vv%s" % i) for i in range(10)]) 9 | r, failures = client.set_multi(v, return_failure=True) 10 | assert not r 11 | assert failures == v.keys() 12 | 13 | 14 | def test_delete_multi_return_failure(): 15 | client = cmemcached.Client([INVALID_SERVER_ADDR]) 16 | keys = [str(i) for i in range(10)] 17 | r, failures = client.delete_multi(keys, return_failure=True) 18 | assert not r 19 | assert failures == keys 20 | -------------------------------------------------------------------------------- /tests/test_prefix.py: -------------------------------------------------------------------------------- 1 | #!/usr/local/bin/python2.7 2 | # coding:utf-8 3 | 4 | from cmemcached import Client 5 | import pytest 6 | 7 | PREFIX = 'dae|admin|' 8 | 9 | 10 | @pytest.fixture 11 | def prefixed_mc(memcached): 12 | return Client([memcached], prefix=PREFIX) 13 | 14 | 15 | @pytest.fixture 16 | def raw_mc(memcached): 17 | return Client([memcached]) 18 | 19 | 20 | def test_prefix(prefixed_mc, raw_mc): 21 | raw_mc.delete('a') 22 | prefixed_mc.set('a', 1) 23 | assert(prefixed_mc.get('a') == 1) 24 | assert(raw_mc.get(PREFIX + 'a') == 1) 25 | assert(raw_mc.get('a') is None) 26 | 27 | prefixed_mc.add('b', 2) 28 | assert(prefixed_mc.get('b') == 2) 29 | assert(raw_mc.get(PREFIX + 'b') == 2) 30 | assert(raw_mc.get('b') is None) 31 | 32 | prefixed_mc.incr('b') 33 | assert(prefixed_mc.get('b') == 3) 34 | assert(raw_mc.get(PREFIX + 'b') == 3) 35 | 36 | raw_mc.decr(PREFIX + 'b') 37 | assert(prefixed_mc.get('b') == 2) 38 | 39 | prefixed_mc.set_multi({'x': 'a', 'y': 'b'}) 40 | ret = prefixed_mc.get_multi(['x', 'y']) 41 | assert(ret.get('x') == 'a' and ret.get('y') == 'b') 42 | assert(prefixed_mc.delete_multi(['a', 'b', 'x', 'y'])) 43 | -------------------------------------------------------------------------------- /tests/test_set_long.py: -------------------------------------------------------------------------------- 1 | def test_set_get_long(mc): 2 | mc.set("key_long_short", long(1L)) 3 | v = mc.get("key_long_short") 4 | assert v == 1L 5 | assert isinstance(v, long) 6 | 7 | big = 1233345435353543L 8 | mc.set("key_long_big", big) 9 | v = mc.get("key_long_big") 10 | assert v == big 11 | assert isinstance(v, long) 12 | --------------------------------------------------------------------------------