├── .github └── workflows │ └── tests.yaml ├── LICENSE ├── MANIFEST.in ├── README.md ├── setup.py ├── tests.py └── ucache.py /.github/workflows/tests.yaml: -------------------------------------------------------------------------------- 1 | name: Tests 2 | on: [push] 3 | jobs: 4 | tests: 5 | name: ${{ matrix.python-version }} 6 | runs-on: ubuntu-16.04 7 | services: 8 | redis: 9 | image: redis 10 | ports: 11 | - 6379:6379 12 | memcached: 13 | image: memcached 14 | ports: 15 | - 11211:11211 16 | strategy: 17 | fail-fast: false 18 | matrix: 19 | python-version: [3.7, "3.10", "3.11"] 20 | steps: 21 | - uses: actions/checkout@v2 22 | - uses: actions/setup-python@v2 23 | with: 24 | python-version: ${{ matrix.python-version }} 25 | - name: package deps 26 | run: | 27 | sudo apt-get install kyototycoon libkyotocabinet-dev libkyototycoon-dev libmemcached-dev 28 | wget https://dbmx.net/kyotocabinet/pythonpkg/kyotocabinet-python-1.23.tar.gz 29 | tar xzf kyotocabinet-python-1.23.tar.gz 30 | cd kyotocabinet-python-1.23 && python setup.py install && cd ../ 31 | - name: pip deps 32 | run: | 33 | pip install cython gevent msgpack-python 34 | pip install greendb peewee pylibmc pymemcache redis ukt 35 | - name: runtests 36 | run: | 37 | ktserver -le& 38 | greendb.py& 39 | python tests.py 40 | pkill ktserver 41 | pkill greendb.py 42 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2010 Charles Leifer 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining a copy 4 | of this software and associated documentation files (the "Software"), to deal 5 | in the Software without restriction, including without limitation the rights 6 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | copies of the Software, and to permit persons to whom the Software is 8 | furnished to do so, subject to the following conditions: 9 | 10 | The above copyright notice and this permission notice shall be included in 11 | all copies or substantial portions of the Software. 12 | 13 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 19 | THE SOFTWARE. 20 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include LICENSE 2 | include README.md 3 | include tests.py 4 | recursive-include docs * 5 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | ![](http://media.charlesleifer.com/blog/photos/ucache-logo-0.png) 2 | 3 | ucache is a lightweight and efficient caching library for python. 4 | 5 | * [Kyoto Tycoon](https://fallabs.com/kyototycoon/) via [kt](https://github.com/coleifer/kt) 6 | * [Redis](https://redis.io) via [redis-py](https://github.com/andymccurdy/redis-py) 7 | * [Sqlite](https://www.sqlite.org/) via [peewee](https://github.com/coleifer/peewee) 8 | * [Kyoto Cabinet](https://fallabs.com/kyotocabinet/) via [kyotocabinet-python bindings](https://fallabs.com/kyotocabinet/pythondoc/) 9 | * [DBM](https://en.wikipedia.org/wiki/DBM_(computing)) via [dbm module from standard library](https://docs.python.org/3/library/dbm.html) 10 | * Simple in-memory cache using Python dictionary. 11 | 12 | Features: 13 | 14 | * Pickle serialization by default, msgpack also supported 15 | * Optional (transparent) compression using `zlib` 16 | * Efficient bulk-operations. 17 | * Preload context manager for efficient pre-loading of cached data 18 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | import os 2 | from setuptools import setup 3 | 4 | 5 | with open(os.path.join(os.path.dirname(__file__), 'README.md')) as fh: 6 | readme = fh.read() 7 | 8 | 9 | setup( 10 | name='ucache', 11 | version=__import__('ucache').__version__, 12 | description='lightweight and efficient caching library', 13 | long_description=readme, 14 | author='Charles Leifer', 15 | author_email='coleifer@gmail.com', 16 | url='http://github.com/coleifer/ucache/', 17 | packages=[], 18 | py_modules=['ucache'], 19 | test_suite='tests') 20 | -------------------------------------------------------------------------------- /tests.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | import glob 4 | import os 5 | import sys 6 | import time 7 | import unittest 8 | 9 | from ucache import * 10 | 11 | 12 | class BaseTestCache(object): 13 | cache_files = [] 14 | 15 | def get_cache(self, **kwargs): 16 | raise NotImplementedError 17 | 18 | def cleanup(self): 19 | for filename in self.cache_files: 20 | if os.path.exists(filename): 21 | os.unlink(filename) 22 | 23 | def setUp(self): 24 | self.cache = self.get_cache() 25 | super(BaseTestCache, self).setUp() 26 | 27 | def tearDown(self): 28 | self.cache.set_prefix() 29 | self.cache.close() 30 | self.cleanup() 31 | super(BaseTestCache, self).tearDown() 32 | 33 | def test_operations(self): 34 | test_data = ( 35 | ('k1', 'v1'), 36 | ('k2', 2), 37 | ('k3', None), 38 | ('k4', [0, '1', [2]]), 39 | ('k5', {'6': ['7', 8, {'9': '10', '11': 12}]}), 40 | ) 41 | test_data_dict = dict(test_data) 42 | 43 | for key, value in test_data: 44 | self.cache.set(key, value, 60) 45 | 46 | for key, value in test_data: 47 | self.assertEqual(self.cache.get(key), value) 48 | 49 | self.cache.delete('k1') 50 | self.cache.delete('k3') 51 | self.cache.delete('k5') 52 | 53 | for key in ('k1', 'k3', 'k5'): 54 | self.assertIsNone(self.cache.get(key)) 55 | 56 | for key in ('k2', 'k4'): 57 | self.assertEqual(self.cache.get(key), test_data_dict[key]) 58 | 59 | self.cache.flush() 60 | self.assertIsNone(self.cache.get('k2')) 61 | self.assertIsNone(self.cache.get('k4')) 62 | 63 | def test_bulk_operations(self): 64 | test_data = { 65 | 'k1': 'v1', 66 | 'k2': 2, 67 | 'k3': [0, '1', [2]]} 68 | 69 | # Do simple bulk-set. 70 | self.cache.set_many(test_data, timeout=60) 71 | 72 | # Do single-set to ensure compatible with bulk-get. 73 | self.cache.set('k4', 'v4') 74 | 75 | # Compare results of bulk-get. 76 | self.assertEqual(self.cache.get_many(['k1', 'k2', 'k3', 'k4']), { 77 | 'k1': 'v1', 78 | 'k2': 2, 79 | 'k3': [0, '1', [2]], 80 | 'k4': 'v4'}) 81 | 82 | # Do individual gets to ensure methods are compatible. 83 | self.assertEqual(self.cache.get('k1'), test_data['k1']) 84 | self.assertEqual(self.cache.get('k3'), test_data['k3']) 85 | 86 | # Do bulk-delete. 87 | self.cache.delete_many(['k1', 'k3', 'kx']) 88 | self.assertTrue(self.cache['k1'] is None) 89 | self.assertTrue(self.cache['k2'] is not None) 90 | self.assertTrue(self.cache['k3'] is None) 91 | 92 | self.assertEqual(self.cache.get_many(['k1', 'k2', 'k3']), {'k2': 2}) 93 | 94 | # Do single-delete to ensure compatibility. 95 | self.cache.delete('k2') 96 | self.assertTrue(self.cache['k2'] is None) 97 | 98 | def test_preload(self): 99 | self.cache.set_many({'k1': 'v1', 'k2': 'v2', 'k3': 'v3'}, timeout=60) 100 | self.assertEqual(self.cache.get('k1'), 'v1') 101 | self.assertTrue(self.cache.get('kx') is None) 102 | 103 | with self.cache.preload(['k1', 'k3']): 104 | self.assertEqual(self.cache.get('k1'), 'v1') 105 | self.assertEqual(self.cache.get('k3'), 'v3') 106 | self.assertTrue(self.cache.get('kx') is None) 107 | 108 | self.cache._preload['kx'] = 'preloaded' 109 | self.assertEqual(self.cache.get('kx'), 'preloaded') 110 | 111 | self.assertEqual(self.cache.get('k1'), 'v1') 112 | self.assertEqual(self.cache.get('k2'), 'v2') 113 | self.assertEqual(self.cache.get('k3'), 'v3') 114 | self.assertTrue(self.cache.get('kx') is None) 115 | 116 | def assertWrites(self, n): 117 | self.assertEqual(self.cache.stats['writes'], n) 118 | def assertHits(self, n): 119 | self.assertEqual(self.cache.stats['hits'], n) 120 | def assertPLHits(self, n): 121 | self.assertEqual(self.cache.stats['preload_hits'], n) 122 | 123 | def test_preload_re_set(self): 124 | self.cache.set_many({'k1': 'v1', 'k2': 'v2', 'k3': 'v3'}, timeout=60) 125 | self.assertWrites(3) 126 | with self.cache.preload(['k1', 'k2']): 127 | self.assertHits(2) 128 | with self.cache.preload(['k3']): 129 | self.assertHits(3) 130 | self.assertPLHits(0) 131 | 132 | self.assertEqual(self.cache.get('k1'), 'v1') 133 | self.assertEqual(self.cache.get('k2'), 'v2') 134 | self.assertEqual(self.cache.get('k3'), 'v3') 135 | 136 | # No more actual trips to the backend - we are pulling from the 137 | # preload cache. 138 | self.assertHits(3) 139 | self.assertPLHits(3) 140 | 141 | self.cache.set('k2', 'v2-x') 142 | self.assertWrites(4) 143 | self.assertEqual(self.cache.get('k2'), 'v2-x') 144 | self.assertHits(3) 145 | self.assertPLHits(4) 146 | 147 | # We lost the scope that k2 was set in, and get a stale value back. 148 | self.assertEqual(self.cache.get('k2'), 'v2') 149 | self.assertHits(3) 150 | self.assertPLHits(5) 151 | 152 | # Lost scope for k3, make trip to the cache. 153 | self.assertEqual(self.cache.get('k3'), 'v3') 154 | self.assertHits(4) 155 | self.assertPLHits(5) 156 | 157 | def test_decorator(self): 158 | @self.cache.cached(10) 159 | def fn(seed=None): 160 | return time.time() 161 | 162 | value = fn() 163 | time.sleep(0.001) 164 | self.assertEqual(fn(), value) 165 | 166 | fn.bust() 167 | self.assertFalse(fn() == value) 168 | self.assertEqual(fn(), fn()) 169 | self.assertFalse(fn(1) == fn(2)) 170 | self.assertEqual(fn(2), fn(2)) 171 | 172 | def test_property(self): 173 | class Dummy(object): 174 | @self.cache.cached_property 175 | def fn(self): 176 | return time.time() 177 | 178 | d = Dummy() 179 | value = d.fn 180 | time.sleep(0.001) 181 | self.assertEqual(d.fn, value) 182 | 183 | def test_compression(self): 184 | self.cache.close() 185 | self.cleanup() 186 | cache = self.get_cache(compression=True) 187 | data = {'k1': 'a' * 1024, 'k2': 'b' * 512, 'k3': 'c' * 200} 188 | cache.set_many(data, timeout=60) 189 | cache.set('k4', 'd' * 1024, timeout=60) 190 | 191 | self.assertEqual(cache.get('k4'), 'd' * 1024) 192 | res = cache.get_many(['k1', 'k2', 'k3']) 193 | self.assertEqual(res, data) 194 | cache.delete_many(['k1', 'k2', 'k3', 'k4']) 195 | 196 | def test_read_expired(self): 197 | self.cache.set('k1', 'v1', -1) 198 | self.assertTrue(self.cache.get('k1') is None) 199 | 200 | def test_clean_expired(self): 201 | if not self.cache.manual_expire: 202 | return 203 | 204 | day = 86400 205 | for i in range(1, 7): 206 | self.cache.set('k%s' % i, 'v%s' % i, (-i * day) - 1) 207 | 208 | self.cache.set('ka', 'va', -5) 209 | self.cache.set('kb', 'vb', 60) 210 | self.cache.set('kc', 'vc', day) 211 | 212 | # k1, -1 days ... k6, -6 days. 213 | self.assertTrue(self.cache.get('k4') is None) # k4 is also deleted. 214 | self.assertEqual(self.cache.clean_expired(3), 3) # k3, k5, k6. 215 | self.assertEqual(self.cache.clean_expired(3), 0) 216 | 217 | self.assertEqual(self.cache.clean_expired(1), 2) # k1, k2. 218 | self.assertEqual(self.cache.clean_expired(), 1) # ka. 219 | self.assertEqual(self.cache.clean_expired(), 0) 220 | 221 | # Cannot retrieve any of the expired data. 222 | for i in range(1, 7): 223 | self.assertTrue(self.cache.get('k%s' % i) is None) 224 | 225 | # Set some new expired keys and values. 226 | for i in range(3): 227 | self.cache.set('k%s' % i, 'v%s' % i, -3) 228 | 229 | self.assertTrue(self.cache.get('k1') is None) 230 | self.assertEqual(self.cache.clean_expired(), 2) 231 | self.assertEqual(self.cache.clean_expired(), 0) 232 | 233 | # Set expired key to a valid time. 234 | self.cache.set('k1', 'v1', 60) 235 | self.assertEqual(self.cache.get('k1'), 'v1') 236 | 237 | # Our original keys are still present. 238 | self.assertEqual(self.cache.get('kb'), 'vb') 239 | self.assertEqual(self.cache.get('kc'), 'vc') 240 | 241 | def test_prefix_and_flush(self): 242 | self.cache.set_prefix('a') 243 | self.cache.set('k0', 'v0-1') 244 | self.cache.set('k1', 'v1-1') 245 | 246 | self.cache.set_prefix('b') 247 | self.cache.set('k0', 'v0-2') 248 | 249 | # Check that keys and values are isolated properly by prefix. 250 | self.cache.set_prefix('a') 251 | self.assertEqual(self.cache.get('k0'), 'v0-1') 252 | 253 | self.cache.set_prefix('b') 254 | self.assertEqual(self.cache.get('k0'), 'v0-2') 255 | 256 | self.cache.set_prefix('a') 257 | try: 258 | self.cache.flush() 259 | except NotImplementedError: 260 | # Memcached does not support prefix match, so we skip. 261 | return 262 | 263 | self.assertTrue(self.cache.get('k0') is None) 264 | self.assertTrue(self.cache.get('k1') is None) 265 | 266 | self.cache.set_prefix('b') 267 | self.assertEqual(self.cache.get('k0'), 'v0-2') 268 | self.assertTrue(self.cache.get('k1') is None) 269 | 270 | 271 | class TestKTCache(BaseTestCache, unittest.TestCase): 272 | def cleanup(self): 273 | self.cache.close(close_all=True) 274 | 275 | def get_cache(self, **kwargs): 276 | return KTCache(connection_pool=False, **kwargs) 277 | 278 | 279 | class TestSqliteCache(BaseTestCache, unittest.TestCase): 280 | cache_files = ['sqlite_cache.db'] 281 | 282 | def get_cache(self, **kwargs): 283 | return SqliteCache('sqlite_cache.db', **kwargs) 284 | 285 | 286 | class TestRedisCache(BaseTestCache, unittest.TestCase): 287 | def get_cache(self, **kwargs): 288 | return RedisCache(**kwargs) 289 | 290 | def test_read_expired(self): 291 | # Redis doesn't support setting a negative timeout. 292 | pass 293 | 294 | 295 | class TestKCCache(BaseTestCache, unittest.TestCase): 296 | def get_cache(self, **kwargs): 297 | return KCCache(filename='*', **kwargs) 298 | 299 | 300 | class TestMemcacheCache(BaseTestCache, unittest.TestCase): 301 | def get_cache(self, **kwargs): 302 | return MemcacheCache(**kwargs) 303 | 304 | 305 | class TestPyMemcacheCache(BaseTestCache, unittest.TestCase): 306 | def get_cache(self, **kwargs): 307 | return PyMemcacheCache(**kwargs) 308 | 309 | 310 | class TestMemoryCache(BaseTestCache, unittest.TestCase): 311 | def get_cache(self, **kwargs): 312 | return MemoryCache(**kwargs) 313 | 314 | 315 | class TestDbmCache(BaseTestCache, unittest.TestCase): 316 | @property 317 | def cache_files(self): 318 | return glob.glob('dbmcache.*') 319 | 320 | def get_cache(self, **kwargs): 321 | return DbmCache('dbmcache.db', **kwargs) 322 | 323 | 324 | class TestGreenDBCache(BaseTestCache, unittest.TestCase): 325 | def get_cache(self, **kwargs): 326 | return GreenDBCache(**kwargs) 327 | 328 | 329 | if __name__ == '__main__': 330 | unittest.main(argv=sys.argv) 331 | -------------------------------------------------------------------------------- /ucache.py: -------------------------------------------------------------------------------- 1 | from collections import ChainMap 2 | from collections import Counter 3 | from contextlib import contextmanager 4 | import atexit 5 | import functools 6 | import hashlib 7 | import pickle 8 | import struct 9 | import threading 10 | import time 11 | import zlib 12 | try: 13 | import msgpack 14 | except ImportError: 15 | msgpack = None 16 | 17 | 18 | __version__ = '0.1.4' 19 | __all__ = [ 20 | 'DbmCache', 21 | 'GreenDBCache', 22 | 'KCCache', 23 | 'KTCache', 24 | 'MemcacheCache', 25 | 'MemoryCache', 26 | 'PyMemcacheCache', 27 | 'RedisCache', 28 | 'SqliteCache', 29 | 'UC_NONE', 30 | 'UC_MSGPACK', 31 | 'UC_PICKLE', 32 | ] 33 | 34 | 35 | class UCacheException(Exception): pass 36 | class ImproperlyConfigured(UCacheException): pass 37 | 38 | 39 | pdumps = lambda o: pickle.dumps(o, pickle.HIGHEST_PROTOCOL) 40 | ploads = pickle.loads 41 | 42 | if msgpack is not None: 43 | mpackb = lambda o: msgpack.packb(o, use_bin_type=True) 44 | munpackb = lambda b: msgpack.unpackb(b, raw=False) 45 | 46 | 47 | def with_compression(pack, unpack, threshold=256): 48 | """ 49 | Given packing and unpacking routines, return new versions that 50 | transparently compress and decompress the serialized data. 51 | """ 52 | def new_pack(value): 53 | flag = b'\x01' 54 | data = pack(value) 55 | if len(data) > threshold: 56 | flag = b'\x02' 57 | data = zlib.compress(data) 58 | return flag + data 59 | 60 | def new_unpack(data): 61 | if not data: return data 62 | buf = memoryview(data) # Avoid copying data. 63 | flag = buf[0] 64 | rest = buf[1:] 65 | if flag == 2: 66 | pdata = zlib.decompress(rest) 67 | elif flag == 1: 68 | pdata = rest 69 | else: 70 | # Backwards-compatibility / safety. If the value is not prefixed, 71 | # just deserialize all of it. 72 | pdata = buf 73 | return unpack(pdata) 74 | 75 | return new_pack, new_unpack 76 | 77 | ts_struct = struct.Struct('>Q') # 64-bit integer, timestamp in milliseconds. 78 | 79 | def encode_timestamp(data, ts): 80 | ts_buf = ts_struct.pack(int(ts * 1000)) 81 | return ts_buf + data 82 | 83 | def decode_timestamp(data): 84 | mv = memoryview(data) 85 | ts_msec, = ts_struct.unpack(mv[:8]) 86 | return ts_msec / 1000., mv[8:] 87 | 88 | __sentinel__ = object() 89 | 90 | def chunked(it, n): 91 | for group in (list(g) for g in izip_longest(*[iter(it)] * n, 92 | fillvalue=__sentinel__)): 93 | if group[-1] is __sentinel__: 94 | del group[group.index(__sentinel__):] 95 | yield group 96 | 97 | 98 | decode = lambda s: s.decode('utf8') if isinstance(s, bytes) else s 99 | encode = lambda s: s.encode('utf8') if isinstance(s, str) else s 100 | 101 | 102 | UC_NONE = 0 103 | UC_PICKLE = 1 104 | UC_MSGPACK = 2 105 | 106 | 107 | class CacheStats(object): 108 | __slots__ = ('preload_hits', 'hits', 'misses', 'writes') 109 | def __init__(self): 110 | self.preload_hits = self.hits = self.misses = self.writes = 0 111 | 112 | def as_dict(self): 113 | return { 114 | 'preload_hits': self.preload_hits, 115 | 'hits': self.hits, 116 | 'misses': self.misses, 117 | 'writes': self.writes} 118 | 119 | 120 | class Cache(object): 121 | # Storage layers that do not support expiration can still be used, in which 122 | # case we include the timestamp in the value and transparently handle 123 | # serialization/deserialization. Periodic cleanup is necessary, however. 124 | manual_expire = False 125 | 126 | def __init__(self, prefix=None, timeout=60, debug=False, compression=False, 127 | compression_len=256, serializer=UC_PICKLE, connect=True, 128 | **params): 129 | self.set_prefix(prefix) 130 | self.timeout = timeout 131 | self.debug = debug 132 | self.compression = compression 133 | self.serializer = serializer 134 | self.params = params 135 | self._preload = ChainMap() # Stack of preload data. 136 | self._stats = CacheStats() 137 | 138 | if self.serializer == UC_NONE: 139 | pack = unpack = lambda o: o 140 | elif self.serializer == UC_PICKLE: 141 | pack, unpack = pdumps, ploads 142 | elif self.serializer == UC_MSGPACK: 143 | pack, unpack = mpackb, munpackb 144 | 145 | if compression: 146 | pack, unpack = with_compression(pack, unpack, compression_len) 147 | 148 | self.pack = pack 149 | self.unpack = unpack 150 | 151 | if connect: 152 | self.open() 153 | 154 | def set_prefix(self, prefix=None): 155 | self.prefix = encode(prefix or '') 156 | self._prefix_len = len(self.prefix) 157 | 158 | @contextmanager 159 | def preload(self, keys): 160 | self._preload = self._preload.new_child(self.get_many(keys)) 161 | yield 162 | self._preload = self._preload.parents 163 | 164 | def open(self): 165 | pass 166 | 167 | def close(self): 168 | pass 169 | 170 | @property 171 | def stats(self): 172 | return self._stats.as_dict() 173 | 174 | def prefix_key(self, key): 175 | if self.prefix: 176 | return b'%s.%s' % (self.prefix, encode(key)) 177 | else: 178 | return encode(key) 179 | 180 | def unprefix_key(self, key): 181 | if self.prefix: 182 | return decode(key[self._prefix_len + 1:]) 183 | else: 184 | return decode(key) 185 | 186 | def get(self, key): 187 | if self.debug: return 188 | 189 | if len(self._preload.maps) > 1 and key in self._preload: 190 | self._stats.preload_hits += 1 191 | return self._preload[key] 192 | 193 | data = self._get(self.prefix_key(key)) 194 | if data is None: 195 | self._stats.misses += 1 196 | return 197 | 198 | if self.manual_expire: 199 | ts, value = decode_timestamp(data) 200 | if ts >= time.time(): 201 | self._stats.hits += 1 202 | return self.unpack(value) 203 | else: 204 | self._stats.misses += 1 205 | self.delete(key) 206 | else: 207 | self._stats.hits += 1 208 | return self.unpack(data) 209 | 210 | def _get(self, key): 211 | raise NotImplementedError 212 | 213 | def get_many(self, keys): 214 | if self.debug: return 215 | 216 | nkeys = len(keys) 217 | prefix_keys = [self.prefix_key(key) for key in keys] 218 | bulk_data = self._get_many(prefix_keys) 219 | accum = {} 220 | hits = 0 221 | 222 | # For tracking keys when manual_expire is true. 223 | timestamp = time.time() 224 | expired = [] 225 | 226 | for key, data in bulk_data.items(): 227 | if data is None: continue 228 | 229 | if self.manual_expire: 230 | ts, value = decode_timestamp(data) 231 | if ts >= timestamp: 232 | accum[self.unprefix_key(key)] = self.unpack(value) 233 | hits += 1 234 | else: 235 | expired.append(key) 236 | else: 237 | accum[self.unprefix_key(key)] = self.unpack(data) 238 | hits += 1 239 | 240 | if expired: 241 | # Handle cleaning-up expired keys. Only applies to manual_expire. 242 | self.delete_many(expired) 243 | 244 | # Update stats. 245 | self._stats.hits += hits 246 | self._stats.misses += (nkeys - hits) 247 | return accum 248 | 249 | def _get_many(self, keys): 250 | raise NotImplementedError 251 | 252 | def set(self, key, value, timeout=None): 253 | if self.debug: return 254 | 255 | if len(self._preload.maps) > 1: 256 | self._preload[key] = value 257 | 258 | timeout = timeout if timeout is not None else self.timeout 259 | data = self.pack(value) 260 | 261 | if self.manual_expire: 262 | # Encode the expiration timestamp as the first 8 bytes of the 263 | # cached value. 264 | data = encode_timestamp(data, time.time() + timeout) 265 | 266 | self._stats.writes += 1 267 | return self._set(self.prefix_key(key), data, timeout) 268 | 269 | def _set(self, key, value, timeout): 270 | raise NotImplementedError 271 | 272 | def set_many(self, __data=None, timeout=None, **kwargs): 273 | if self.debug: return 274 | 275 | timeout = timeout if timeout is not None else self.timeout 276 | if __data is not None: 277 | kwargs.update(__data) 278 | 279 | if len(self._preload.maps) > 1: 280 | self._preload.update(kwargs) 281 | 282 | accum = {} 283 | expires = time.time() + timeout 284 | 285 | for key, value in kwargs.items(): 286 | data = self.pack(value) 287 | if self.manual_expire: 288 | data = encode_timestamp(data, expires) 289 | accum[self.prefix_key(key)] = data 290 | 291 | self._stats.writes += len(accum) 292 | return self._set_many(accum, timeout) 293 | 294 | def _set_many(self, data, timeout): 295 | raise NotImplementedError 296 | 297 | def delete(self, key): 298 | if self.debug: return 299 | if len(self._preload.maps) > 1: self._preload.pop(key, None) 300 | return self._delete(self.prefix_key(key)) 301 | 302 | def _delete(self, key): 303 | raise NotImplementedError 304 | 305 | def delete_many(self, keys): 306 | if self.debug: return 307 | if len(self._preload.maps) > 1: 308 | for key in keys: 309 | self._preload.pop(key, None) 310 | return self._delete_many([self.prefix_key(key) for key in keys]) 311 | 312 | def _delete_many(self, keys): 313 | raise NotImplementedError 314 | 315 | __getitem__ = get 316 | __setitem__ = set 317 | __delitem__ = delete 318 | 319 | def flush(self): 320 | if self.debug: return 321 | return self._flush() 322 | 323 | def _flush(self): 324 | raise NotImplementedError 325 | 326 | def clean_expired(self, ndays=0): 327 | n = 0 328 | if self.manual_expire: 329 | cutoff = time.time() - (ndays * 86400) 330 | for expired_key in self.get_expired_keys(cutoff): 331 | self._delete(expired_key) 332 | n += 1 333 | return n 334 | 335 | def get_expired_keys(self, cutoff): 336 | raise NotImplementedError 337 | 338 | def _key_fn(a, k): 339 | # Generic function for converting an arbitrary function call's 340 | # arguments (and keyword args) into a consistent hash key. 341 | return hashlib.md5(pdumps((a, k))).hexdigest() 342 | 343 | def cached(self, timeout=None, key_fn=_key_fn): 344 | def decorator(fn): 345 | def make_key(args, kwargs): 346 | return '%s:%s' % (fn.__name__, key_fn(args, kwargs)) 347 | 348 | def bust(*a, **k): 349 | self.delete(make_key(a, k)) 350 | 351 | @functools.wraps(fn) 352 | def inner(*a, **k): 353 | key = make_key(a, k) 354 | res = self.get(key) 355 | if res is None: 356 | res = fn(*a, **k) 357 | self.set(key, res, timeout) 358 | return res 359 | 360 | inner.bust = bust 361 | inner.make_key = make_key 362 | return inner 363 | return decorator 364 | 365 | def cached_property(self, timeout=None, key_fn=_key_fn): 366 | this = self 367 | class _cached_property(object): 368 | def __init__(self, fn): 369 | self._fn = this.cached(timeout=timeout, key_fn=key_fn)(fn) 370 | def __get__(self, instance, instance_type=None): 371 | if instance is None: 372 | return self 373 | return self._fn(instance) 374 | def __delete__(self, obj): 375 | self._fn.bust(obj) 376 | def __set__(self, instance, value): 377 | raise ValueError('Cannot set value of a cached property.') 378 | def decorator(fn): 379 | return _cached_property(fn) 380 | return decorator 381 | 382 | 383 | class DummyLock(object): 384 | def __enter__(self): 385 | return self 386 | def __exit__(self, exc_type, exc_val, exc_tb): 387 | pass 388 | 389 | 390 | class MemoryCache(Cache): 391 | manual_expire = True 392 | 393 | def __init__(self, thread_safe=True, *args, **kwargs): 394 | self._data = {} 395 | if thread_safe: 396 | self._lock = threading.RLock() 397 | else: 398 | self._lock = DummyLock() 399 | super(MemoryCache, self).__init__(*args, **kwargs) 400 | 401 | def _get(self, key): 402 | with self._lock: 403 | return self._data.get(key) 404 | 405 | def _get_many(self, keys): 406 | with self._lock: 407 | return {key: self._data[key] for key in keys if key in self._data} 408 | 409 | def _set(self, key, value, timeout): 410 | with self._lock: 411 | self._data[key] = value # Ignore timeout, it is packed in value. 412 | 413 | def _set_many(self, data, timeout): 414 | with self._lock: 415 | self._data.update(data) 416 | 417 | def _delete(self, key): 418 | with self._lock: 419 | if key in self._data: 420 | del self._data[key] 421 | 422 | def _delete_many(self, keys): 423 | with self._lock: 424 | for key in keys: 425 | if key in self._data: 426 | del self._data[key] 427 | 428 | def _flush(self): 429 | with self._lock: 430 | if not self.prefix: 431 | self._data = {} 432 | else: 433 | self._data = {k: v for k, v in self._data.items() 434 | if not k.startswith(self.prefix)} 435 | return True 436 | 437 | def clean_expired(self, ndays=0): 438 | timestamp = time.time() - (ndays * 86400) 439 | n = 0 440 | 441 | with self._lock: 442 | for key, value in list(self._data.items()): 443 | ts, _ = decode_timestamp(value) 444 | if ts <= timestamp: 445 | del self._data[key] 446 | n += 1 447 | return n 448 | 449 | 450 | try: 451 | from ukt import KT_NONE 452 | from ukt import KyotoTycoon 453 | except ImportError: 454 | KyotoTycoon = None 455 | 456 | 457 | class KTCache(Cache): 458 | def __init__(self, host='127.0.0.1', port=1978, db=0, client_timeout=5, 459 | connection=None, no_reply=False, **params): 460 | if KyotoTycoon is None: 461 | raise ImproperlyConfigured('Cannot use KTCache - ukt python ' 462 | 'module is not installed.') 463 | 464 | self._host = host 465 | self._port = port 466 | self._db = db 467 | self._client_timeout = client_timeout 468 | self._no_reply = no_reply 469 | 470 | if connection is not None: 471 | self._client = connection 472 | else: 473 | self._client = self._get_client() 474 | super(KTCache, self).__init__(**params) 475 | 476 | def _get_client(self): 477 | return KyotoTycoon(host=self._host, port=self._port, 478 | timeout=self._client_timeout, default_db=self._db, 479 | serializer=KT_NONE) 480 | 481 | def close(self, close_all=False): 482 | if close_all: 483 | return self._client.close_all() 484 | 485 | def _get(self, key): 486 | return self._client.get(key, self._db, decode_value=False) 487 | 488 | def _get_many(self, keys): 489 | return self._client.get_bulk(keys, self._db, decode_values=False) 490 | 491 | def _set(self, key, value, timeout): 492 | return self._client.set(key, value, self._db, timeout, self._no_reply, 493 | encode_value=False) 494 | 495 | def _set_many(self, data, timeout): 496 | return self._client.set_bulk(data, self._db, timeout, self._no_reply, 497 | encode_values=False) 498 | 499 | def _delete(self, key): 500 | return self._client.remove(key, self._db, self._no_reply) 501 | 502 | def _delete_many(self, keys): 503 | return self._client.remove_bulk(keys, self._db, self._no_reply) 504 | 505 | def touch(self, key, timeout=None): 506 | # NOTE: requires ukt running with the scripts/kt.lua script. 507 | if self.debug: return 508 | 509 | timeout = timeout if timeout is not None else self.timeout 510 | return self._client.touch(self.prefix_key(key), timeout, db=self._db) 511 | 512 | def touch_many(self, keys, timeout=None): 513 | # NOTE: requires ukt running with the scripts/kt.lua script. 514 | if self.debug: return 515 | 516 | timeout = timeout if timeout is not None else self.timeout 517 | prefix_keys = [self.prefix_key(key) for key in keys] 518 | out = self._client.touch_bulk(prefix_keys, timeout, db=self._db) 519 | return {self.unprefix_key(key): value for key, value in out.items()} 520 | 521 | def _flush(self): 522 | if not self.prefix: 523 | return self._client.clear() 524 | 525 | n = 0 526 | while True: 527 | keys = self._client.match_prefix(self.prefix, 1000, self._db) 528 | if not keys: 529 | break 530 | n += self._client.remove_bulk(keys, self._db, self._no_reply) 531 | return n 532 | 533 | 534 | try: 535 | import kyotocabinet as kc 536 | except ImportError: 537 | kc = None 538 | 539 | 540 | class KCCache(Cache): 541 | manual_expire = True 542 | 543 | def __init__(self, filename, **params): 544 | self._filename = filename 545 | self._kc = None 546 | if kc is None: 547 | raise ImproperlyConfigured('Cannot use KCCache, kyotocabinet ' 548 | 'python bindings are not installed.') 549 | super(KCCache, self).__init__(**params) 550 | 551 | def open(self): 552 | if self._kc is not None: 553 | return False 554 | 555 | self._kc = kc.DB() 556 | mode = kc.DB.OWRITER | kc.DB.OCREATE | kc.DB.OTRYLOCK 557 | if not self._kc.open(self._filename, mode): 558 | raise UCacheException('kyotocabinet could not open cache ' 559 | 'database: "%s"' % self._filename) 560 | return True 561 | 562 | def close(self): 563 | if self._kc is None: return False 564 | self._kc.synchronize(True) 565 | if not self._kc.close(): 566 | raise UCacheException('kyotocabinet error while closing cache ' 567 | 'database: "%s"' % self._filename) 568 | self._kc = None 569 | return True 570 | 571 | def _get(self, key): 572 | return self._kc.get(key) 573 | 574 | def _get_many(self, keys): 575 | return self._kc.get_bulk(keys) 576 | 577 | def _set(self, key, value, timeout): 578 | return self._kc.set(key, value) 579 | 580 | def _set_many(self, data, timeout): 581 | return self._kc.set_bulk(data) 582 | 583 | def _delete(self, key): 584 | return self._kc.remove(key) 585 | 586 | def _delete_many(self, keys): 587 | return self._kc.remove_bulk(keys) 588 | 589 | def _flush(self): 590 | if not self.prefix: 591 | self._kc.clear() 592 | else: 593 | keys = self._kc.match_prefix(self.prefix) 594 | if keys: 595 | self._kc.remove_bulk(keys) 596 | return self._kc.synchronize() 597 | 598 | def clean_expired(self, ndays=0): 599 | timestamp = time.time() - (ndays * 86400) 600 | 601 | class Visitor(kc.Visitor): 602 | n_deleted = 0 603 | def visit_full(self, key, value): 604 | ts, _ = decode_timestamp(value) 605 | if ts <= timestamp: 606 | self.n_deleted += 1 607 | return self.REMOVE 608 | else: 609 | return self.NOP 610 | 611 | def visit_empty(self, key): 612 | return self.NOP 613 | 614 | visitor = Visitor() 615 | if not self._kc.iterate(visitor, True): 616 | raise UCacheException('kyotocabinet: error cleaning expired keys.') 617 | 618 | return visitor.n_deleted 619 | 620 | 621 | try: 622 | from peewee import * 623 | try: 624 | from playhouse.sqlite_ext import CSqliteExtDatabase as SqliteDatabase 625 | except ImportError: 626 | from playhouse.sqlite_ext import SqliteExtDatabase as SqliteDatabase 627 | except ImportError: 628 | SqliteDatabase = None 629 | 630 | 631 | class SqliteCache(Cache): 632 | def __init__(self, filename, cache_size=32, thread_safe=True, **params): 633 | if SqliteDatabase is None: 634 | raise ImproperlyConfigured('Cannot use SqliteCache - peewee is ' 635 | 'not installed') 636 | self._filename = filename 637 | self._cache_size = cache_size # In MiB. 638 | self._thread_safe = thread_safe 639 | self._db = SqliteDatabase( 640 | self._filename, 641 | thread_safe=self._thread_safe, 642 | pragmas={ 643 | 'cache_size': self._cache_size * -1000, 644 | 'journal_mode': 'wal', # Multiple readers + one writer. 645 | 'synchronous': 0, 646 | 'wal_synchronous': 0}) 647 | 648 | class Cache(Model): 649 | key = TextField(primary_key=True) 650 | value = BlobField(null=True) 651 | expires = FloatField() 652 | 653 | class Meta: 654 | database = self._db 655 | indexes = ( 656 | (('key', 'expires'), False), 657 | ) 658 | without_rowid = True 659 | 660 | self.cache = Cache 661 | super(SqliteCache, self).__init__(**params) 662 | 663 | def open(self): 664 | if not self._db.is_closed(): return False 665 | 666 | self._db.connect() 667 | self.cache.create_table() 668 | return True 669 | 670 | def close(self): 671 | if self._db.is_closed(): return False 672 | self._db.close() 673 | return True 674 | 675 | def _get(self, key): 676 | query = (self.cache 677 | .select(self.cache.value) 678 | .where((self.cache.key == key) & 679 | (self.cache.expires >= time.time())) 680 | .limit(1) 681 | .tuples()) 682 | try: 683 | return query.get()[0] 684 | except self.cache.DoesNotExist: 685 | pass 686 | 687 | def _get_many(self, keys): 688 | query = (self.cache 689 | .select(self.cache.key, self.cache.value) 690 | .where(self.cache.key.in_(keys) & 691 | (self.cache.expires >= time.time())) 692 | .tuples()) 693 | return dict(query) 694 | 695 | def _set(self, key, value, timeout): 696 | expires = time.time() + timeout 697 | self.cache.replace(key=key, value=value, expires=expires).execute() 698 | 699 | def _set_many(self, data, timeout): 700 | expires = time.time() + timeout 701 | normalized = [(key, value, expires) for key, value in data.items()] 702 | fields = [self.cache.key, self.cache.value, self.cache.expires] 703 | return (self.cache 704 | .replace_many(normalized, fields=fields) 705 | .execute()) 706 | 707 | def _delete(self, key): 708 | return self.cache.delete().where(self.cache.key == key).execute() 709 | 710 | def _delete_many(self, keys): 711 | return self.cache.delete().where(self.cache.key.in_(keys)).execute() 712 | 713 | def _flush(self): 714 | query = self.cache.delete() 715 | if self.prefix: 716 | prefix = decode(self.prefix) 717 | query = query.where( 718 | fn.SUBSTR(self.cache.key, 1, len(prefix)) == prefix) 719 | return query.execute() 720 | 721 | def clean_expired(self, ndays=0): 722 | timestamp = time.time() - (ndays * 86400) 723 | return (self.cache 724 | .delete() 725 | .where(self.cache.expires <= timestamp) 726 | .execute()) 727 | 728 | 729 | try: 730 | from redis import Redis 731 | except ImportError: 732 | Redis = None 733 | 734 | class RedisCache(Cache): 735 | def __init__(self, host='127.0.0.1', port=6379, db=0, connection=None, 736 | **params): 737 | if Redis is None: 738 | raise ImproperlyConfigured('Cannot use RedisCache - redis python ' 739 | 'bindings are not installed.') 740 | self._host = host 741 | self._port = port 742 | self._db = db 743 | self._connection = connection 744 | self._client = None 745 | super(RedisCache, self).__init__(**params) 746 | 747 | def open(self): 748 | if self._client is not None: return False 749 | 750 | if self._connection is None: 751 | self._connection = Redis(host=self._host, port=self._port, 752 | db=self._db, **self.params) 753 | self._client = self._connection 754 | return True 755 | 756 | def close(self): 757 | if self._client is None: 758 | return False 759 | self._client = None 760 | return True 761 | 762 | def _set(self, key, value, timeout): 763 | return self._client.setex(key, timeout, value) 764 | 765 | def _get(self, key): 766 | return self._client.get(key) 767 | 768 | def _delete(self, key): 769 | return self._client.delete(key) 770 | 771 | def _get_many(self, keys): 772 | values = self._client.mget(keys) 773 | return dict(zip(keys, values)) 774 | 775 | def _set_many(self, data, timeout): 776 | pipeline = self._client.pipeline() 777 | pipeline.mset(data) 778 | for key in data: 779 | pipeline.expire(key, timeout) 780 | return pipeline.execute()[0] 781 | 782 | def _delete_many(self, keys): 783 | return self._client.delete(*keys) 784 | 785 | def _flush(self): 786 | if not self.prefix: 787 | return self._client.flushdb() 788 | 789 | scanner = self._client.scan_iter(self.prefix + b'*') 790 | for key_list in chunked(scanner, 500): 791 | self._client.delete(*key_list) 792 | 793 | 794 | try: 795 | import pylibmc as mc 796 | except ImportError: 797 | mc = None 798 | 799 | class MemcacheCache(Cache): 800 | def __init__(self, servers='127.0.0.1:11211', client=None, **params): 801 | if mc is None: 802 | raise ImproperlyConfigured('Cannot use MemcacheCache - pylibmc ' 803 | 'module is not installed.') 804 | self._servers = [servers] if isinstance(servers, str) else servers 805 | self._client = client 806 | super(MemcacheCache, self).__init__(**params) 807 | 808 | def open(self): 809 | if self._client is not None: return False 810 | self._client = mc.Client(self._servers, **self.params) 811 | return True 812 | 813 | def close(self): 814 | if self._client is None: 815 | return False 816 | self._client.disconnect_all() 817 | return True 818 | 819 | def _get(self, key): 820 | return self._client.get(key) 821 | 822 | def _get_many(self, keys): 823 | return self._client.get_multi(keys) 824 | 825 | def _set(self, key, value, timeout): 826 | return self._client.set(key, value, timeout) 827 | 828 | def _set_many(self, data, timeout): 829 | return self._client.set_multi(data, timeout) 830 | 831 | def _delete(self, key): 832 | return self._client.delete(key) 833 | 834 | def _delete_many(self, keys): 835 | return self._client.delete_multi(keys) 836 | 837 | def _flush(self): 838 | if self.prefix: 839 | raise NotImplementedError('Cannot flush memcached when a prefix ' 840 | 'is in use.') 841 | return self._client.flush_all() 842 | 843 | 844 | try: 845 | import pymemcache 846 | except ImportError: 847 | pymemcache = None 848 | 849 | class PyMemcacheCache(MemcacheCache): 850 | def __init__(self, server=('127.0.0.1', 11211), client=None, **params): 851 | if pymemcache is None: 852 | raise ImproperlyConfigured('Cannot use PyMemcacheCache - ' 853 | 'pymemcache module is not installed.') 854 | if isinstance(server, str) and server.find(':') >= 0: 855 | host, port = server.rsplit(':', 1) 856 | server = (host, int(port)) 857 | self._server = server 858 | self._client = client 859 | Cache.__init__(self, **params) 860 | 861 | def open(self): 862 | if self._client is not None: return False 863 | self._client = pymemcache.Client(self._server, **self.params) 864 | return True 865 | 866 | def close(self): 867 | if self._client is None: return False 868 | self._client.close() 869 | return True 870 | 871 | def __del__(self): 872 | if self._client is not None: 873 | self._client.close() 874 | 875 | 876 | try: 877 | import dbm 878 | except ImportError: 879 | try: 880 | from dbm import ndbm as dbm 881 | except ImportError: 882 | dbm = None 883 | 884 | 885 | class DbmCache(MemoryCache): 886 | def __init__(self, filename, mode=None, *args, **kwargs): 887 | if dbm is None: 888 | raise ImproperlyConfigured('Cannot use DbmCache - dbm python ' 889 | 'module is not available.') 890 | self._filename = filename 891 | self._mode = mode or 0o644 892 | super(DbmCache, self).__init__(*args, **kwargs) 893 | 894 | def open(self, flag='c'): 895 | if self._data: 896 | return False 897 | self._data = dbm.open(self._filename, flag=flag, mode=self._mode) 898 | atexit.register(self._data.close) 899 | return True 900 | 901 | def close(self): 902 | if not self._data: 903 | return False 904 | atexit.unregister(self._data.close) 905 | self._data = None 906 | return True 907 | 908 | def _set_many(self, data, timeout): 909 | for key, value in data.items(): 910 | self._set(key, value, timeout) 911 | 912 | def iter_keys(self): 913 | key = self._data.firstkey() 914 | while key is not None: 915 | if self._prefix_len == 0 or key.startswith(self.prefix): 916 | yield key 917 | key = self._data.nextkey(key) 918 | 919 | def _flush(self): 920 | with self._lock: 921 | if not self.prefix: 922 | self.close() 923 | self.open('n') 924 | else: 925 | self._delete_many(list(self.iter_keys())) 926 | 927 | def clean_expired(self, ndays=0): 928 | with self._lock: 929 | timestamp = time.time() - (ndays * 86400) 930 | accum = [] 931 | for key in self.iter_keys(): 932 | ts, _ = decode_timestamp(self._data[key]) 933 | if ts <= timestamp: 934 | accum.append(key) 935 | 936 | self._delete_many(accum) 937 | return len(accum) 938 | 939 | 940 | try: 941 | from greendb import Client as GreenClient 942 | except ImportError: 943 | GreenClient = None 944 | 945 | 946 | class GreenDBCache(Cache): 947 | manual_expire = True 948 | 949 | def __init__(self, host='127.0.0.1', port=31337, db=0, pool=True, 950 | max_age=None, **params): 951 | if GreenClient is None: 952 | raise ImproperlyConfigured('Cannot use GreenDBCache, greendb is ' 953 | 'not installed.') 954 | self._client = GreenClient(host, port, pool=pool, max_age=max_age) 955 | self._db = db 956 | super(GreenDBCache, self).__init__(**params) 957 | 958 | def open(self): 959 | if self._client.connect(): 960 | self._client.use(self._db) 961 | return True 962 | return False 963 | 964 | def close(self): 965 | return self._client.close() 966 | 967 | def _get(self, key): 968 | return self._client.getraw(key) 969 | 970 | def _get_many(self, keys): 971 | return self._client.mgetraw(keys) 972 | 973 | def _set(self, key, value, timeout): 974 | return self._client.setraw(key, value) 975 | 976 | def _set_many(self, data, timeout): 977 | return self._client.msetraw(data) 978 | 979 | def _delete(self, key): 980 | return self._client.delete(key) 981 | 982 | def _delete_many(self, keys): 983 | return self._client.mdelete(keys) 984 | 985 | def _flush(self): 986 | if self.prefix: 987 | self._client.deleterange(self.prefix, self.prefix + b'\xff') 988 | else: 989 | self._client.flush() 990 | 991 | def clean_expired(self, ndays=0): 992 | timestamp = time.time() - (ndays * 86400) 993 | n_deleted = 0 994 | items = self._client.getrangeraw(self.prefix, self.prefix + b'\xff') 995 | for key, value in items: 996 | ts, _ = decode_timestamp(value) 997 | if ts <= timestamp: 998 | self._client.delete(key) 999 | n_deleted += 1 1000 | return n_deleted 1001 | --------------------------------------------------------------------------------