├── .gitignore ├── .travis.yml ├── LICENSE ├── README.md ├── README_zh.md ├── setup.py ├── tests ├── do_test.sh └── test_unitest.py ├── usage_demo.py └── wrapcache ├── __init__.py ├── adapter ├── BaseAdapter.py ├── CacheException.py ├── MemcachedAdapter.py ├── MemoryAdapter.py ├── RedisAdapter.py └── __init__.py └── database └── __init__.py /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | 5 | # C extensions 6 | *.so 7 | 8 | # Distribution / packaging 9 | .Python 10 | env/ 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | .eggs/ 17 | lib/ 18 | lib64/ 19 | parts/ 20 | sdist/ 21 | var/ 22 | *.egg-info/ 23 | .installed.cfg 24 | *.egg 25 | 26 | # PyInstaller 27 | # Usually these files are written by a python script from a template 28 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 29 | *.manifest 30 | *.spec 31 | 32 | # Installer logs 33 | pip-log.txt 34 | pip-delete-this-directory.txt 35 | 36 | # Unit test / coverage reports 37 | htmlcov/ 38 | .tox/ 39 | .coverage 40 | .coverage.* 41 | .cache 42 | nosetests.xml 43 | coverage.xml 44 | *,cover 45 | 46 | # Translations 47 | *.mo 48 | *.pot 49 | 50 | # Django stuff: 51 | *.log 52 | 53 | # Sphinx documentation 54 | docs/_build/ 55 | 56 | # PyBuilder 57 | target/ 58 | 59 | .settings/ 60 | .project 61 | .pydevproject 62 | MANIFEST 63 | README 64 | test.py 65 | .idea/ -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: python 2 | python: 3 | - "2.6" 4 | - "2.7" 5 | - "3.3" 6 | - "3.4" 7 | before_install: 8 | - pip install codecov 9 | # command to install dependencies 10 | install: 11 | - pip install . 12 | - pip install redis 13 | before_script: 14 | - chmod 777 ./tests/do_test.sh 15 | # command to run tests 16 | script: ./tests/do_test.sh 17 | after_success: 18 | - codecov -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2016 atool 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | 23 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # wrapcache 2 | 3 | A python Function / Method OUTPUT cache system base on function Decorators. Supports to cache into `redis` or `LRU` memory. [中文README.md](README_zh.md) 4 | 5 | [![Build Status](https://travis-ci.org/hustcc/wrapcache.svg)](https://travis-ci.org/hustcc/wrapcache) [![codecov Status](https://codecov.io/github/hustcc/wrapcache/coverage.svg?branch=master)](https://codecov.io/github/hustcc/wrapcache?branch=master) [![PyPi Status](https://img.shields.io/pypi/v/wrapcache.svg)](https://pypi.python.org/pypi/wrapcache) [![Python Versions](https://img.shields.io/pypi/pyversions/wrapcache.svg)](https://pypi.python.org/pypi/wrapcache) [![PyPi Downloads](https://img.shields.io/pypi/dm/wrapcache.svg)](https://pypi.python.org/pypi/wrapcache) 6 | 7 | > [Homepage](http://git.atool.org/wrapcache/) | [Usage & Doc](http://git.atool.org/wrapcache/) | [Source Code](https://github.com/hustcc/wrapcache) | [Issue & Bug](https://github.com/hustcc/wrapcache/issues/new) 8 | 9 | 10 | ## What's wrapcache? 11 | 12 | `wrapcache` is a decorator that enables caching of return values for functions. 13 | 14 | The cache keys are dependent completely on the arguments passed to the function. very simple to use. 15 | 16 | Also has some `API` to `Programmatic` get cache or remove cache. `Support python 2.6 ~ python3.5`. 17 | 18 | Here's an example of how you might use wrapcache: 19 | 20 | ```python 21 | 22 | import wrapcache 23 | 24 | from time import sleep 25 | import random 26 | 27 | @wrapcache.wrapcache(timeout = 3) 28 | def need_cache_function(input, t = 2, o = 3): 29 | sleep(2) 30 | return random.randint(1, 100) 31 | 32 | if __name__ == "__main__": 33 | for i in range(10): 34 | sleep(1) 35 | print(need_cache_function(1, t = 2, o = 3)) 36 | 37 | #get cache Programmatic 38 | key_func = wrapcache.keyof(need_cache_function, 1, o = 3, t = 2) 39 | print(wrapcache.get(key_func)) 40 | #remove cache Programmatic 41 | print(wrapcache.remove(wrapcache.keyof(need_cache_function, 1, o = 3, t = 2))) 42 | 43 | ``` 44 | 45 | Some config: 46 | 47 | ```python 48 | @wrapcache.wrapcache(timeout = timeout, adapter = adapter) 49 | ``` 50 | 51 | - **`timeout`**: how much seconds the cache exist. Default is `-1`, not cached. 52 | - **`adapter`**: cache where, now can be `RedisAdapter` and `MemoryAdapter`. Default is `MemoryAdapter`. 53 | 54 | 55 | When use `RedisAdapter`, you need to set redis instance before use. e.g. 56 | 57 | ```python 58 | REDIS_POOL = redis.ConnectionPool(host = '10.246.13.1', port = 6379, password = 'redis_pwd', db = 1) 59 | REDIS_INST = redis.Redis(connection_pool = REDIS_POOL, charset = 'utf8') 60 | RedisAdapter.db = REDIS_INST 61 | ``` 62 | 63 | When use `MemoryAdapter`, you can set `Memory Storage Policy` before use, that is choose where to store. Default is store in `{}`, also it provide `LUR`(Supported by [jlhutch/pylru](https://github.com/jlhutch/pylru), add `dynamic` size.) algorithm. e.g. 64 | 65 | 66 | ```python 67 | from wrapcache.database import LruCacheDB 68 | lruDB = MemoryAdapter.db = LruCacheDB(size = 100) 69 | RedisAdapter.db = lruDB 70 | ``` 71 | 72 | ## How to Install and Use? 73 | 74 | ### Install 75 | 76 | Three ways to install wrapcache: 77 | 78 | #### 1. Use tool install 79 | 80 | - `pip install wrapcache` 81 | 82 | #### 2. Download to install 83 | 84 | - Download from [https://pypi.python.org/pypi/wrapcache/](https://pypi.python.org/pypi/wrapcache/), and run `python setup.py install`. 85 | 86 | #### 3. Manual installation 87 | 88 | - Manual installation: Put `wrapcache` folder into current directory or `site-packages`, then `import wrapcache` to use. 89 | 90 | 91 | ### Usage 92 | 93 | #### Decorators 94 | 95 | ```python 96 | 97 | import wrapcache 98 | @wrapcache.wrapcache(timeout = 3) 99 | def need_cache_function(): 100 | return 'hello wrapcache' 101 | 102 | ``` 103 | 104 | #### API 105 | 106 | 1. **`wrapcache.keyof(func, *args, **kws)`**: get the key of function output cache. 107 | 2. **`wrapcache.get(key, adapter = MemoryAdapter)`**: get the value of cache. 108 | 3. **`wrapcache.set(key, value, timeout = -1, adapter = MemoryAdapter)`**: set cache use code. 109 | 4. **`wrapcache.remove(key, adapter = MemoryAdapter)`**: remove a cache. 110 | 5. **`wrapcache.flush(adapter = MemoryAdapter)`**: clear all the cache. 111 | 112 | The API 2~5, need to input a `adapter` to set which db to operate. 113 | 114 | 115 | ## TODO 116 | 117 | - usage wiki. -------------------------------------------------------------------------------- /README_zh.md: -------------------------------------------------------------------------------- 1 | # wrapcache 2 | 3 | 一个基于Python装饰器Decorators的方法缓存系统，用于缓存Python方法的输出值，可以支持复杂数据类型，可以缓存到Redis中、Python dict、LUR算法存储中。[English README.md](README.md) 4 | 5 | [![Build Status](https://travis-ci.org/hustcc/wrapcache.svg)](https://travis-ci.org/hustcc/wrapcache) [![codecov Status](https://codecov.io/github/hustcc/wrapcache/coverage.svg?branch=master)](https://codecov.io/github/hustcc/wrapcache?branch=master) [![PyPi Status](https://img.shields.io/pypi/v/wrapcache.svg)](https://pypi.python.org/pypi/wrapcache) [![Python Versions](https://img.shields.io/pypi/pyversions/wrapcache.svg)](https://pypi.python.org/pypi/wrapcache) [![PyPi Downloads](https://img.shields.io/pypi/dm/wrapcache.svg)](https://pypi.python.org/pypi/wrapcache) 6 | 7 | > [Homepage](http://git.atool.org/wrapcache/) | [Usage & Doc](http://git.atool.org/wrapcache/) | [Source Code](https://github.com/hustcc/wrapcache) | [Issue & Bug](https://github.com/hustcc/wrapcache/issues/new) 8 | 9 | 10 | ## 什么是 wrapcache? 11 | 12 | `wrapcache` 是一个可以缓存方法输出的装饰器，即简单的缓存方法的输出值。 13 | 14 | 缓存数据的键值Key完全依赖于方法和传入方法的参数，这部分完全透明，使用起来非常方便。 15 | 16 | 同时还提供部分API方法来通过代码获取缓存、删除缓存，`支持Python2.6 ~ Python3.5`。 17 | 18 | 看了下面的一个示例，你就明白如何使用了： 19 | 20 | ### DEMO 21 | 22 | ```python 23 | 24 | import wrapcache 25 | 26 | from time import sleep 27 | import random 28 | 29 | @wrapcache.wrapcache(timeout = 3) 30 | def need_cache_function(input, t = 2, o = 3): 31 | sleep(2) 32 | return random.randint(1, 100) 33 | 34 | if __name__ == "__main__": 35 | print(need_cache_function(1, t = 2, o = 3)) #会在2秒之后打印随机数（例如59） 36 | print(need_cache_function(1, t = 2, o = 3)) #会很快就数据缓存数据59，不需要等待2秒 37 | sleep(3) # cache timeout 38 | print(need_cache_function(1, t = 2, o = 3)) #会在2秒后打印另外一个随机数，一般不会等于59 39 | 40 | 41 | ``` 42 | 43 | ### 配置项 44 | 45 | ```python 46 | @wrapcache.wrapcache(timeout = timeout, adapter = adapter) 47 | ``` 48 | 49 | - **`timeout`**: 缓存会持续多长时间，单位为秒，如果为`-1`，表示不缓存。 50 | - **`adapter`**: 缓存到何处？`RedisAdapter` 和 `MemoryAdapter` 可选. 默认 `MemoryAdapter`. 51 | 52 | ### 缓存存储器 53 | 54 | 当使用`MemoryAdapter`和`RedisAdapter`适配的时候，需要在使用之前设置他们的`数据存储器`。 55 | 56 | 57 | #### MemoryAdapter存储器 58 | 59 | 对于`MemoryAdapter`来说，提供两种缓存器： 60 | 61 | 1. 一种是基本的Python字典`{}`，也是MemoryAdapter存储`器默认的存储器`。可以无限量存储数据，具有最高的效率。但是需要手动进行清理缓存，否则可能导致内存泄露，建议在Key值比较固定的方法是哪个使用，例如无参数的方法。 62 | 2. 一种是本项目提供的`LruCacheDB`存储器（加入了`LUR`算法的Python字典）。LUR算法经过优化，所有操作的算法复杂度均为o(1)，使用时需要指定`size`，为缓存的大小，默认为-1，不限定大小，建议按照个人项目需要进行设置。 63 | 64 | 例如： 65 | 66 | ```python 67 | from wrapcache.database import LruCacheDB 68 | lruDB = MemoryAdapter.db = LruCacheDB(size = 100) 69 | RedisAdapter.db = lruDB 70 | ``` 71 | 72 | #### RedisAdapter存储器 73 | 74 | 当使用`RedisAdapter`，必须设置其存储器，否则无法使用，RedisAdapter存储器极为redis连接实例，需要`pip install redis`。例如： 75 | 76 | ```python 77 | REDIS_POOL = redis.ConnectionPool(host = '10.246.13.1', port = 6379, password = 'redis_pwd', db = 1) 78 | REDIS_INST = redis.Redis(connection_pool = REDIS_POOL, charset = 'utf8') 79 | RedisAdapter.db = REDIS_INST 80 | ``` 81 | 82 | ## 如何安装使用? 83 | 84 | ### 安装 85 | 86 | 三种方法安装 wrapcache: 87 | 88 | #### 1. 使用PIP工具 89 | 90 | - `pip install wrapcache` 91 | 92 | #### 2. 下载安装 93 | 94 | - 从 [https://pypi.python.org/pypi/wrapcache/](https://pypi.python.org/pypi/wrapcache/)下载安装包解压, 并在目录中执行 `python setup.py install`即可。 95 | 96 | #### 3. 手动安装使用 97 | 98 | - 将项目中的`wrapcache` 复杂到当前目录，或者 Python的`site-packages`目录, 然后 `import wrapcache` 即可使用。 99 | 100 | 101 | ### 使用方法 102 | 103 | #### 装饰器 104 | 105 | ```python 106 | 107 | import wrapcache 108 | @wrapcache.wrapcache(timeout = 3) 109 | def need_cache_function(): 110 | return 'hello wrapcache' 111 | 112 | ``` 113 | 114 | #### API方法 115 | 116 | 1. **`wrapcache.keyof(func, *args, **kws)`**: 获取方法的缓存Key值. 117 | 2. **`wrapcache.get(key, adapter = MemoryAdapter)`**: 或者缓存之. 118 | 3. **`wrapcache.set(key, value, timeout = -1, adapter = MemoryAdapter)`**: 设置缓存值. 119 | 4. **`wrapcache.remove(key, adapter = MemoryAdapter)`**: 移除一个缓存. 120 | 5. **`wrapcache.flush(adapter = MemoryAdapter)`**: 移除所有的缓存. 121 | 122 | API方法中，第2~5个API在使用的时候，需要传入 `adapter` 来设置需要操作哪一个适配器。 123 | 124 | 125 | ## TODO 126 | 127 | - 使用文档 wiki. -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | from distutils.core import setup 3 | LONGDOC = """ 4 | wrapcache：wrap cache， python方法装饰器缓存系统。 5 | 6 | wrapcache: Short for wraps cache. A Function / Method OUTPUT cache system base on function Decorators. 7 | 8 | GitHub: https://github.com/hustcc/wrapcache 9 | 10 | 特点 11 | 12 | - 兼容各种版本的python，包括python2和python3的个版本； 13 | - 使用方便：一个装饰器放到方法的头部即可缓存该方法； 14 | - 配置简单：cache指定缓存方式、timeout指定缓存过期时间； 15 | - MIT 授权协议； 16 | 17 | 18 | 安装说明 19 | 20 | 代码对 Python 2/3 均兼容 21 | 22 | 1. 全自动安装： ``easy_install wrapcache`` 或者 ``pip install wrapcache`` / ``pip3 install wrapcache`` 23 | 2. 半自动安装：先下载 https://pypi.python.org/pypi/wrapcache/ ，解压后运行 24 | python setup.py install 25 | 3. 手动安装：将 jieba 目录放置于当前目录或者 site-packages 目录 26 | 27 | 通过 import wrapcache 来引用 28 | 29 | """ 30 | 31 | setup(name = 'wrapcache', 32 | version = '1.0.8', 33 | description = 'Short for wraps cache. A method cache system base on method Decorators.', 34 | long_description = LONGDOC, 35 | author = 'hustcc', 36 | author_email = 'i@atool.org', 37 | url = 'https://github.com/hustcc/wrapcache', 38 | license = "MIT", 39 | classifiers = [ 40 | 'Intended Audience :: Developers', 41 | 'License :: OSI Approved :: MIT License', 42 | 'Operating System :: OS Independent', 43 | 'Natural Language :: Chinese (Simplified)', 44 | 'Natural Language :: Chinese (Traditional)', 45 | 'Natural Language :: English', 46 | 'Programming Language :: Python', 47 | 'Programming Language :: Python :: 2', 48 | 'Programming Language :: Python :: 2.5', 49 | 'Programming Language :: Python :: 2.6', 50 | 'Programming Language :: Python :: 2.7', 51 | 'Programming Language :: Python :: 3', 52 | 'Programming Language :: Python :: 3.2', 53 | 'Programming Language :: Python :: 3.3', 54 | 'Programming Language :: Python :: 3.4', 55 | 'Programming Language :: Python :: 3.5', 56 | 'Topic :: Utilities', 57 | 'Topic :: Software Development :: Embedded Systems' 58 | ], 59 | keywords = 'wrapcache,Wraps Cache,Cache System,Decorators Cache,Function Cache,Method Cache,Redis Cache,LRU Memory Cache', 60 | packages = ['wrapcache'], 61 | package_dir = {'wrapcache':'wrapcache'}, 62 | package_data = {'wrapcache':['*.*', 'adapter/*', 'database/*']} 63 | ) 64 | -------------------------------------------------------------------------------- /tests/do_test.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | coverage run wrapcache/database/__init__.py 3 | coverage run tests/test_unitest.py 4 | coverage run wrapcache/adapter/MemoryAdapter.py 5 | # coverage run wrapcache/adapter/RedisAdapter.py -------------------------------------------------------------------------------- /tests/test_unitest.py: -------------------------------------------------------------------------------- 1 | #-*-coding: utf-8 -*- 2 | 3 | import unittest, time 4 | import sys, random 5 | 6 | import wrapcache 7 | from wrapcache.adapter.MemoryAdapter import MemoryAdapter 8 | from wrapcache.adapter.RedisAdapter import RedisAdapter 9 | from wrapcache.database import LruCacheDB 10 | #########################Memory test 11 | class TestMemoryInstance: 12 | @wrapcache.wrapcache(timeout = 3, adapter = MemoryAdapter) 13 | def test_cache(self): 14 | return time.time() 15 | 16 | @wrapcache.wrapcache(timeout = 3, adapter = MemoryAdapter) 17 | def test_input_cache(self, i, j): 18 | return (time.time() + i, j) 19 | 20 | @wrapcache.wrapcache(timeout = 10, adapter = MemoryAdapter) 21 | def test_input_order_cache(self, i = 1, j = 's'): 22 | return (time.time() + i, j) 23 | 24 | @wrapcache.wrapcache(timeout = 3, adapter = MemoryAdapter) 25 | def need_cache_function(self): 26 | time.sleep(2) 27 | print('cache timeout, new...') 28 | return random.randint(1, 100) 29 | 30 | 31 | class TestMemoryUnitest(unittest.TestCase): 32 | def setUp(self): 33 | self.test_class = TestMemoryInstance() 34 | 35 | def tearDown(self): 36 | pass 37 | 38 | def test_cache(self): 39 | val_1 = self.test_class.test_cache() 40 | self.assertEqual(self.test_class.test_cache(), val_1, 'test_cache fail') 41 | time.sleep(5) 42 | self.assertNotEqual(self.test_class.test_cache(), val_1, 'test_cache fail') 43 | 44 | def test_input_cache(self): 45 | val_1 = self.test_class.test_input_cache(1, 'hello world') 46 | self.assertEqual(self.test_class.test_input_cache(1, 'hello world'), val_1, 'test_input_cache fail') 47 | time.sleep(5) 48 | self.assertNotEqual(self.test_class.test_input_cache(1, 'hello world'), val_1, 'test_input_cache fail') 49 | self.assertNotEqual(self.test_class.test_input_cache(1, 'hello world'), self.test_class.test_input_cache(2, 'hello world'), 'test_input_cache fail') 50 | 51 | def test_input_order_cache(self): 52 | val_1 = self.test_class.test_input_order_cache(i = 1, j = 'hello world') 53 | self.assertNotEqual(self.test_class.test_input_order_cache(j = 'hello world', i = 1), self.test_class.test_input_order_cache(j = 'hello wrapcache', i = 1), 'test_input_order_cache fail') 54 | 55 | def test_keyof_api(self): 56 | key_1 = wrapcache.keyof(self.test_class.test_input_cache, i = 1, j = 'hello world') 57 | key_2 = wrapcache.keyof(self.test_class.test_input_cache, i = 1, j = 'hello world') 58 | 59 | key_3 = wrapcache.keyof(self.test_class.test_input_cache, j = 'hello world', i = 1) 60 | key_4 = wrapcache.keyof(self.test_class.test_input_cache, j = 'hello wrapcache', i = 1) 61 | self.assertEqual(key_1, key_2, 'test_keyof_api fail') 62 | self.assertEqual(key_1, key_3, 'test_keyof_api fail') 63 | self.assertNotEqual(key_1, key_4, 'test_keyof_api fail') 64 | 65 | def test_apis(self): 66 | #get api 67 | key_1 = wrapcache.keyof(self.test_class.test_input_cache, i = 1, j = 'hello world') 68 | value_1 = wrapcache.get(key_1) 69 | if not value_1: 70 | keyNone = True 71 | self.assertEqual(keyNone, True, 'test_apis fail') 72 | #set api 73 | value_2 = wrapcache.set(key_1, 'test_value', timeout = 3) 74 | self.assertEqual(value_2, 'test_value', 'test_keyof_api fail') 75 | #get api / timeout 76 | value_3 = wrapcache.get(key_1) 77 | self.assertEqual(value_3, 'test_value', 'test_keyof_api fail') 78 | time.sleep(3) 79 | value_3 = wrapcache.get(key_1) 80 | if not value_3: 81 | keyNone = True 82 | self.assertEqual(keyNone, True, 'test_apis fail') 83 | 84 | #remove api 85 | value_4 = wrapcache.set(key_1, 'test_value 4', timeout = 3) 86 | self.assertEqual(value_4, 'test_value 4', 'test_keyof_api fail') 87 | value_5 = wrapcache.remove(key_1) 88 | self.assertEqual(value_4, value_5, 'test_keyof_api fail') 89 | 90 | value_3 = wrapcache.get(key_1) 91 | if not value_5: 92 | keyNone = True 93 | self.assertEqual(keyNone, True, 'test_apis fail') 94 | 95 | #flush api 96 | value_6 = wrapcache.set(key_1, 'test_value 4', timeout = 3) 97 | self.assertEqual(value_6, 'test_value 4', 'test_keyof_api fail') 98 | self.assertTrue(wrapcache.flush(), 'test_keyof_api fail') 99 | 100 | value_6 = wrapcache.get(key_1) 101 | if not value_6: 102 | keyNone = True 103 | self.assertEqual(keyNone, True, 'test_apis fail') 104 | 105 | def test_need_cache_function(self): 106 | for i in range(10): 107 | time.sleep(1) 108 | print(self.test_class.need_cache_function()) 109 | 110 | 111 | class TestMemoryLRUUnitest(unittest.TestCase): 112 | def setUp(self): 113 | self.test_class = TestMemoryInstance() 114 | MemoryAdapter.db = LruCacheDB(size = 100) 115 | 116 | def tearDown(self): 117 | pass 118 | 119 | def test_cache(self): 120 | val_1 = self.test_class.test_cache() 121 | self.assertEqual(self.test_class.test_cache(), val_1, 'test_cache fail') 122 | time.sleep(5) 123 | self.assertNotEqual(self.test_class.test_cache(), val_1, 'test_cache fail') 124 | 125 | def test_input_cache(self): 126 | val_1 = self.test_class.test_input_cache(1, 'hello world') 127 | self.assertEqual(self.test_class.test_input_cache(1, 'hello world'), val_1, 'test_input_cache fail') 128 | time.sleep(5) 129 | self.assertNotEqual(self.test_class.test_input_cache(1, 'hello world'), val_1, 'test_input_cache fail') 130 | self.assertNotEqual(self.test_class.test_input_cache(1, 'hello world'), self.test_class.test_input_cache(2, 'hello world'), 'test_input_cache fail') 131 | 132 | def test_input_order_cache(self): 133 | val_1 = self.test_class.test_input_order_cache(i = 1, j = 'hello world') 134 | self.assertNotEqual(self.test_class.test_input_order_cache(j = 'hello world', i = 1), self.test_class.test_input_order_cache(j = 'hello wrapcache', i = 1), 'test_input_order_cache fail') 135 | 136 | def test_keyof_api(self): 137 | key_1 = wrapcache.keyof(self.test_class.test_input_cache, i = 1, j = 'hello world') 138 | key_2 = wrapcache.keyof(self.test_class.test_input_cache, i = 1, j = 'hello world') 139 | 140 | key_3 = wrapcache.keyof(self.test_class.test_input_cache, j = 'hello world', i = 1) 141 | key_4 = wrapcache.keyof(self.test_class.test_input_cache, j = 'hello wrapcache', i = 1) 142 | self.assertEqual(key_1, key_2, 'test_keyof_api fail') 143 | self.assertEqual(key_1, key_3, 'test_keyof_api fail') 144 | self.assertNotEqual(key_1, key_4, 'test_keyof_api fail') 145 | 146 | def test_apis(self): 147 | #get api 148 | key_1 = wrapcache.keyof(self.test_class.test_input_cache, i = 1, j = 'hello world') 149 | value_1 = wrapcache.get(key_1) 150 | if not value_1: 151 | keyNone = True 152 | self.assertEqual(keyNone, True, 'test_apis fail') 153 | #set api 154 | value_2 = wrapcache.set(key_1, 'test_value', timeout = 3) 155 | self.assertEqual(value_2, 'test_value', 'test_keyof_api fail') 156 | #get api / timeout 157 | value_3 = wrapcache.get(key_1) 158 | self.assertEqual(value_3, 'test_value', 'test_keyof_api fail') 159 | time.sleep(3) 160 | value_3 = wrapcache.get(key_1) 161 | if not value_3: 162 | keyNone = True 163 | self.assertEqual(keyNone, True, 'test_apis fail') 164 | 165 | #remove api 166 | value_4 = wrapcache.set(key_1, 'test_value 4', timeout = 3) 167 | self.assertEqual(value_4, 'test_value 4', 'test_keyof_api fail') 168 | value_5 = wrapcache.remove(key_1) 169 | self.assertEqual(value_4, value_5, 'test_keyof_api fail') 170 | 171 | value_3 = wrapcache.get(key_1) 172 | if not value_5: 173 | keyNone = True 174 | self.assertEqual(keyNone, True, 'test_apis fail') 175 | 176 | #flush api 177 | value_6 = wrapcache.set(key_1, 'test_value 4', timeout = 3) 178 | self.assertEqual(value_6, 'test_value 4', 'test_keyof_api fail') 179 | self.assertTrue(wrapcache.flush(), 'test_keyof_api fail') 180 | 181 | value_6 = wrapcache.get(key_1) 182 | if not value_6: 183 | keyNone = True 184 | self.assertEqual(keyNone, True, 'test_apis fail') 185 | 186 | def test_need_cache_function(self): 187 | for i in range(10): 188 | time.sleep(1) 189 | print(self.test_class.need_cache_function()) 190 | ################end memory test 191 | 192 | 193 | #########################redis test 194 | class TestRedisInstance: 195 | @wrapcache.wrapcache(timeout = 3, adapter = RedisAdapter) 196 | def test_cache(self): 197 | return time.time() 198 | 199 | @wrapcache.wrapcache(timeout = 3, adapter = RedisAdapter) 200 | def test_input_cache(self, i, j): 201 | return (time.time() + i, j) 202 | 203 | @wrapcache.wrapcache(timeout = 10, adapter = RedisAdapter) 204 | def test_input_order_cache(self, i = 1, j = 's'): 205 | return (time.time() + i, j) 206 | 207 | @wrapcache.wrapcache(timeout = 3, adapter = RedisAdapter) 208 | def need_cache_function(self): 209 | time.sleep(2) 210 | print('cache timeout, new...') 211 | return random.randint(1, 100) 212 | 213 | 214 | class TestRedisUnitest(unittest.TestCase): 215 | def setUp(self): 216 | self.test_class = TestRedisInstance() 217 | import redis 218 | #init redis instance 219 | REDIS_CACHE_POOL = redis.ConnectionPool(host = '162.211.225.209', port = 6739, password = 'wzwacxl', db = 2) 220 | REDIS_CACHE_INST = redis.Redis(connection_pool = REDIS_CACHE_POOL, charset = 'utf8') 221 | RedisAdapter.db = REDIS_CACHE_INST #初始化装饰器缓存 222 | 223 | 224 | def tearDown(self): 225 | pass 226 | 227 | def test_cache(self): 228 | val_1 = self.test_class.test_cache() 229 | self.assertEqual(self.test_class.test_cache(), val_1, 'test_cache fail') 230 | time.sleep(5) 231 | self.assertNotEqual(self.test_class.test_cache(), val_1, 'test_cache fail') 232 | 233 | def test_input_cache(self): 234 | val_1 = self.test_class.test_input_cache(1, 'hello world') 235 | self.assertEqual(self.test_class.test_input_cache(1, 'hello world'), val_1, 'test_input_cache fail') 236 | time.sleep(5) 237 | self.assertNotEqual(self.test_class.test_input_cache(1, 'hello world'), val_1, 'test_input_cache fail') 238 | self.assertNotEqual(self.test_class.test_input_cache(1, 'hello world'), self.test_class.test_input_cache(2, 'hello world'), 'test_input_cache fail') 239 | 240 | def test_input_order_cache(self): 241 | val_1 = self.test_class.test_input_order_cache(i = 1, j = 'hello world') 242 | self.assertNotEqual(self.test_class.test_input_order_cache(j = 'hello world', i = 1), self.test_class.test_input_order_cache(j = 'hello wrapcache', i = 1), 'test_input_order_cache fail') 243 | 244 | def test_keyof_api(self): 245 | key_1 = wrapcache.keyof(self.test_class.test_input_cache, i = 1, j = 'hello world') 246 | key_2 = wrapcache.keyof(self.test_class.test_input_cache, i = 1, j = 'hello world') 247 | 248 | key_3 = wrapcache.keyof(self.test_class.test_input_cache, j = 'hello world', i = 1) 249 | key_4 = wrapcache.keyof(self.test_class.test_input_cache, j = 'hello wrapcache', i = 1) 250 | self.assertEqual(key_1, key_2, 'test_keyof_api fail') 251 | self.assertEqual(key_1, key_3, 'test_keyof_api fail') 252 | self.assertNotEqual(key_1, key_4, 'test_keyof_api fail') 253 | 254 | def test_apis(self): 255 | #get api 256 | key_1 = wrapcache.keyof(self.test_class.test_input_cache, i = 1, j = 'hello world') 257 | value_1 = wrapcache.get(key_1) 258 | if not value_1: 259 | keyNone = True 260 | self.assertEqual(keyNone, True, 'test_apis fail') 261 | #set api 262 | value_2 = wrapcache.set(key_1, 'test_value', timeout = 3) 263 | self.assertEqual(value_2, 'test_value', 'test_keyof_api fail') 264 | #get api / timeout 265 | value_3 = wrapcache.get(key_1) 266 | self.assertEqual(value_3, 'test_value', 'test_keyof_api fail') 267 | time.sleep(3) 268 | value_3 = wrapcache.get(key_1) 269 | if not value_3: 270 | keyNone = True 271 | self.assertEqual(keyNone, True, 'test_apis fail') 272 | 273 | #remove api 274 | value_4 = wrapcache.set(key_1, 'test_value 4', timeout = 3) 275 | self.assertEqual(value_4, 'test_value 4', 'test_keyof_api fail') 276 | value_5 = wrapcache.remove(key_1) 277 | self.assertEqual(value_4, value_5, 'test_keyof_api fail') 278 | 279 | value_3 = wrapcache.get(key_1) 280 | if not value_5: 281 | keyNone = True 282 | self.assertEqual(keyNone, True, 'test_apis fail') 283 | 284 | #flush api 285 | value_6 = wrapcache.set(key_1, 'test_value 4', timeout = 3) 286 | self.assertEqual(value_6, 'test_value 4', 'test_keyof_api fail') 287 | self.assertTrue(wrapcache.flush(), 'test_keyof_api fail') 288 | 289 | value_6 = wrapcache.get(key_1) 290 | if not value_6: 291 | keyNone = True 292 | self.assertEqual(keyNone, True, 'test_apis fail') 293 | 294 | def test_need_cache_function(self): 295 | for i in range(10): 296 | time.sleep(1) 297 | print(self.test_class.need_cache_function()) 298 | 299 | ################end memory test 300 | 301 | if __name__ =='__main__': 302 | unittest.main() -------------------------------------------------------------------------------- /usage_demo.py: -------------------------------------------------------------------------------- 1 | #-*-coding: utf-8 -*- 2 | 3 | import unittest, time 4 | import sys, random 5 | 6 | import wrapcache 7 | from wrapcache.adapter.RedisAdapter import RedisAdapter 8 | from wrapcache.adapter.MemoryAdapter import MemoryAdapter 9 | from wrapcache.database import LruCacheDB 10 | 11 | @wrapcache.wrapcache(timeout = 3) 12 | def need_mem_cache_function(): 13 | time.sleep(2) 14 | print('mem cache timeout, new...') 15 | return random.randint(1, 100) 16 | 17 | @wrapcache.wrapcache(timeout = 3, adapter = RedisAdapter) 18 | def need_redis_cache_function(): 19 | time.sleep(2) 20 | print('redis cache timeout, new...') 21 | return (random.randint(1, 100), 'Hello wrapcache') 22 | 23 | 24 | MemoryAdapter.db = LruCacheDB(size = 100) 25 | @wrapcache.wrapcache(timeout = 3, adapter = MemoryAdapter) 26 | def need_mem_lru_cache_function(): 27 | time.sleep(2) 28 | print('mem cache timeout, new...') 29 | return random.randint(1, 100) 30 | 31 | if __name__ =='__main__': 32 | 33 | #set redid db before use. 34 | import redis 35 | REDIS_POOL = redis.ConnectionPool(host = '10.246.13.189', port = 6379, password = '', db = 1) 36 | REDIS_INST = redis.Redis(connection_pool = REDIS_POOL, charset = 'utf8') 37 | RedisAdapter.db = REDIS_INST 38 | #redis cache 39 | for i in range(10): 40 | time.sleep(1) 41 | print(need_redis_cache_function()[0]) 42 | 43 | #memory cache 44 | for i in range(10): 45 | time.sleep(1) 46 | print(need_mem_cache_function()) -------------------------------------------------------------------------------- /wrapcache/__init__.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | __version__ = '1.0.8' 3 | __license__ = 'MIT' 4 | 5 | import time 6 | import sys 7 | import hashlib 8 | try: 9 | import cPickle as pickle 10 | except: 11 | import pickle 12 | from functools import wraps 13 | from wrapcache.adapter.CacheException import CacheExpiredException 14 | from wrapcache.adapter.MemoryAdapter import MemoryAdapter 15 | 16 | ''' 17 | wrapcache: wrap cache 18 | 19 | A python Function / Method OUTPUT cache system base on function Decorators. 20 | 21 | Auto cache the Funtion OUTPUT for sometime. 22 | ''' 23 | def _from_file(function): 24 | if hasattr(function, '__code__'): 25 | return function.__code__.co_filename 26 | else: 27 | return '' 28 | 29 | def _wrap_key(function, args, kws): 30 | ''' 31 | get the key from the function input. 32 | ''' 33 | return hashlib.md5(pickle.dumps((_from_file(function) + function.__name__, args, kws))).hexdigest() 34 | 35 | def keyof(function, *args, **kws): 36 | ''' 37 | get the function cache key 38 | ''' 39 | return _wrap_key(function, args, kws) 40 | 41 | def get(key, adapter = MemoryAdapter): 42 | ''' 43 | get the cache value 44 | ''' 45 | try: 46 | return pickle.loads(adapter().get(key)) 47 | except CacheExpiredException: 48 | return None 49 | 50 | def remove(key, adapter = MemoryAdapter): 51 | ''' 52 | remove cache by key 53 | ''' 54 | return pickle.loads(adapter().remove(key)) 55 | 56 | def set(key, value, timeout = -1, adapter = MemoryAdapter): 57 | ''' 58 | set cache by code, must set timeout length 59 | ''' 60 | if adapter(timeout = timeout).set(key, pickle.dumps(value)): 61 | return value 62 | else: 63 | return None 64 | 65 | def flush(adapter = MemoryAdapter): 66 | ''' 67 | clear all the caches 68 | ''' 69 | return adapter().flush() 70 | 71 | 72 | def wrapcache(timeout = -1, adapter = MemoryAdapter): 73 | ''' 74 | the Decorator to cache Function. 75 | ''' 76 | def _wrapcache(function): 77 | @wraps(function) 78 | def __wrapcache(*args, **kws): 79 | hash_key = _wrap_key(function, args, kws) 80 | try: 81 | adapter_instance = adapter() 82 | return pickle.loads(adapter_instance.get(hash_key)) 83 | except CacheExpiredException: 84 | #timeout 85 | value = function(*args, **kws) 86 | set(hash_key, value, timeout, adapter) 87 | return value 88 | return __wrapcache 89 | return _wrapcache -------------------------------------------------------------------------------- /wrapcache/adapter/BaseAdapter.py: -------------------------------------------------------------------------------- 1 | #-*-coding: utf-8 -*- 2 | ''' 3 | Base cache Adapter object. 4 | ''' 5 | 6 | class BaseAdapter(object): 7 | db = None 8 | def __init__(self, timeout = -1): 9 | self.timeout = timeout 10 | 11 | def get(self, key): 12 | raise NotImplementedError() 13 | 14 | def set(self, key, value): 15 | raise NotImplementedError() 16 | 17 | def remove(self, key): 18 | raise NotImplementedError() 19 | 20 | def flush(self): 21 | raise NotImplementedError() -------------------------------------------------------------------------------- /wrapcache/adapter/CacheException.py: -------------------------------------------------------------------------------- 1 | #-*-coding: utf-8 -*- 2 | ''' 3 | Cache Exceptions 4 | ''' 5 | 6 | class CacheExpiredException(Exception): 7 | def __init__(self, value): 8 | self.value = value 9 | 10 | def __str__(self): 11 | return repr(self.value) 12 | 13 | class DBNotSetException(Exception): 14 | def __init__(self, value): 15 | self.value = value 16 | 17 | def __str__(self): 18 | return repr(self.value) -------------------------------------------------------------------------------- /wrapcache/adapter/MemcachedAdapter.py: -------------------------------------------------------------------------------- 1 | #-*-coding: utf-8 -*- 2 | ''' 3 | Memcached Adapter object. 4 | ''' 5 | from wrapcache.adapter.BaseAdapter import BaseAdapter 6 | from wrapcache.adapter.CacheException import CacheExpiredException, DBNotSetException 7 | 8 | class MemcachedAdapter(BaseAdapter): 9 | ''' 10 | use for memcached cache 11 | ''' 12 | def __init__(self, timeout = -1): 13 | super(MemcachedAdapter, self).__init__(timeout = timeout) 14 | if not MemcachedAdapter.db: 15 | MemcachedAdapter.db = None 16 | 17 | def _check_db_instanse(self): 18 | if MemcachedAdapter.db == None: 19 | raise DBNotSetException('memcached instanse not set, use MemcachedAdapter.db = memcache_instance before use.') 20 | 21 | def get(self, key): 22 | self._check_db_instanse() 23 | value = MemcachedAdapter.db.get(key) 24 | if value == None: 25 | raise CacheExpiredException(key) 26 | return value 27 | 28 | def set(self, key, value): 29 | MemcachedAdapter.db.set(key, value, time = self.timeout) 30 | return True 31 | 32 | def remove(self, key): 33 | try: 34 | v = self.get(key) 35 | MemcachedAdapter.db.delete(key) 36 | return v 37 | except CacheExpiredException, _: 38 | return False 39 | 40 | def flush(self): 41 | self._check_db_instanse() 42 | MemcachedAdapter.db.flush_all() 43 | return True 44 | 45 | if __name__ == '__main__': 46 | import unittest, memcache, time 47 | 48 | class TestCase(unittest.TestCase): 49 | def setUp(self): 50 | #init redis instance 51 | self.test_class = MemcachedAdapter(timeout = 3) 52 | def tearDown(self): 53 | pass 54 | 55 | def test_memory_adapter(self): 56 | # test redis error 57 | self.assertRaises(DBNotSetException, self.test_class.get, 'test_key') 58 | 59 | memcache_inst = memcache.Client(['10.246.14.164:11211']) 60 | MemcachedAdapter.db = memcache_inst #初始化装饰器缓存 61 | self.assertRaises(CacheExpiredException, self.test_class.get, 'test_key') #链接不上 62 | 63 | memcache_inst = memcache.Client(['10.246.14.165:11211']) 64 | MemcachedAdapter.db = memcache_inst #初始化装饰器缓存 65 | 66 | key = 'test_key_1' 67 | value = str(time.time()) 68 | 69 | #test set / get 70 | self.test_class.set(key, value) 71 | self.assertEqual(self.test_class.get(key).decode('utf-8'), value) 72 | 73 | #test remove 74 | self.test_class.set(key, value) 75 | self.test_class.remove(key) 76 | self.assertRaises(CacheExpiredException, self.test_class.get, key) 77 | 78 | #test flush 79 | self.test_class.set(key, value) 80 | self.test_class.flush() 81 | self.assertRaises(CacheExpiredException, self.test_class.get, key) 82 | 83 | unittest.main() -------------------------------------------------------------------------------- /wrapcache/adapter/MemoryAdapter.py: -------------------------------------------------------------------------------- 1 | #-*-coding: utf-8 -*- 2 | ''' 3 | Memory Adapter object. 4 | ''' 5 | import time 6 | from wrapcache.adapter.BaseAdapter import BaseAdapter 7 | from wrapcache.adapter.CacheException import CacheExpiredException 8 | 9 | 10 | class MemoryAdapter(BaseAdapter): 11 | ''' 12 | use for memory cache 13 | ''' 14 | def __init__(self, timeout = -1): 15 | super(MemoryAdapter, self).__init__(timeout = timeout) 16 | if MemoryAdapter.db is None: 17 | MemoryAdapter.db = {} 18 | 19 | def get(self, key): 20 | cache = MemoryAdapter.db.get(key, {}) 21 | if time.time() - cache.get('time', 0) > 0: 22 | self.remove(key) #timeout, rm key, reduce memory 23 | raise CacheExpiredException(key) 24 | else: 25 | return cache.get('value', None) 26 | 27 | def set(self, key, value): 28 | cache = { 29 | 'value' : value, 30 | 'time' : time.time() + self.timeout 31 | } 32 | MemoryAdapter.db[key] = cache 33 | return True 34 | 35 | def remove(self, key): 36 | return MemoryAdapter.db.pop(key, {}).get('value', None) 37 | 38 | def flush(self): 39 | MemoryAdapter.db.clear() 40 | return True 41 | 42 | if __name__ == '__main__': 43 | import unittest 44 | 45 | class TestCase(unittest.TestCase): 46 | def setUp(self): 47 | self.test_class = MemoryAdapter(timeout = 3) 48 | def tearDown(self): 49 | pass 50 | 51 | def test_init_db_with_singleton(self): 52 | pre_db = self.test_class.db 53 | # get a new instance without cache 54 | new_adapter = MemoryAdapter(timeout = 1) 55 | cur_db = new_adapter.db 56 | self.assertEqual(id(pre_db), id(cur_db)) 57 | 58 | def test_memory_adapter(self): 59 | key = 'test_key_1' 60 | value = str(time.time()) 61 | 62 | #test set / get 63 | self.test_class.set(key, value) 64 | self.assertEqual(self.test_class.get(key), value) 65 | time.sleep(4) 66 | self.assertRaises(CacheExpiredException, self.test_class.get, key) 67 | 68 | #test remove 69 | self.test_class.set(key, value) 70 | self.test_class.remove(key) 71 | self.assertRaises(CacheExpiredException, self.test_class.get, key) 72 | 73 | #test flush 74 | self.test_class.set(key, value) 75 | self.test_class.flush() 76 | self.assertRaises(CacheExpiredException, self.test_class.get, key) 77 | 78 | unittest.main() 79 | -------------------------------------------------------------------------------- /wrapcache/adapter/RedisAdapter.py: -------------------------------------------------------------------------------- 1 | #-*-coding: utf-8 -*- 2 | ''' 3 | Redis Adapter object. 4 | ''' 5 | from wrapcache.adapter.BaseAdapter import BaseAdapter 6 | from wrapcache.adapter.CacheException import CacheExpiredException, DBNotSetException 7 | 8 | class RedisAdapter(BaseAdapter): 9 | ''' 10 | use for redis cache 11 | ''' 12 | def __init__(self, timeout = -1): 13 | super(RedisAdapter, self).__init__(timeout = timeout) 14 | if not RedisAdapter.db: 15 | RedisAdapter.db = None 16 | 17 | def _check_db_instanse(self): 18 | if RedisAdapter.db == None: 19 | raise DBNotSetException('redis instanse not set, use RedisAdapter.db = redis_instance before use.') 20 | 21 | def get(self, key): 22 | self._check_db_instanse() 23 | value = RedisAdapter.db.get(key) 24 | if value == None: 25 | raise CacheExpiredException(key) 26 | return value 27 | 28 | def set(self, key, value): 29 | RedisAdapter.db.setex(key, value, self.timeout) 30 | return True 31 | 32 | def remove(self, key): 33 | try: 34 | v = self.get(key) 35 | RedisAdapter.db.delete(key) 36 | return v 37 | except CacheExpiredException, _: 38 | return False 39 | 40 | def flush(self): 41 | self._check_db_instanse() 42 | RedisAdapter.db.flushdb() 43 | return True 44 | 45 | if __name__ == '__main__': 46 | import unittest, redis, time 47 | from redis.exceptions import ConnectionError 48 | 49 | class TestCase(unittest.TestCase): 50 | def setUp(self): 51 | #init redis instance 52 | self.test_class = RedisAdapter(timeout = 3) 53 | def tearDown(self): 54 | pass 55 | 56 | def test_memory_adapter(self): 57 | # test redis error 58 | self.assertRaises(DBNotSetException, self.test_class.get, 'test_key') 59 | 60 | REDIS_CACHE_POOL = redis.ConnectionPool(host = '162.211.225.208', port = 6739, password = '123456', db = 2) 61 | REDIS_CACHE_INST = redis.Redis(connection_pool = REDIS_CACHE_POOL, charset = 'utf8') 62 | RedisAdapter.db = REDIS_CACHE_INST #初始化装饰器缓存 63 | self.assertRaises(ConnectionError, self.test_class.get, 'test_key') 64 | 65 | REDIS_CACHE_POOL = redis.ConnectionPool(host = '162.211.225.209', port = 6739, password = 'wzwacxl', db = 2) 66 | REDIS_CACHE_INST = redis.Redis(connection_pool = REDIS_CACHE_POOL, charset = 'utf8') 67 | RedisAdapter.db = REDIS_CACHE_INST #初始化装饰器缓存 68 | 69 | key = 'test_key_1' 70 | value = str(time.time()) 71 | 72 | #test set / get 73 | self.test_class.set(key, value) 74 | self.assertEqual(self.test_class.get(key).decode('utf-8'), value) 75 | 76 | #test remove 77 | self.test_class.set(key, value) 78 | self.test_class.remove(key) 79 | self.assertRaises(CacheExpiredException, self.test_class.get, key) 80 | 81 | #test flush 82 | self.test_class.set(key, value) 83 | self.test_class.flush() 84 | self.assertRaises(CacheExpiredException, self.test_class.get, key) 85 | 86 | unittest.main() -------------------------------------------------------------------------------- /wrapcache/adapter/__init__.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- -------------------------------------------------------------------------------- /wrapcache/database/__init__.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # The cache is implemented using a combination of a python dictionary (hash 3 | # table) and a circular doubly linked list. Items in the cache are stored in 4 | # nodes. These nodes make up the linked list. The list is used to efficiently 5 | # maintain the order that the items have been used in. The front or head of 6 | # the list contains the most recently used item, the tail of the list 7 | # contains the least recently used item. When an item is used it can easily 8 | # (in a constant amount of time) be moved to the front of the list, thus 9 | # updating its position in the ordering. These nodes are also placed in the 10 | # hash table under their associated key. The hash table allows efficient 11 | # lookup of values by key. 12 | 13 | # Class for the node objects. 14 | class _dlnode(object): 15 | def __init__(self): 16 | self.empty = True 17 | 18 | 19 | class LruCacheDB(object): 20 | ''' 21 | LRU cache database 22 | ''' 23 | def __init__(self, size = -1): 24 | 25 | # Create an empty hash table. 26 | self.table = {} 27 | 28 | # Initialize the doubly linked list with one empty node. This is an 29 | # invariant. The cache size must always be greater than zero. Each 30 | # node has a 'prev' and 'next' variable to hold the node that comes 31 | # before it and after it respectively. Initially the two variables 32 | # each point to the head node itself, creating a circular doubly 33 | # linked list of size one. Then the size() method is used to adjust 34 | # the list to the desired size. 35 | 36 | self.head = _dlnode() 37 | self.head.next = self.head 38 | self.head.prev = self.head 39 | 40 | self.size = size 41 | if self.size <= 0: 42 | self.size = -1 43 | self.listSize = 1 44 | 45 | #status var 46 | self.hit_cnt = 0 47 | self.miss_cnt = 0 48 | self.remove_cnt = 0 49 | 50 | def __len__(self): 51 | ''' 52 | used length of cache table 53 | ''' 54 | return len(self.table) 55 | 56 | def clear(self): 57 | ''' 58 | claar all the cache, and release memory 59 | ''' 60 | for node in self.dli(): 61 | node.empty = True 62 | node.key = None 63 | node.value = None 64 | 65 | self.head = _dlnode() 66 | self.head.next = self.head 67 | self.head.prev = self.head 68 | self.listSize = 1 69 | 70 | self.table.clear() 71 | 72 | # status var 73 | self.hit_cnt = 0 74 | self.miss_cnt = 0 75 | self.remove_cnt = 0 76 | 77 | def __contains__(self, key): 78 | return key in self.table 79 | 80 | def peek(self, key): 81 | ''' 82 | Looks up a value in the cache without affecting cache order. 83 | ''' 84 | node = self.table[key] 85 | return node.value 86 | 87 | 88 | def __getitem__(self, key): 89 | # Look up the node 90 | try: 91 | node = self.table[key] 92 | self.hit_cnt += 1 93 | except: 94 | self.miss_cnt += 1 95 | raise KeyError 96 | 97 | # Update the list ordering. Move this node so that is directly 98 | # proceeds the head node. Then set the 'head' variable to it. This 99 | # makes it the new head of the list. 100 | self.mtf(node) 101 | self.head = node 102 | 103 | # Return the value. 104 | return node.value 105 | 106 | def get(self, key, default = None): 107 | """Get an item - return default (None) if not present""" 108 | try: 109 | return self[key] 110 | except KeyError: 111 | return default 112 | 113 | def __setitem__(self, key, value): 114 | # First, see if any value is stored under 'key' in the cache already. 115 | # If so we are going to replace that value with the new one. 116 | if key in self.table: 117 | # Lookup the node 118 | node = self.table[key] 119 | 120 | # Replace the value. 121 | node.value = value 122 | 123 | # Update the list ordering. 124 | self.mtf(node) 125 | self.head = node 126 | 127 | return value 128 | 129 | if self.size == -1: 130 | # if size = -1, then no limit of length 131 | self.addTailNode(1) 132 | else: 133 | if self.listSize < self.size: 134 | #add 3 node per time. 135 | self.addTailNode(1) 136 | # Ok, no value is currently stored under 'key' in the cache. We need 137 | # to choose a node to place the new item in. There are two cases. If 138 | # the cache is full some item will have to be pushed out of the 139 | # cache. We want to choose the node with the least recently used 140 | # item. This is the node at the tail of the list. If the cache is not 141 | # full we want to choose a node that is empty. Because of the way the 142 | # list is managed, the empty nodes are always together at the tail 143 | # end of the list. Thus, in either case, by chooseing the node at the 144 | # tail of the list our conditions are satisfied. 145 | 146 | # Since the list is circular, the tail node directly preceeds the 147 | # 'head' node. 148 | node = self.head.prev 149 | 150 | # If the node already contains something we need to remove the old 151 | # key from the dictionary. 152 | if not node.empty: 153 | self.remove_cnt += 1 154 | del self.table[node.key] 155 | 156 | # Place the new key and value in the node 157 | node.empty = False 158 | node.key = key 159 | node.value = value 160 | 161 | # Add the node to the dictionary under the new key. 162 | self.table[key] = node 163 | 164 | # We need to move the node to the head of the list. The node is the 165 | # tail node, so it directly preceeds the head node due to the list 166 | # being circular. Therefore, the ordering is already correct, we just 167 | # need to adjust the 'head' variable. 168 | self.head = node 169 | 170 | 171 | def __delitem__(self, key): 172 | # Lookup the node, then remove it from the hash table. 173 | node = self.table.get(key, None) 174 | try: 175 | del self.table[key] 176 | except: 177 | raise KeyError 178 | node.empty = True 179 | 180 | # Not strictly necessary. 181 | node.key = None 182 | node.value = None 183 | 184 | # Because this node is now empty we want to reuse it before any 185 | # non-empty node. To do that we want to move it to the tail of the 186 | # list. We move it so that it directly preceeds the 'head' node. This 187 | # makes it the tail node. The 'head' is then adjusted. This 188 | # adjustment ensures correctness even for the case where the 'node' 189 | # is the 'head' node. 190 | self.mtf(node) 191 | self.head = node.next 192 | 193 | self.remove += 1 194 | 195 | def pop(self, key, default = None): 196 | """Delete the item""" 197 | node = self.get(key, None) 198 | 199 | if node == None: 200 | value = default 201 | else: 202 | value = node 203 | try: 204 | del self[key] 205 | except: 206 | return value 207 | return value 208 | 209 | def __iter__(self): 210 | 211 | # Return an iterator that returns the keys in the cache in order from 212 | # the most recently to least recently used. Does not modify the cache 213 | # order. 214 | for node in self.dli(): 215 | yield node.key 216 | 217 | def items(self): 218 | 219 | # Return an iterator that returns the (key, value) pairs in the cache 220 | # in order from the most recently to least recently used. Does not 221 | # modify the cache order. 222 | for node in self.dli(): 223 | yield (node.key, node.value) 224 | 225 | def keys(self): 226 | 227 | # Return an iterator that returns the keys in the cache in order from 228 | # the most recently to least recently used. Does not modify the cache 229 | # order. 230 | for node in self.dli(): 231 | yield node.key 232 | 233 | def values(self): 234 | 235 | # Return an iterator that returns the values in the cache in order 236 | # from the most recently to least recently used. Does not modify the 237 | # cache order. 238 | for node in self.dli(): 239 | yield node.value 240 | 241 | def size(self, size = None): 242 | return self.size 243 | 244 | # Increases the size of the cache by inserting n empty nodes at the tail 245 | # of the list. 246 | def addTailNode(self, n): 247 | for _ in range(n): 248 | node = _dlnode() 249 | node.next = self.head 250 | node.prev = self.head.prev 251 | 252 | self.head.prev.next = node 253 | self.head.prev = node 254 | self.listSize += n 255 | 256 | 257 | # Decreases the size of the list by removing n nodes from the tail of the 258 | # list. 259 | def removeTailNode(self, n): 260 | assert self.listSize > n 261 | for _ in range(n): 262 | node = self.head.prev 263 | if not node.empty: 264 | del self.table[node.key] 265 | 266 | # Splice the tail node out of the list 267 | self.head.prev = node.prev 268 | node.prev.next = self.head 269 | 270 | # The next four lines are not strictly necessary. 271 | node.prev = None 272 | node.next = None 273 | 274 | node.key = None 275 | node.value = None 276 | 277 | self.listSize -= n 278 | 279 | 280 | 281 | def mtf(self, node): 282 | ''' 283 | This method adjusts the ordering of the doubly linked list so that 284 | 'node' directly precedes the 'head' node. Because of the order of 285 | operations, if 'node' already directly precedes the 'head' node or if 286 | 'node' is the 'head' node the order of the list will be unchanged. 287 | ''' 288 | node.prev.next = node.next 289 | node.next.prev = node.prev 290 | 291 | node.prev = self.head.prev 292 | node.next = self.head.prev.next 293 | 294 | node.next.prev = node 295 | node.prev.next = node 296 | 297 | # This method returns an iterator that iterates over the non-empty nodes 298 | # in the doubly linked list in order from the most recently to the least 299 | # recently used. 300 | def dli(self): 301 | node = self.head 302 | for _ in range(len(self.table)): 303 | yield node 304 | node = node.next 305 | 306 | def status(self): 307 | return {'max': self.size, 'used': len(self.table), 'hit': self.hit_cnt, 'miss': self.miss_cnt, 'remove': self.remove_cnt} 308 | 309 | if __name__ == '__main__': 310 | import unittest, time 311 | 312 | class TestLruCase(unittest.TestCase): 313 | def setUp(self): 314 | self.test_class = LruCacheDB(2) 315 | 316 | self.test_class_nolimit = LruCacheDB() 317 | 318 | def tearDown(self): 319 | pass 320 | 321 | def test_lru_db(self): 322 | key1 = 'test_key_1' 323 | value1 = 'test_value_1' 324 | key2 = 'test_key_2' 325 | value2 = 'test_value_2' 326 | key3 = 'test_key_3' 327 | value3 = 'test_value_3' 328 | 329 | self.assertRaises(KeyError, self.test_class.__getitem__, key1) 330 | 331 | self.test_class[key1] = value1 332 | self.assertEqual(self.test_class[key1], value1) 333 | 334 | self.test_class.clear() 335 | 336 | self.assertEqual(self.test_class.status(), {'max': 2, 'used': 0, 'hit': 0, 'miss': 0, 'remove': 0}) 337 | self.assertRaises(KeyError, self.test_class.__getitem__, key1) 338 | 339 | self.test_class[key1] = value1 340 | self.test_class[key2] = value2 341 | self.test_class[key3] = value3 342 | x_array = [] 343 | for x in self.test_class.keys(): 344 | x_array.append(x) 345 | 346 | self.assertEqual(x_array, ['test_key_3', 'test_key_2']) 347 | 348 | self.test_class[key2] 349 | x_array = [] 350 | for x in self.test_class.keys(): 351 | x_array.append(x) 352 | 353 | self.assertEqual(x_array, ['test_key_2', 'test_key_3']) 354 | 355 | self.test_class.peek(key3) 356 | x_array = [] 357 | for x in self.test_class.keys(): 358 | x_array.append(x) 359 | 360 | self.assertEqual(x_array, ['test_key_2', 'test_key_3']) 361 | 362 | self.assertEqual(self.test_class.get('test_key_2'), 'test_value_2') 363 | self.assertEqual(self.test_class.status(), {'max': 2, 'used': 2, 'hit': 2, 'miss': 1, 'remove': 1}) 364 | 365 | def test_lru_db_nolomit(self): 366 | key1 = 'test_key_1' 367 | value1 = 'test_value_1' 368 | key2 = 'test_key_2' 369 | value2 = 'test_value_2' 370 | key3 = 'test_key_3' 371 | value3 = 'test_value_3' 372 | 373 | self.assertRaises(KeyError, self.test_class_nolimit.__getitem__, key1) 374 | 375 | self.test_class_nolimit[key1] = value1 376 | self.assertEqual(self.test_class_nolimit[key1], value1) 377 | 378 | self.test_class_nolimit.clear() 379 | self.assertEqual(self.test_class_nolimit.status(), {'max': -1, 'used': 0, 'hit': 0, 'miss': 0, 'remove': 0}) 380 | self.assertRaises(KeyError, self.test_class_nolimit.__getitem__, key1) 381 | 382 | self.test_class_nolimit[key1] = value1 383 | self.test_class_nolimit[key2] = value2 384 | self.test_class_nolimit[key3] = value3 385 | x_array = [] 386 | for x in self.test_class_nolimit.keys(): 387 | x_array.append(x) 388 | 389 | self.assertEqual(x_array, ['test_key_3', 'test_key_2', 'test_key_1']) 390 | 391 | self.test_class_nolimit[key2] 392 | x_array = [] 393 | for x in self.test_class_nolimit.keys(): 394 | x_array.append(x) 395 | 396 | self.assertEqual(x_array, ['test_key_2', 'test_key_3', 'test_key_1']) 397 | 398 | self.test_class_nolimit.peek(key3) 399 | x_array = [] 400 | for x in self.test_class_nolimit.keys(): 401 | x_array.append(x) 402 | 403 | self.assertEqual(x_array, ['test_key_2', 'test_key_3', 'test_key_1']) 404 | 405 | self.assertEqual(self.test_class_nolimit.get('test_key_2'), 'test_value_2') 406 | 407 | self.assertEqual(self.test_class_nolimit.status(), {'max': -1, 'used': 3, 'hit': 2, 'miss': 1, 'remove': 0}) 408 | 409 | unittest.main() --------------------------------------------------------------------------------