├── MANIFEST.in ├── cachelper ├── __init__.py ├── local.py └── remote.py ├── .bumpversion.cfg ├── tox.ini ├── .travis.yml ├── setup.py ├── LICENSE.md ├── tests ├── test_local.py └── test_remote.py ├── .gitignore └── README.rst /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include README.rst 2 | -------------------------------------------------------------------------------- /cachelper/__init__.py: -------------------------------------------------------------------------------- 1 | from cachelper.local import * 2 | from cachelper.remote import * 3 | -------------------------------------------------------------------------------- /.bumpversion.cfg: -------------------------------------------------------------------------------- 1 | [bumpversion] 2 | current_version = 0.3.0 3 | commit = True 4 | tag = True 5 | 6 | [bumpversion:file:setup.py] 7 | 8 | -------------------------------------------------------------------------------- /tox.ini: -------------------------------------------------------------------------------- 1 | [tox] 2 | envlist = py27, py35, py36 3 | 4 | [testenv] 5 | deps = 6 | pytest==3.1.1 7 | pytest-mock==1.2 8 | redislite==3.2.311 9 | werkzeug==0.12.2 10 | commands = py.test -vs {toxinidir}/tests 11 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | --- 2 | sudo: false 3 | 4 | language: python 5 | 6 | python: 7 | - "2.7" 8 | - "3.5" 9 | - "3.6" 10 | 11 | install: 12 | - pip install --upgrade tox-travis 13 | 14 | script: 15 | - tox 16 | -------------------------------------------------------------------------------- /cachelper/local.py: -------------------------------------------------------------------------------- 1 | import time 2 | import functools 3 | 4 | 5 | __all__ = ['memoize'] 6 | 7 | 8 | def memoize(timeout=300): 9 | def decorator(f): 10 | cache = {} 11 | 12 | @functools.wraps(f) 13 | def _(*args, **kwargs): 14 | sorted_kw = sorted(kwargs.items()) 15 | key = (args, tuple(sorted_kw)) 16 | value, cache_time = cache.get(key, (None, None)) 17 | now = time.time() 18 | if cache_time is None or (now - cache_time) > timeout: 19 | value, _ = cache[key] = (f(*args, **kwargs), now) 20 | return value 21 | 22 | _.clear_cachelper_cache = cache.clear 23 | 24 | return _ 25 | 26 | return decorator 27 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | from distutils.core import setup 2 | 3 | with open('README.rst') as f: 4 | long_description = f.read() 5 | 6 | 7 | setup( 8 | name='cachelper', 9 | version='0.3.0', 10 | description='A collection of cache helpers', 11 | long_description=long_description, 12 | url='https://github.com/suzaku/cachelper', 13 | license='MIT', 14 | author='Satoru Logic', 15 | author_email='satorulogic@gmail.com', 16 | packages=['cachelper'], 17 | # See https://pypi.python.org/pypi?%3Aaction=list_classifiers 18 | classifiers=[ 19 | 'Development Status :: 3 - Alpha', 20 | 'Intended Audience :: Developers', 21 | 'License :: OSI Approved :: MIT License', 22 | 'Programming Language :: Python :: 2', 23 | 'Programming Language :: Python :: 2.7', 24 | 'Programming Language :: Python :: 3', 25 | 'Programming Language :: Python :: 3.5', 26 | 'Programming Language :: Python :: 3.6', 27 | ], 28 | keywords='cache', 29 | ) 30 | -------------------------------------------------------------------------------- /LICENSE.md: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2017 satoru 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /tests/test_local.py: -------------------------------------------------------------------------------- 1 | import cachelper 2 | 3 | 4 | class TestMemorize: 5 | 6 | def test_should_cache_return_value(self, mocker): 7 | func = mocker.Mock() 8 | func.side_effect = lambda i: i * 2 9 | func.__name__ = 'double' 10 | 11 | cached = cachelper.memoize()(func) 12 | assert cached(2) == 4 13 | assert cached(2) == 4 14 | assert func.call_count == 1 15 | assert cached(4) == 8 16 | assert cached(4) == 8 17 | assert func.call_count == 2 18 | 19 | def test_can_clear_cache(self, mocker): 20 | func = mocker.Mock() 21 | func.side_effect = lambda i: i * 2 22 | func.__name__ = 'double' 23 | 24 | decorator = cachelper.memoize() 25 | cached = decorator(func) 26 | cached(10) 27 | cached.clear_cachelper_cache() 28 | cached(10) 29 | assert func.call_count == 2 30 | 31 | def test_can_decorate_method(self, mocker): 32 | tracker = mocker.Mock() 33 | 34 | class A(object): 35 | 36 | @cachelper.memoize() 37 | def calculate(self, x, y): 38 | tracker() 39 | return x + y 40 | 41 | a1 = A() 42 | assert a1.calculate(1, 2) == 3 43 | assert a1.calculate(1, 2) == 3 44 | assert tracker.call_count == 1 45 | a2 = A() 46 | assert a2.calculate(1, 2) == 3 47 | assert tracker.call_count == 2 48 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | MANIFEST 2 | 3 | # Byte-compiled / optimized / DLL files 4 | __pycache__/ 5 | *.py[cod] 6 | *$py.class 7 | 8 | # C extensions 9 | *.so 10 | 11 | # Distribution / packaging 12 | .Python 13 | build/ 14 | develop-eggs/ 15 | dist/ 16 | downloads/ 17 | eggs/ 18 | .eggs/ 19 | lib/ 20 | lib64/ 21 | parts/ 22 | sdist/ 23 | var/ 24 | wheels/ 25 | *.egg-info/ 26 | .installed.cfg 27 | *.egg 28 | 29 | # PyInstaller 30 | # Usually these files are written by a python script from a template 31 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 32 | *.manifest 33 | *.spec 34 | 35 | # Installer logs 36 | pip-log.txt 37 | pip-delete-this-directory.txt 38 | 39 | # Unit test / coverage reports 40 | htmlcov/ 41 | .tox/ 42 | .coverage 43 | .coverage.* 44 | .cache 45 | nosetests.xml 46 | coverage.xml 47 | *.cover 48 | .hypothesis/ 49 | 50 | # Translations 51 | *.mo 52 | *.pot 53 | 54 | # Django stuff: 55 | *.log 56 | local_settings.py 57 | 58 | # Flask stuff: 59 | instance/ 60 | .webassets-cache 61 | 62 | # Scrapy stuff: 63 | .scrapy 64 | 65 | # Sphinx documentation 66 | docs/_build/ 67 | 68 | # PyBuilder 69 | target/ 70 | 71 | # Jupyter Notebook 72 | .ipynb_checkpoints 73 | 74 | # pyenv 75 | .python-version 76 | 77 | # celery beat schedule file 78 | celerybeat-schedule 79 | 80 | # SageMath parsed files 81 | *.sage.py 82 | 83 | # Environments 84 | .env 85 | .venv 86 | env/ 87 | venv/ 88 | ENV/ 89 | 90 | # Spyder project settings 91 | .spyderproject 92 | .spyproject 93 | 94 | # Rope project settings 95 | .ropeproject 96 | 97 | # mkdocs documentation 98 | /site 99 | 100 | # mypy 101 | .mypy_cache/ -------------------------------------------------------------------------------- /tests/test_remote.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | from redislite import StrictRedis 3 | from werkzeug.contrib.cache import RedisCache as _RedisCache 4 | 5 | import cachelper 6 | 7 | 8 | class RedisCache(_RedisCache, cachelper.HelperMixin): 9 | 10 | def get_many(self, keys): 11 | return _RedisCache.get_many(self, *keys) 12 | 13 | 14 | @pytest.fixture() 15 | def redis(tmpdir): 16 | yield StrictRedis(str(tmpdir.join('redis.db'))) 17 | 18 | 19 | @pytest.fixture() 20 | def cache(redis): 21 | yield RedisCache(redis) 22 | 23 | 24 | class TestCacheCall: 25 | 26 | def test_should_cache_result(self, cache, mocker): 27 | key = 'some-key' 28 | func = mocker.Mock() 29 | ret = func.return_value = {'name': 'Jojo'} 30 | 31 | assert cache.call(func, key) == ret 32 | assert cache.call(func, key) == ret 33 | assert func.call_count == 1 34 | 35 | 36 | def test_cache_map(cache, mocker): 37 | tracker = mocker.Mock() 38 | 39 | def add(a, b): 40 | tracker() 41 | if a == 5: 42 | return None 43 | return a + b 44 | 45 | key_pat = "key-{a}-{b}" 46 | 47 | assert cache.map(key_pat, add, [(1, 2), (3, 4), (5, 1)]) == [3, 7, None] 48 | assert cache.map(key_pat, add, [(1, 2), (3, 4), (5, 1)]) == [3, 7, None] 49 | assert cache.map( 50 | lambda a, b: key_pat.format(a=a, b=b), 51 | add, [(1, 2), (3, 4), (5, 1)] 52 | ) == [3, 7, None] 53 | assert tracker.call_count == 3 54 | 55 | assert cache.map(key_pat, add, [(10, 10), (1, 2)]) == [20, 3] 56 | assert tracker.call_count == 4 57 | 58 | 59 | class TestDecorator: 60 | 61 | def test_should_support_function_as_key_tmpl(self, cache, mocker): 62 | tracker = mocker.Mock() 63 | 64 | def key(first, last): 65 | return "key-%s-%s" % (first, last) 66 | 67 | @cache(key, timeout=300) 68 | def get_name(first, last): 69 | tracker() 70 | return first + ' ' + last 71 | 72 | assert get_name('Kujo', 'Jotaro') == 'Kujo Jotaro' 73 | assert get_name('Kujo', 'Jotaro') == 'Kujo Jotaro' 74 | assert tracker.call_count == 1 75 | assert get_name('Kujo', 'Jolyne') == 'Kujo Jolyne' 76 | assert tracker.call_count == 2 77 | 78 | def test_should_cache_result(self, cache, mocker): 79 | tracker = mocker.Mock() 80 | 81 | @cache("key-{first}-{last}", timeout=300) 82 | def get_name(first, last): 83 | tracker() 84 | return first + ' ' + last 85 | 86 | assert get_name('Kujo', 'Jotaro') == 'Kujo Jotaro' 87 | assert get_name('Kujo', 'Jotaro') == 'Kujo Jotaro' 88 | assert tracker.call_count == 1 89 | 90 | def test_should_cache_empty_result(self, cache, mocker): 91 | tracker = mocker.Mock() 92 | 93 | @cache("key-{first}-{last}", timeout=300) 94 | def get_name(first, last): 95 | tracker() 96 | return None 97 | 98 | assert get_name('Kujo', 'Jotaro') is None 99 | assert get_name('Kujo', 'Jotaro') is None 100 | assert tracker.call_count == 1 101 | 102 | def test_can_clear_cache(self, cache, mocker): 103 | tracker = mocker.Mock() 104 | 105 | @cache("key-{first}-{last}", timeout=300) 106 | def get_name(first, last): 107 | tracker() 108 | return first + ' ' + last 109 | 110 | assert get_name('Kujo', 'Jotaro') == 'Kujo Jotaro' 111 | get_name.clear_cachelper_cache('Kujo', 'Jotaro') 112 | assert get_name('Kujo', 'Jotaro') == 'Kujo Jotaro' 113 | assert tracker.call_count == 2 114 | -------------------------------------------------------------------------------- /README.rst: -------------------------------------------------------------------------------- 1 | cachelper 2 | ########## 3 | 4 | .. image:: https://travis-ci.org/suzaku/cachelper.svg?branch=master 5 | :target: https://travis-ci.org/suzaku/cachelper 6 | .. image:: https://img.shields.io/pypi/v/cachelper.svg 7 | :target: https://pypi.python.org/pypi/cachelper 8 | .. image:: https://bettercodehub.com/edge/badge/suzaku/cachelper?branch=master 9 | 10 | Useful cache helpers in one package! 11 | 12 | Install 13 | ******* 14 | 15 | .. code-block:: bash 16 | 17 | pip install cachelper 18 | 19 | Helpers 20 | ******* 21 | 22 | In memory cache 23 | =============== 24 | 25 | memoize 26 | --------------- 27 | 28 | Caching function return values in memory. 29 | 30 | 31 | .. code-block:: python 32 | 33 | import cachelper 34 | 35 | @cachelper.memoize() 36 | def fibo(n): 37 | if n in (0, 1): 38 | return 1 39 | return fibo(n - 1) + fibo(n - 2) 40 | 41 | fibo(10) 42 | 43 | Cache with Redis/Memcached 44 | ============================== 45 | 46 | Using the HelperMixin 47 | --------------------- 48 | 49 | ``HelperMixin`` can be used with any cache client classes that implement the following methods: 50 | 51 | - ``def get(self, key)`` 52 | - ``def set(self, key, value, timeout=None)`` 53 | - ``def delete(self, key)`` 54 | - ``def get_many(self, keys)`` 55 | - ``def set_many(self, mapping, timeout=None)`` 56 | 57 | For example, `RedisCache` from `werkzeug` is one such class: 58 | 59 | .. code-block:: python 60 | 61 | from redis import StrictRedis 62 | from werkzeug.contrib.cache import RedisCache as _RedisCache 63 | 64 | from cachelper import HelperMixin 65 | 66 | class RedisCache(_RedisCache, HelperMixin): 67 | '''werkzeug.contrib.cache.RedisCache mixed with HelperMixin''' 68 | 69 | def get_many(self, keys): 70 | return super().get_many(*keys) 71 | 72 | rds = StrictRedis() 73 | cache = RedisCache(rds) 74 | 75 | This mixin defines these methods: ``call``, ``map``, ``__call__``. 76 | If your class already defines methods of the same name, the mixin methods may not work correctly. 77 | 78 | 79 | cache decorator 80 | --------------- 81 | 82 | Add cache by decorating a function or method. 83 | 84 | .. code-block:: python 85 | 86 | @cache("key-{user_id}", timeout=300) 87 | def get_name(user_id): 88 | # Fetch user name from database 89 | ... 90 | 91 | The cache key template can also be a function which acts as a key factory: 92 | 93 | .. code-block:: python 94 | 95 | def name_key(user_id): 96 | return "key-%s" % user_id 97 | 98 | @cache(name_key, timeout=300) 99 | def get_name(user_id): 100 | # Fetch user name from database 101 | ... 102 | 103 | Just make sure the key factory function accepts the same parameters as the cached 104 | function and returns the key. 105 | 106 | cached function calls 107 | ------------------------------ 108 | 109 | Sometimes we don't want to cache all calls to a specific function. 110 | So the decorator is not suitable, we may cache the call instead the function in this case: 111 | 112 | 113 | .. code-block:: python 114 | 115 | def get_name(user_id): 116 | # Fetch user name from database 117 | ... 118 | 119 | user_id = 42 120 | key = "key-{user_id}".format(user_id=user_id) 121 | cache.call(lambda: get_name(user_id), key, timeout=300) 122 | 123 | cached multiple calls 124 | ------------------------------ 125 | 126 | For most cache backends, it's much faster to get or set caches in bulk. 127 | 128 | .. code-block:: python 129 | 130 | def get_name(user_id): 131 | # Fetch user name from database 132 | ... 133 | 134 | user_ids = [1, 2, 42, 1984] 135 | names = cache.map("key-{user_id}", get_name, user_ids, timeout=300) 136 | 137 | .. image:: https://app.codesponsor.io/embed/MY7qFCdB7bDgiBqdjtV9ASYi/suzaku/cachelper.svg 138 | :width: 888px 139 | :height: 68px 140 | :alt: Sponsor 141 | :target: https://app.codesponsor.io/link/MY7qFCdB7bDgiBqdjtV9ASYi/suzaku/cachelper 142 | -------------------------------------------------------------------------------- /cachelper/remote.py: -------------------------------------------------------------------------------- 1 | import inspect 2 | import functools 3 | 4 | 5 | __all__ = ['HelperMixin'] 6 | 7 | 8 | class HelperMixin(object): 9 | 10 | '''A mixin class that adds helpers to basic cache classes. 11 | 12 | Basic cache classes are classes with the following methods: 13 | 14 | - def get(self, key) 15 | - def set(self, key, value, timeout=None) 16 | - def delete(self, key) 17 | - def get_dict(self, keys) 18 | - def set_many(self, mapping, timeout=None) 19 | ''' 20 | 21 | def call(self, func, key, timeout=None): 22 | '''Wraps a function call with cache. 23 | 24 | Args: 25 | func (function): the function to call. 26 | key (str): the cache key for this call. 27 | timeout (int): the cache timeout for the key (the 28 | unit of this parameter depends on 29 | the cache class you use, for example, 30 | if you use the classes from werkzeug, 31 | then timeout is in seconds.) 32 | 33 | Returns: 34 | The return value of calling func 35 | ''' 36 | result = self.get(key) 37 | if result == NONE_RESULT: 38 | return None 39 | if result is None: 40 | result = func() 41 | self.set( 42 | key, 43 | result if result is not None else NONE_RESULT, 44 | timeout 45 | ) 46 | return result 47 | 48 | def map(self, key_pattern, func, all_args, timeout=None): 49 | '''Cache return value of multiple calls. 50 | 51 | Args: 52 | key_pattern (str): the key pattern to use for generating 53 | keys for caches of the decorated function. 54 | func (function): the function to call. 55 | all_args (list): a list of args to be used to make calls to 56 | the function. 57 | timeout (int): the cache timeout 58 | 59 | Returns: 60 | A list of the return values of the calls. 61 | 62 | Example:: 63 | 64 | def add(a, b): 65 | return a + b 66 | 67 | cache.map(key_pat, add, [(1, 2), (3, 4)]) == [3, 7] 68 | ''' 69 | results = [] 70 | keys = [ 71 | make_key(key_pattern, func, args, {}) 72 | for args in all_args 73 | ] 74 | cached = dict(zip(keys, self.get_many(keys))) 75 | cache_to_add = {} 76 | for key, args in zip(keys, all_args): 77 | val = cached[key] 78 | if val is None: 79 | val = func(*args) 80 | cache_to_add[key] = val if val is not None else NONE_RESULT 81 | if val == NONE_RESULT: 82 | val = None 83 | results.append(val) 84 | if cache_to_add: 85 | self.set_many(cache_to_add, timeout) 86 | return results 87 | 88 | def __call__(self, key_pattern, timeout=None): 89 | '''Use the cache object as a decorator factory. 90 | 91 | Args: 92 | key_pattern (str): the key pattern to use for generating 93 | keys for caches of the decorated function 94 | timeout (int): the cache timeout for the key 95 | 96 | Returns: 97 | a decorator that cache return values of the decorated functions 98 | in keys generated based on the given key pattern 99 | and the calling args. 100 | 101 | Here's an example:: 102 | 103 | @cache('key-name-for-{id}') 104 | def get_name(id): 105 | return 'name' + id 106 | ''' 107 | 108 | def decorator(f): 109 | 110 | @functools.wraps(f) 111 | def inner(*args, **kwargs): 112 | key = make_key(key_pattern, f, args, kwargs) 113 | 114 | def x(): 115 | return f(*args, **kwargs) 116 | 117 | return self.call(x, key, timeout) 118 | 119 | def clear(*args, **kwargs): 120 | key = make_key(key_pattern, f, args, kwargs) 121 | self.delete(key) 122 | 123 | inner.clear_cachelper_cache = clear 124 | 125 | return inner 126 | 127 | return decorator 128 | 129 | 130 | def make_key(key_pattern, func, args, kwargs): 131 | if not callable(key_pattern): 132 | def gen_key(*args, **kwargs): 133 | callargs = inspect.getcallargs(func, *args, **kwargs) 134 | return key_pattern.format(**callargs) 135 | else: 136 | gen_key = key_pattern 137 | return gen_key(*args, **kwargs) 138 | 139 | 140 | class Empty(object): 141 | 142 | obj = None 143 | 144 | def __new__(cls): 145 | if not cls.obj: 146 | cls.obj = object.__new__(cls) 147 | return cls.obj 148 | 149 | def __eq__(self, other): 150 | return isinstance(other, Empty) 151 | 152 | def __ne__(self, other): 153 | return not self == other 154 | 155 | def __nonzero__(self): 156 | return False 157 | 158 | def __str__(self): 159 | return "" 160 | 161 | __repr__ = __str__ 162 | 163 | 164 | NONE_RESULT = Empty() 165 | --------------------------------------------------------------------------------