├── .github └── workflows │ ├── integration.yml │ └── release.yml ├── .gitignore ├── .pylintrc ├── LICENSE ├── MANIFEST.in ├── README.rst ├── diskcache ├── __init__.py ├── cli.py ├── core.py ├── djangocache.py ├── fanout.py ├── persistent.py └── recipes.py ├── docs ├── Makefile ├── _static │ ├── core-p1-delete.png │ ├── core-p1-get.png │ ├── core-p1-set.png │ ├── core-p8-delete.png │ ├── core-p8-get.png │ ├── core-p8-set.png │ ├── custom.css │ ├── djangocache-delete.png │ ├── djangocache-get.png │ ├── djangocache-set.png │ ├── early-recomputation-03.png │ ├── early-recomputation-05.png │ ├── early-recomputation.png │ ├── gj-logo.png │ ├── no-caching.png │ ├── synchronized-locking.png │ └── traditional-caching.png ├── _templates │ └── gumroad.html ├── api.rst ├── cache-benchmarks.rst ├── case-study-landing-page-caching.rst ├── case-study-web-crawler.rst ├── conf.py ├── development.rst ├── djangocache-benchmarks.rst ├── index.rst ├── make.bat ├── sf-python-2017-meetup-talk.rst └── tutorial.rst ├── mypy.ini ├── requirements-dev.txt ├── requirements.txt ├── setup.py ├── tests ├── __init__.py ├── benchmark_core.py ├── benchmark_djangocache.py ├── benchmark_glob.py ├── benchmark_incr.py ├── benchmark_kv_store.py ├── db.sqlite3 ├── issue_109.py ├── issue_85.py ├── models.py ├── plot.py ├── plot_early_recompute.py ├── settings.py ├── settings_benchmark.py ├── stress_test_core.py ├── stress_test_deque.py ├── stress_test_deque_mp.py ├── stress_test_fanout.py ├── stress_test_index.py ├── stress_test_index_mp.py ├── test_core.py ├── test_deque.py ├── test_djangocache.py ├── test_doctest.py ├── test_fanout.py ├── test_index.py ├── test_recipes.py ├── timings_core_p1.txt ├── timings_core_p8.txt ├── timings_djangocache.txt ├── timings_glob.txt └── utils.py └── tox.ini /.github/workflows/integration.yml: -------------------------------------------------------------------------------- 1 | name: integration 2 | 3 | on: [push, pull_request] 4 | 5 | jobs: 6 | 7 | checks: 8 | runs-on: ubuntu-latest 9 | strategy: 10 | max-parallel: 8 11 | matrix: 12 | check: [bluecheck, doc8, docs, flake8, isortcheck, mypy, pylint, rstcheck] 13 | 14 | steps: 15 | - uses: actions/checkout@v3 16 | - name: Set up Python 17 | uses: actions/setup-python@v4 18 | with: 19 | python-version: '3.11' 20 | - name: Install dependencies 21 | run: | 22 | pip install --upgrade pip 23 | pip install tox 24 | - name: Run checks with tox 25 | run: | 26 | tox -e ${{ matrix.check }} 27 | 28 | tests: 29 | needs: checks 30 | runs-on: ${{ matrix.os }} 31 | strategy: 32 | max-parallel: 8 33 | matrix: 34 | os: [ubuntu-latest, macos-latest, windows-latest] 35 | python-version: [3.8, 3.9, '3.10', 3.11] 36 | 37 | steps: 38 | - name: Set up Python ${{ matrix.python-version }} x64 39 | uses: actions/setup-python@v4 40 | with: 41 | python-version: ${{ matrix.python-version }} 42 | architecture: x64 43 | 44 | - uses: actions/checkout@v3 45 | 46 | - name: Install tox 47 | run: | 48 | pip install --upgrade pip 49 | pip install tox 50 | 51 | - name: Test with tox 52 | run: tox -e py 53 | -------------------------------------------------------------------------------- /.github/workflows/release.yml: -------------------------------------------------------------------------------- 1 | name: release 2 | 3 | on: 4 | push: 5 | tags: 6 | - v* 7 | 8 | jobs: 9 | 10 | upload: 11 | runs-on: ubuntu-latest 12 | permissions: 13 | id-token: write 14 | 15 | steps: 16 | - uses: actions/checkout@v3 17 | 18 | - name: Set up Python 19 | uses: actions/setup-python@v4 20 | with: 21 | python-version: '3.11' 22 | 23 | - name: Install build 24 | run: pip install build 25 | 26 | - name: Create build 27 | run: python -m build 28 | 29 | - name: Publish package distributions to PyPI 30 | uses: pypa/gh-action-pypi-publish@release/v1 31 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Python byte-code 2 | *.py[co] 3 | 4 | # virutalenv directories 5 | /env*/ 6 | /.venv*/ 7 | 8 | # test files/directories 9 | /.cache/ 10 | .coverage* 11 | .pytest_cache/ 12 | /.tox/ 13 | 14 | # setup and upload directories 15 | /build/ 16 | /dist/ 17 | /diskcache.egg-info/ 18 | /docs/_build/ 19 | 20 | # macOS metadata 21 | .DS_Store 22 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright 2016-2022 Grant Jenks 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); you may not use 4 | this file except in compliance with the License. You may obtain a copy of the 5 | License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software distributed 10 | under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR 11 | CONDITIONS OF ANY KIND, either express or implied. See the License for the 12 | specific language governing permissions and limitations under the License. 13 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include README.rst LICENSE 2 | -------------------------------------------------------------------------------- /diskcache/__init__.py: -------------------------------------------------------------------------------- 1 | """ 2 | DiskCache API Reference 3 | ======================= 4 | 5 | The :doc:`tutorial` provides a helpful walkthrough of most methods. 6 | """ 7 | 8 | from .core import ( 9 | DEFAULT_SETTINGS, 10 | ENOVAL, 11 | EVICTION_POLICY, 12 | UNKNOWN, 13 | Cache, 14 | Disk, 15 | EmptyDirWarning, 16 | JSONDisk, 17 | Timeout, 18 | UnknownFileWarning, 19 | ) 20 | from .fanout import FanoutCache 21 | from .persistent import Deque, Index 22 | from .recipes import ( 23 | Averager, 24 | BoundedSemaphore, 25 | Lock, 26 | RLock, 27 | barrier, 28 | memoize_stampede, 29 | throttle, 30 | ) 31 | 32 | __all__ = [ 33 | 'Averager', 34 | 'BoundedSemaphore', 35 | 'Cache', 36 | 'DEFAULT_SETTINGS', 37 | 'Deque', 38 | 'Disk', 39 | 'ENOVAL', 40 | 'EVICTION_POLICY', 41 | 'EmptyDirWarning', 42 | 'FanoutCache', 43 | 'Index', 44 | 'JSONDisk', 45 | 'Lock', 46 | 'RLock', 47 | 'Timeout', 48 | 'UNKNOWN', 49 | 'UnknownFileWarning', 50 | 'barrier', 51 | 'memoize_stampede', 52 | 'throttle', 53 | ] 54 | 55 | try: 56 | from .djangocache import DjangoCache # noqa 57 | 58 | __all__.append('DjangoCache') 59 | except Exception: # pylint: disable=broad-except # pragma: no cover 60 | # Django not installed or not setup so ignore. 61 | pass 62 | 63 | __title__ = 'diskcache' 64 | __version__ = '5.6.3' 65 | __build__ = 0x050603 66 | __author__ = 'Grant Jenks' 67 | __license__ = 'Apache 2.0' 68 | __copyright__ = 'Copyright 2016-2023 Grant Jenks' 69 | -------------------------------------------------------------------------------- /diskcache/cli.py: -------------------------------------------------------------------------------- 1 | """Command line interface to disk cache.""" 2 | -------------------------------------------------------------------------------- /diskcache/djangocache.py: -------------------------------------------------------------------------------- 1 | """Django-compatible disk and file backed cache.""" 2 | 3 | from functools import wraps 4 | 5 | from django.core.cache.backends.base import BaseCache 6 | 7 | try: 8 | from django.core.cache.backends.base import DEFAULT_TIMEOUT 9 | except ImportError: # pragma: no cover 10 | # For older versions of Django simply use 300 seconds. 11 | DEFAULT_TIMEOUT = 300 12 | 13 | from .core import ENOVAL, args_to_key, full_name 14 | from .fanout import FanoutCache 15 | 16 | 17 | class DjangoCache(BaseCache): 18 | """Django-compatible disk and file backed cache.""" 19 | 20 | def __init__(self, directory, params): 21 | """Initialize DjangoCache instance. 22 | 23 | :param str directory: cache directory 24 | :param dict params: cache parameters 25 | 26 | """ 27 | super().__init__(params) 28 | shards = params.get('SHARDS', 8) 29 | timeout = params.get('DATABASE_TIMEOUT', 0.010) 30 | options = params.get('OPTIONS', {}) 31 | self._cache = FanoutCache(directory, shards, timeout, **options) 32 | 33 | @property 34 | def directory(self): 35 | """Cache directory.""" 36 | return self._cache.directory 37 | 38 | def cache(self, name): 39 | """Return Cache with given `name` in subdirectory. 40 | 41 | :param str name: subdirectory name for Cache 42 | :return: Cache with given name 43 | 44 | """ 45 | return self._cache.cache(name) 46 | 47 | def deque(self, name, maxlen=None): 48 | """Return Deque with given `name` in subdirectory. 49 | 50 | :param str name: subdirectory name for Deque 51 | :param maxlen: max length (default None, no max) 52 | :return: Deque with given name 53 | 54 | """ 55 | return self._cache.deque(name, maxlen=maxlen) 56 | 57 | def index(self, name): 58 | """Return Index with given `name` in subdirectory. 59 | 60 | :param str name: subdirectory name for Index 61 | :return: Index with given name 62 | 63 | """ 64 | return self._cache.index(name) 65 | 66 | def add( 67 | self, 68 | key, 69 | value, 70 | timeout=DEFAULT_TIMEOUT, 71 | version=None, 72 | read=False, 73 | tag=None, 74 | retry=True, 75 | ): 76 | """Set a value in the cache if the key does not already exist. If 77 | timeout is given, that timeout will be used for the key; otherwise the 78 | default cache timeout will be used. 79 | 80 | Return True if the value was stored, False otherwise. 81 | 82 | :param key: key for item 83 | :param value: value for item 84 | :param float timeout: seconds until the item expires 85 | (default 300 seconds) 86 | :param int version: key version number (default None, cache parameter) 87 | :param bool read: read value as bytes from file (default False) 88 | :param str tag: text to associate with key (default None) 89 | :param bool retry: retry if database timeout occurs (default True) 90 | :return: True if item was added 91 | 92 | """ 93 | # pylint: disable=arguments-differ 94 | key = self.make_key(key, version=version) 95 | timeout = self.get_backend_timeout(timeout=timeout) 96 | return self._cache.add(key, value, timeout, read, tag, retry) 97 | 98 | def get( 99 | self, 100 | key, 101 | default=None, 102 | version=None, 103 | read=False, 104 | expire_time=False, 105 | tag=False, 106 | retry=False, 107 | ): 108 | """Fetch a given key from the cache. If the key does not exist, return 109 | default, which itself defaults to None. 110 | 111 | :param key: key for item 112 | :param default: return value if key is missing (default None) 113 | :param int version: key version number (default None, cache parameter) 114 | :param bool read: if True, return file handle to value 115 | (default False) 116 | :param float expire_time: if True, return expire_time in tuple 117 | (default False) 118 | :param tag: if True, return tag in tuple (default False) 119 | :param bool retry: retry if database timeout occurs (default False) 120 | :return: value for item if key is found else default 121 | 122 | """ 123 | # pylint: disable=arguments-differ 124 | key = self.make_key(key, version=version) 125 | return self._cache.get(key, default, read, expire_time, tag, retry) 126 | 127 | def read(self, key, version=None): 128 | """Return file handle corresponding to `key` from Cache. 129 | 130 | :param key: Python key to retrieve 131 | :param int version: key version number (default None, cache parameter) 132 | :return: file open for reading in binary mode 133 | :raises KeyError: if key is not found 134 | 135 | """ 136 | key = self.make_key(key, version=version) 137 | return self._cache.read(key) 138 | 139 | def set( 140 | self, 141 | key, 142 | value, 143 | timeout=DEFAULT_TIMEOUT, 144 | version=None, 145 | read=False, 146 | tag=None, 147 | retry=True, 148 | ): 149 | """Set a value in the cache. If timeout is given, that timeout will be 150 | used for the key; otherwise the default cache timeout will be used. 151 | 152 | :param key: key for item 153 | :param value: value for item 154 | :param float timeout: seconds until the item expires 155 | (default 300 seconds) 156 | :param int version: key version number (default None, cache parameter) 157 | :param bool read: read value as bytes from file (default False) 158 | :param str tag: text to associate with key (default None) 159 | :param bool retry: retry if database timeout occurs (default True) 160 | :return: True if item was set 161 | 162 | """ 163 | # pylint: disable=arguments-differ 164 | key = self.make_key(key, version=version) 165 | timeout = self.get_backend_timeout(timeout=timeout) 166 | return self._cache.set(key, value, timeout, read, tag, retry) 167 | 168 | def touch(self, key, timeout=DEFAULT_TIMEOUT, version=None, retry=True): 169 | """Touch a key in the cache. If timeout is given, that timeout will be 170 | used for the key; otherwise the default cache timeout will be used. 171 | 172 | :param key: key for item 173 | :param float timeout: seconds until the item expires 174 | (default 300 seconds) 175 | :param int version: key version number (default None, cache parameter) 176 | :param bool retry: retry if database timeout occurs (default True) 177 | :return: True if key was touched 178 | 179 | """ 180 | # pylint: disable=arguments-differ 181 | key = self.make_key(key, version=version) 182 | timeout = self.get_backend_timeout(timeout=timeout) 183 | return self._cache.touch(key, timeout, retry) 184 | 185 | def pop( 186 | self, 187 | key, 188 | default=None, 189 | version=None, 190 | expire_time=False, 191 | tag=False, 192 | retry=True, 193 | ): 194 | """Remove corresponding item for `key` from cache and return value. 195 | 196 | If `key` is missing, return `default`. 197 | 198 | Operation is atomic. Concurrent operations will be serialized. 199 | 200 | :param key: key for item 201 | :param default: return value if key is missing (default None) 202 | :param int version: key version number (default None, cache parameter) 203 | :param float expire_time: if True, return expire_time in tuple 204 | (default False) 205 | :param tag: if True, return tag in tuple (default False) 206 | :param bool retry: retry if database timeout occurs (default True) 207 | :return: value for item if key is found else default 208 | 209 | """ 210 | key = self.make_key(key, version=version) 211 | return self._cache.pop(key, default, expire_time, tag, retry) 212 | 213 | def delete(self, key, version=None, retry=True): 214 | """Delete a key from the cache, failing silently. 215 | 216 | :param key: key for item 217 | :param int version: key version number (default None, cache parameter) 218 | :param bool retry: retry if database timeout occurs (default True) 219 | :return: True if item was deleted 220 | 221 | """ 222 | # pylint: disable=arguments-differ 223 | key = self.make_key(key, version=version) 224 | return self._cache.delete(key, retry) 225 | 226 | def incr(self, key, delta=1, version=None, default=None, retry=True): 227 | """Increment value by delta for item with key. 228 | 229 | If key is missing and default is None then raise KeyError. Else if key 230 | is missing and default is not None then use default for value. 231 | 232 | Operation is atomic. All concurrent increment operations will be 233 | counted individually. 234 | 235 | Assumes value may be stored in a SQLite column. Most builds that target 236 | machines with 64-bit pointer widths will support 64-bit signed 237 | integers. 238 | 239 | :param key: key for item 240 | :param int delta: amount to increment (default 1) 241 | :param int version: key version number (default None, cache parameter) 242 | :param int default: value if key is missing (default None) 243 | :param bool retry: retry if database timeout occurs (default True) 244 | :return: new value for item on success else None 245 | :raises ValueError: if key is not found and default is None 246 | 247 | """ 248 | # pylint: disable=arguments-differ 249 | key = self.make_key(key, version=version) 250 | try: 251 | return self._cache.incr(key, delta, default, retry) 252 | except KeyError: 253 | raise ValueError("Key '%s' not found" % key) from None 254 | 255 | def decr(self, key, delta=1, version=None, default=None, retry=True): 256 | """Decrement value by delta for item with key. 257 | 258 | If key is missing and default is None then raise KeyError. Else if key 259 | is missing and default is not None then use default for value. 260 | 261 | Operation is atomic. All concurrent decrement operations will be 262 | counted individually. 263 | 264 | Unlike Memcached, negative values are supported. Value may be 265 | decremented below zero. 266 | 267 | Assumes value may be stored in a SQLite column. Most builds that target 268 | machines with 64-bit pointer widths will support 64-bit signed 269 | integers. 270 | 271 | :param key: key for item 272 | :param int delta: amount to decrement (default 1) 273 | :param int version: key version number (default None, cache parameter) 274 | :param int default: value if key is missing (default None) 275 | :param bool retry: retry if database timeout occurs (default True) 276 | :return: new value for item on success else None 277 | :raises ValueError: if key is not found and default is None 278 | 279 | """ 280 | # pylint: disable=arguments-differ 281 | return self.incr(key, -delta, version, default, retry) 282 | 283 | def has_key(self, key, version=None): 284 | """Returns True if the key is in the cache and has not expired. 285 | 286 | :param key: key for item 287 | :param int version: key version number (default None, cache parameter) 288 | :return: True if key is found 289 | 290 | """ 291 | key = self.make_key(key, version=version) 292 | return key in self._cache 293 | 294 | def expire(self): 295 | """Remove expired items from cache. 296 | 297 | :return: count of items removed 298 | 299 | """ 300 | return self._cache.expire() 301 | 302 | def stats(self, enable=True, reset=False): 303 | """Return cache statistics hits and misses. 304 | 305 | :param bool enable: enable collecting statistics (default True) 306 | :param bool reset: reset hits and misses to 0 (default False) 307 | :return: (hits, misses) 308 | 309 | """ 310 | return self._cache.stats(enable=enable, reset=reset) 311 | 312 | def create_tag_index(self): 313 | """Create tag index on cache database. 314 | 315 | Better to initialize cache with `tag_index=True` than use this. 316 | 317 | :raises Timeout: if database timeout occurs 318 | 319 | """ 320 | self._cache.create_tag_index() 321 | 322 | def drop_tag_index(self): 323 | """Drop tag index on cache database. 324 | 325 | :raises Timeout: if database timeout occurs 326 | 327 | """ 328 | self._cache.drop_tag_index() 329 | 330 | def evict(self, tag): 331 | """Remove items with matching `tag` from cache. 332 | 333 | :param str tag: tag identifying items 334 | :return: count of items removed 335 | 336 | """ 337 | return self._cache.evict(tag) 338 | 339 | def cull(self): 340 | """Cull items from cache until volume is less than size limit. 341 | 342 | :return: count of items removed 343 | 344 | """ 345 | return self._cache.cull() 346 | 347 | def clear(self): 348 | """Remove *all* values from the cache at once.""" 349 | return self._cache.clear() 350 | 351 | def close(self, **kwargs): 352 | """Close the cache connection.""" 353 | # pylint: disable=unused-argument 354 | self._cache.close() 355 | 356 | def get_backend_timeout(self, timeout=DEFAULT_TIMEOUT): 357 | """Return seconds to expiration. 358 | 359 | :param float timeout: seconds until the item expires 360 | (default 300 seconds) 361 | 362 | """ 363 | if timeout == DEFAULT_TIMEOUT: 364 | timeout = self.default_timeout 365 | elif timeout == 0: 366 | # ticket 21147 - avoid time.time() related precision issues 367 | timeout = -1 368 | return None if timeout is None else timeout 369 | 370 | def memoize( 371 | self, 372 | name=None, 373 | timeout=DEFAULT_TIMEOUT, 374 | version=None, 375 | typed=False, 376 | tag=None, 377 | ignore=(), 378 | ): 379 | """Memoizing cache decorator. 380 | 381 | Decorator to wrap callable with memoizing function using cache. 382 | Repeated calls with the same arguments will lookup result in cache and 383 | avoid function evaluation. 384 | 385 | If name is set to None (default), the callable name will be determined 386 | automatically. 387 | 388 | When timeout is set to zero, function results will not be set in the 389 | cache. Cache lookups still occur, however. Read 390 | :doc:`case-study-landing-page-caching` for example usage. 391 | 392 | If typed is set to True, function arguments of different types will be 393 | cached separately. For example, f(3) and f(3.0) will be treated as 394 | distinct calls with distinct results. 395 | 396 | The original underlying function is accessible through the __wrapped__ 397 | attribute. This is useful for introspection, for bypassing the cache, 398 | or for rewrapping the function with a different cache. 399 | 400 | An additional `__cache_key__` attribute can be used to generate the 401 | cache key used for the given arguments. 402 | 403 | Remember to call memoize when decorating a callable. If you forget, 404 | then a TypeError will occur. 405 | 406 | :param str name: name given for callable (default None, automatic) 407 | :param float timeout: seconds until the item expires 408 | (default 300 seconds) 409 | :param int version: key version number (default None, cache parameter) 410 | :param bool typed: cache different types separately (default False) 411 | :param str tag: text to associate with arguments (default None) 412 | :param set ignore: positional or keyword args to ignore (default ()) 413 | :return: callable decorator 414 | 415 | """ 416 | # Caution: Nearly identical code exists in Cache.memoize 417 | if callable(name): 418 | raise TypeError('name cannot be callable') 419 | 420 | def decorator(func): 421 | """Decorator created by memoize() for callable `func`.""" 422 | base = (full_name(func),) if name is None else (name,) 423 | 424 | @wraps(func) 425 | def wrapper(*args, **kwargs): 426 | """Wrapper for callable to cache arguments and return values.""" 427 | key = wrapper.__cache_key__(*args, **kwargs) 428 | result = self.get(key, ENOVAL, version, retry=True) 429 | 430 | if result is ENOVAL: 431 | result = func(*args, **kwargs) 432 | valid_timeout = ( 433 | timeout is None 434 | or timeout == DEFAULT_TIMEOUT 435 | or timeout > 0 436 | ) 437 | if valid_timeout: 438 | self.set( 439 | key, 440 | result, 441 | timeout, 442 | version, 443 | tag=tag, 444 | retry=True, 445 | ) 446 | 447 | return result 448 | 449 | def __cache_key__(*args, **kwargs): 450 | """Make key for cache given function arguments.""" 451 | return args_to_key(base, args, kwargs, typed, ignore) 452 | 453 | wrapper.__cache_key__ = __cache_key__ 454 | return wrapper 455 | 456 | return decorator 457 | -------------------------------------------------------------------------------- /diskcache/recipes.py: -------------------------------------------------------------------------------- 1 | """Disk Cache Recipes 2 | """ 3 | 4 | import functools 5 | import math 6 | import os 7 | import random 8 | import threading 9 | import time 10 | 11 | from .core import ENOVAL, args_to_key, full_name 12 | 13 | 14 | class Averager: 15 | """Recipe for calculating a running average. 16 | 17 | Sometimes known as "online statistics," the running average maintains the 18 | total and count. The average can then be calculated at any time. 19 | 20 | Assumes the key will not be evicted. Set the eviction policy to 'none' on 21 | the cache to guarantee the key is not evicted. 22 | 23 | >>> import diskcache 24 | >>> cache = diskcache.FanoutCache() 25 | >>> ave = Averager(cache, 'latency') 26 | >>> ave.add(0.080) 27 | >>> ave.add(0.120) 28 | >>> ave.get() 29 | 0.1 30 | >>> ave.add(0.160) 31 | >>> ave.pop() 32 | 0.12 33 | >>> print(ave.get()) 34 | None 35 | 36 | """ 37 | 38 | def __init__(self, cache, key, expire=None, tag=None): 39 | self._cache = cache 40 | self._key = key 41 | self._expire = expire 42 | self._tag = tag 43 | 44 | def add(self, value): 45 | """Add `value` to average.""" 46 | with self._cache.transact(retry=True): 47 | total, count = self._cache.get(self._key, default=(0.0, 0)) 48 | total += value 49 | count += 1 50 | self._cache.set( 51 | self._key, 52 | (total, count), 53 | expire=self._expire, 54 | tag=self._tag, 55 | ) 56 | 57 | def get(self): 58 | """Get current average or return `None` if count equals zero.""" 59 | total, count = self._cache.get(self._key, default=(0.0, 0), retry=True) 60 | return None if count == 0 else total / count 61 | 62 | def pop(self): 63 | """Return current average and delete key.""" 64 | total, count = self._cache.pop(self._key, default=(0.0, 0), retry=True) 65 | return None if count == 0 else total / count 66 | 67 | 68 | class Lock: 69 | """Recipe for cross-process and cross-thread lock. 70 | 71 | Assumes the key will not be evicted. Set the eviction policy to 'none' on 72 | the cache to guarantee the key is not evicted. 73 | 74 | >>> import diskcache 75 | >>> cache = diskcache.Cache() 76 | >>> lock = Lock(cache, 'report-123') 77 | >>> lock.acquire() 78 | >>> lock.release() 79 | >>> with lock: 80 | ... pass 81 | 82 | """ 83 | 84 | def __init__(self, cache, key, expire=None, tag=None): 85 | self._cache = cache 86 | self._key = key 87 | self._expire = expire 88 | self._tag = tag 89 | 90 | def acquire(self): 91 | """Acquire lock using spin-lock algorithm.""" 92 | while True: 93 | added = self._cache.add( 94 | self._key, 95 | None, 96 | expire=self._expire, 97 | tag=self._tag, 98 | retry=True, 99 | ) 100 | if added: 101 | break 102 | time.sleep(0.001) 103 | 104 | def release(self): 105 | """Release lock by deleting key.""" 106 | self._cache.delete(self._key, retry=True) 107 | 108 | def locked(self): 109 | """Return true if the lock is acquired.""" 110 | return self._key in self._cache 111 | 112 | def __enter__(self): 113 | self.acquire() 114 | 115 | def __exit__(self, *exc_info): 116 | self.release() 117 | 118 | 119 | class RLock: 120 | """Recipe for cross-process and cross-thread re-entrant lock. 121 | 122 | Assumes the key will not be evicted. Set the eviction policy to 'none' on 123 | the cache to guarantee the key is not evicted. 124 | 125 | >>> import diskcache 126 | >>> cache = diskcache.Cache() 127 | >>> rlock = RLock(cache, 'user-123') 128 | >>> rlock.acquire() 129 | >>> rlock.acquire() 130 | >>> rlock.release() 131 | >>> with rlock: 132 | ... pass 133 | >>> rlock.release() 134 | >>> rlock.release() 135 | Traceback (most recent call last): 136 | ... 137 | AssertionError: cannot release un-acquired lock 138 | 139 | """ 140 | 141 | def __init__(self, cache, key, expire=None, tag=None): 142 | self._cache = cache 143 | self._key = key 144 | self._expire = expire 145 | self._tag = tag 146 | 147 | def acquire(self): 148 | """Acquire lock by incrementing count using spin-lock algorithm.""" 149 | pid = os.getpid() 150 | tid = threading.get_ident() 151 | pid_tid = '{}-{}'.format(pid, tid) 152 | 153 | while True: 154 | with self._cache.transact(retry=True): 155 | value, count = self._cache.get(self._key, default=(None, 0)) 156 | if pid_tid == value or count == 0: 157 | self._cache.set( 158 | self._key, 159 | (pid_tid, count + 1), 160 | expire=self._expire, 161 | tag=self._tag, 162 | ) 163 | return 164 | time.sleep(0.001) 165 | 166 | def release(self): 167 | """Release lock by decrementing count.""" 168 | pid = os.getpid() 169 | tid = threading.get_ident() 170 | pid_tid = '{}-{}'.format(pid, tid) 171 | 172 | with self._cache.transact(retry=True): 173 | value, count = self._cache.get(self._key, default=(None, 0)) 174 | is_owned = pid_tid == value and count > 0 175 | assert is_owned, 'cannot release un-acquired lock' 176 | self._cache.set( 177 | self._key, 178 | (value, count - 1), 179 | expire=self._expire, 180 | tag=self._tag, 181 | ) 182 | 183 | def __enter__(self): 184 | self.acquire() 185 | 186 | def __exit__(self, *exc_info): 187 | self.release() 188 | 189 | 190 | class BoundedSemaphore: 191 | """Recipe for cross-process and cross-thread bounded semaphore. 192 | 193 | Assumes the key will not be evicted. Set the eviction policy to 'none' on 194 | the cache to guarantee the key is not evicted. 195 | 196 | >>> import diskcache 197 | >>> cache = diskcache.Cache() 198 | >>> semaphore = BoundedSemaphore(cache, 'max-cons', value=2) 199 | >>> semaphore.acquire() 200 | >>> semaphore.acquire() 201 | >>> semaphore.release() 202 | >>> with semaphore: 203 | ... pass 204 | >>> semaphore.release() 205 | >>> semaphore.release() 206 | Traceback (most recent call last): 207 | ... 208 | AssertionError: cannot release un-acquired semaphore 209 | 210 | """ 211 | 212 | def __init__(self, cache, key, value=1, expire=None, tag=None): 213 | self._cache = cache 214 | self._key = key 215 | self._value = value 216 | self._expire = expire 217 | self._tag = tag 218 | 219 | def acquire(self): 220 | """Acquire semaphore by decrementing value using spin-lock algorithm.""" 221 | while True: 222 | with self._cache.transact(retry=True): 223 | value = self._cache.get(self._key, default=self._value) 224 | if value > 0: 225 | self._cache.set( 226 | self._key, 227 | value - 1, 228 | expire=self._expire, 229 | tag=self._tag, 230 | ) 231 | return 232 | time.sleep(0.001) 233 | 234 | def release(self): 235 | """Release semaphore by incrementing value.""" 236 | with self._cache.transact(retry=True): 237 | value = self._cache.get(self._key, default=self._value) 238 | assert self._value > value, 'cannot release un-acquired semaphore' 239 | value += 1 240 | self._cache.set( 241 | self._key, 242 | value, 243 | expire=self._expire, 244 | tag=self._tag, 245 | ) 246 | 247 | def __enter__(self): 248 | self.acquire() 249 | 250 | def __exit__(self, *exc_info): 251 | self.release() 252 | 253 | 254 | def throttle( 255 | cache, 256 | count, 257 | seconds, 258 | name=None, 259 | expire=None, 260 | tag=None, 261 | time_func=time.time, 262 | sleep_func=time.sleep, 263 | ): 264 | """Decorator to throttle calls to function. 265 | 266 | Assumes keys will not be evicted. Set the eviction policy to 'none' on the 267 | cache to guarantee the keys are not evicted. 268 | 269 | >>> import diskcache, time 270 | >>> cache = diskcache.Cache() 271 | >>> count = 0 272 | >>> @throttle(cache, 2, 1) # 2 calls per 1 second 273 | ... def increment(): 274 | ... global count 275 | ... count += 1 276 | >>> start = time.time() 277 | >>> while (time.time() - start) <= 2: 278 | ... increment() 279 | >>> count in (6, 7) # 6 or 7 calls depending on CPU load 280 | True 281 | 282 | """ 283 | 284 | def decorator(func): 285 | rate = count / float(seconds) 286 | key = full_name(func) if name is None else name 287 | now = time_func() 288 | cache.set(key, (now, count), expire=expire, tag=tag, retry=True) 289 | 290 | @functools.wraps(func) 291 | def wrapper(*args, **kwargs): 292 | while True: 293 | with cache.transact(retry=True): 294 | last, tally = cache.get(key) 295 | now = time_func() 296 | tally += (now - last) * rate 297 | delay = 0 298 | 299 | if tally > count: 300 | cache.set(key, (now, count - 1), expire) 301 | elif tally >= 1: 302 | cache.set(key, (now, tally - 1), expire) 303 | else: 304 | delay = (1 - tally) / rate 305 | 306 | if delay: 307 | sleep_func(delay) 308 | else: 309 | break 310 | 311 | return func(*args, **kwargs) 312 | 313 | return wrapper 314 | 315 | return decorator 316 | 317 | 318 | def barrier(cache, lock_factory, name=None, expire=None, tag=None): 319 | """Barrier to calling decorated function. 320 | 321 | Supports different kinds of locks: Lock, RLock, BoundedSemaphore. 322 | 323 | Assumes keys will not be evicted. Set the eviction policy to 'none' on the 324 | cache to guarantee the keys are not evicted. 325 | 326 | >>> import diskcache, time 327 | >>> cache = diskcache.Cache() 328 | >>> @barrier(cache, Lock) 329 | ... def work(num): 330 | ... print('worker started') 331 | ... time.sleep(1) 332 | ... print('worker finished') 333 | >>> import multiprocessing.pool 334 | >>> pool = multiprocessing.pool.ThreadPool(2) 335 | >>> _ = pool.map(work, range(2)) 336 | worker started 337 | worker finished 338 | worker started 339 | worker finished 340 | >>> pool.terminate() 341 | 342 | """ 343 | 344 | def decorator(func): 345 | key = full_name(func) if name is None else name 346 | lock = lock_factory(cache, key, expire=expire, tag=tag) 347 | 348 | @functools.wraps(func) 349 | def wrapper(*args, **kwargs): 350 | with lock: 351 | return func(*args, **kwargs) 352 | 353 | return wrapper 354 | 355 | return decorator 356 | 357 | 358 | def memoize_stampede( 359 | cache, expire, name=None, typed=False, tag=None, beta=1, ignore=() 360 | ): 361 | """Memoizing cache decorator with cache stampede protection. 362 | 363 | Cache stampedes are a type of system overload that can occur when parallel 364 | computing systems using memoization come under heavy load. This behaviour 365 | is sometimes also called dog-piling, cache miss storm, cache choking, or 366 | the thundering herd problem. 367 | 368 | The memoization decorator implements cache stampede protection through 369 | early recomputation. Early recomputation of function results will occur 370 | probabilistically before expiration in a background thread of 371 | execution. Early probabilistic recomputation is based on research by 372 | Vattani, A.; Chierichetti, F.; Lowenstein, K. (2015), Optimal Probabilistic 373 | Cache Stampede Prevention, VLDB, pp. 886-897, ISSN 2150-8097 374 | 375 | If name is set to None (default), the callable name will be determined 376 | automatically. 377 | 378 | If typed is set to True, function arguments of different types will be 379 | cached separately. For example, f(3) and f(3.0) will be treated as distinct 380 | calls with distinct results. 381 | 382 | The original underlying function is accessible through the `__wrapped__` 383 | attribute. This is useful for introspection, for bypassing the cache, or 384 | for rewrapping the function with a different cache. 385 | 386 | >>> from diskcache import Cache 387 | >>> cache = Cache() 388 | >>> @memoize_stampede(cache, expire=1) 389 | ... def fib(number): 390 | ... if number == 0: 391 | ... return 0 392 | ... elif number == 1: 393 | ... return 1 394 | ... else: 395 | ... return fib(number - 1) + fib(number - 2) 396 | >>> print(fib(100)) 397 | 354224848179261915075 398 | 399 | An additional `__cache_key__` attribute can be used to generate the cache 400 | key used for the given arguments. 401 | 402 | >>> key = fib.__cache_key__(100) 403 | >>> del cache[key] 404 | 405 | Remember to call memoize when decorating a callable. If you forget, then a 406 | TypeError will occur. 407 | 408 | :param cache: cache to store callable arguments and return values 409 | :param float expire: seconds until arguments expire 410 | :param str name: name given for callable (default None, automatic) 411 | :param bool typed: cache different types separately (default False) 412 | :param str tag: text to associate with arguments (default None) 413 | :param set ignore: positional or keyword args to ignore (default ()) 414 | :return: callable decorator 415 | 416 | """ 417 | # Caution: Nearly identical code exists in Cache.memoize 418 | def decorator(func): 419 | """Decorator created by memoize call for callable.""" 420 | base = (full_name(func),) if name is None else (name,) 421 | 422 | def timer(*args, **kwargs): 423 | """Time execution of `func` and return result and time delta.""" 424 | start = time.time() 425 | result = func(*args, **kwargs) 426 | delta = time.time() - start 427 | return result, delta 428 | 429 | @functools.wraps(func) 430 | def wrapper(*args, **kwargs): 431 | """Wrapper for callable to cache arguments and return values.""" 432 | key = wrapper.__cache_key__(*args, **kwargs) 433 | pair, expire_time = cache.get( 434 | key, 435 | default=ENOVAL, 436 | expire_time=True, 437 | retry=True, 438 | ) 439 | 440 | if pair is not ENOVAL: 441 | result, delta = pair 442 | now = time.time() 443 | ttl = expire_time - now 444 | 445 | if (-delta * beta * math.log(random.random())) < ttl: 446 | return result # Cache hit. 447 | 448 | # Check whether a thread has started for early recomputation. 449 | 450 | thread_key = key + (ENOVAL,) 451 | thread_added = cache.add( 452 | thread_key, 453 | None, 454 | expire=delta, 455 | retry=True, 456 | ) 457 | 458 | if thread_added: 459 | # Start thread for early recomputation. 460 | def recompute(): 461 | with cache: 462 | pair = timer(*args, **kwargs) 463 | cache.set( 464 | key, 465 | pair, 466 | expire=expire, 467 | tag=tag, 468 | retry=True, 469 | ) 470 | 471 | thread = threading.Thread(target=recompute) 472 | thread.daemon = True 473 | thread.start() 474 | 475 | return result 476 | 477 | pair = timer(*args, **kwargs) 478 | cache.set(key, pair, expire=expire, tag=tag, retry=True) 479 | return pair[0] 480 | 481 | def __cache_key__(*args, **kwargs): 482 | """Make key for cache given function arguments.""" 483 | return args_to_key(base, args, kwargs, typed, ignore) 484 | 485 | wrapper.__cache_key__ = __cache_key__ 486 | return wrapper 487 | 488 | return decorator 489 | -------------------------------------------------------------------------------- /docs/Makefile: -------------------------------------------------------------------------------- 1 | # Minimal makefile for Sphinx documentation 2 | # 3 | 4 | # You can set these variables from the command line, and also 5 | # from the environment for the first two. 6 | SPHINXOPTS ?= 7 | SPHINXBUILD ?= sphinx-build 8 | SOURCEDIR = . 9 | BUILDDIR = _build 10 | 11 | # Put it first so that "make" without argument is like "make help". 12 | help: 13 | @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) 14 | 15 | .PHONY: help Makefile 16 | 17 | # Catch-all target: route all unknown targets to Sphinx using the new 18 | # "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). 19 | %: Makefile 20 | @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) 21 | -------------------------------------------------------------------------------- /docs/_static/core-p1-delete.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/grantjenks/python-diskcache/ebfa37cd99d7ef716ec452ad8af4b4276a8e2233/docs/_static/core-p1-delete.png -------------------------------------------------------------------------------- /docs/_static/core-p1-get.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/grantjenks/python-diskcache/ebfa37cd99d7ef716ec452ad8af4b4276a8e2233/docs/_static/core-p1-get.png -------------------------------------------------------------------------------- /docs/_static/core-p1-set.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/grantjenks/python-diskcache/ebfa37cd99d7ef716ec452ad8af4b4276a8e2233/docs/_static/core-p1-set.png -------------------------------------------------------------------------------- /docs/_static/core-p8-delete.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/grantjenks/python-diskcache/ebfa37cd99d7ef716ec452ad8af4b4276a8e2233/docs/_static/core-p8-delete.png -------------------------------------------------------------------------------- /docs/_static/core-p8-get.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/grantjenks/python-diskcache/ebfa37cd99d7ef716ec452ad8af4b4276a8e2233/docs/_static/core-p8-get.png -------------------------------------------------------------------------------- /docs/_static/core-p8-set.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/grantjenks/python-diskcache/ebfa37cd99d7ef716ec452ad8af4b4276a8e2233/docs/_static/core-p8-set.png -------------------------------------------------------------------------------- /docs/_static/custom.css: -------------------------------------------------------------------------------- 1 | table { 2 | font-size: 80%; 3 | width: 100%; 4 | } 5 | 6 | #comparison table { 7 | display: block; 8 | overflow: scroll; 9 | } 10 | 11 | th.head { 12 | text-align: center; 13 | } 14 | 15 | div.body { 16 | min-width: 240px; 17 | } 18 | -------------------------------------------------------------------------------- /docs/_static/djangocache-delete.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/grantjenks/python-diskcache/ebfa37cd99d7ef716ec452ad8af4b4276a8e2233/docs/_static/djangocache-delete.png -------------------------------------------------------------------------------- /docs/_static/djangocache-get.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/grantjenks/python-diskcache/ebfa37cd99d7ef716ec452ad8af4b4276a8e2233/docs/_static/djangocache-get.png -------------------------------------------------------------------------------- /docs/_static/djangocache-set.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/grantjenks/python-diskcache/ebfa37cd99d7ef716ec452ad8af4b4276a8e2233/docs/_static/djangocache-set.png -------------------------------------------------------------------------------- /docs/_static/early-recomputation-03.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/grantjenks/python-diskcache/ebfa37cd99d7ef716ec452ad8af4b4276a8e2233/docs/_static/early-recomputation-03.png -------------------------------------------------------------------------------- /docs/_static/early-recomputation-05.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/grantjenks/python-diskcache/ebfa37cd99d7ef716ec452ad8af4b4276a8e2233/docs/_static/early-recomputation-05.png -------------------------------------------------------------------------------- /docs/_static/early-recomputation.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/grantjenks/python-diskcache/ebfa37cd99d7ef716ec452ad8af4b4276a8e2233/docs/_static/early-recomputation.png -------------------------------------------------------------------------------- /docs/_static/gj-logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/grantjenks/python-diskcache/ebfa37cd99d7ef716ec452ad8af4b4276a8e2233/docs/_static/gj-logo.png -------------------------------------------------------------------------------- /docs/_static/no-caching.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/grantjenks/python-diskcache/ebfa37cd99d7ef716ec452ad8af4b4276a8e2233/docs/_static/no-caching.png -------------------------------------------------------------------------------- /docs/_static/synchronized-locking.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/grantjenks/python-diskcache/ebfa37cd99d7ef716ec452ad8af4b4276a8e2233/docs/_static/synchronized-locking.png -------------------------------------------------------------------------------- /docs/_static/traditional-caching.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/grantjenks/python-diskcache/ebfa37cd99d7ef716ec452ad8af4b4276a8e2233/docs/_static/traditional-caching.png -------------------------------------------------------------------------------- /docs/_templates/gumroad.html: -------------------------------------------------------------------------------- 1 |

Give Support

2 |

If you or your organization uses DiskCache, consider financial support:

3 |

4 | Give to Python DiskCache 5 |

6 | -------------------------------------------------------------------------------- /docs/api.rst: -------------------------------------------------------------------------------- 1 | .. automodule:: diskcache 2 | 3 | .. contents:: 4 | :local: 5 | 6 | Cache 7 | ----- 8 | 9 | Read the :ref:`Cache tutorial ` for example usage. 10 | 11 | .. autoclass:: diskcache.Cache 12 | :members: 13 | :special-members: 14 | :exclude-members: __weakref__ 15 | 16 | FanoutCache 17 | ----------- 18 | 19 | Read the :ref:`FanoutCache tutorial ` for example usage. 20 | 21 | .. autoclass:: diskcache.FanoutCache 22 | :members: 23 | :special-members: 24 | :exclude-members: __weakref__ 25 | 26 | DjangoCache 27 | ----------- 28 | 29 | Read the :ref:`DjangoCache tutorial ` for example usage. 30 | 31 | .. autoclass:: diskcache.DjangoCache 32 | :members: 33 | :special-members: 34 | 35 | Deque 36 | ----- 37 | 38 | .. autoclass:: diskcache.Deque 39 | :members: 40 | :special-members: 41 | :exclude-members: __weakref__ 42 | 43 | Index 44 | ----- 45 | 46 | .. autoclass:: diskcache.Index 47 | :members: 48 | :special-members: 49 | :exclude-members: __weakref__ 50 | 51 | Recipes 52 | ------- 53 | 54 | .. autoclass:: diskcache.Averager 55 | :members: 56 | 57 | .. autoclass:: diskcache.Lock 58 | :members: 59 | 60 | .. autoclass:: diskcache.RLock 61 | :members: 62 | 63 | .. autoclass:: diskcache.BoundedSemaphore 64 | :members: 65 | 66 | .. autodecorator:: diskcache.throttle 67 | 68 | .. autodecorator:: diskcache.barrier 69 | 70 | .. autodecorator:: diskcache.memoize_stampede 71 | 72 | .. _constants: 73 | 74 | Constants 75 | --------- 76 | 77 | Read the :ref:`Settings tutorial ` for details. 78 | 79 | .. data:: diskcache.DEFAULT_SETTINGS 80 | 81 | * `statistics` (int) default 0 - disabled when 0, enabled when 1. 82 | * `tag_index` (int) default 0 - disabled when 0, enabled when 1. 83 | * `eviction_policy` (str) default "least-recently-stored" - any of the keys 84 | in `EVICTION_POLICY` as described below. 85 | * `size_limit` (int, in bytes) default one gigabyte - approximate size limit 86 | of cache. 87 | * `cull_limit` (int) default ten - maximum number of items culled during 88 | `set` or `add` operations. 89 | * `sqlite_auto_vacuum` (int) default 1, "FULL" - SQLite auto vacuum pragma. 90 | * `sqlite_cache_size` (int, in pages) default 8,192 - SQLite cache size 91 | pragma. 92 | * `sqlite_journal_mode` (str) default "wal" - SQLite journal mode pragma. 93 | * `sqlite_mmap_size` (int, in bytes) default 64 megabytes - SQLite mmap size 94 | pragma. 95 | * `sqlite_synchronous` (int) default 1, "NORMAL" - SQLite synchronous 96 | pragma. 97 | * `disk_min_file_size` (int, in bytes) default 32 kilobytes - values with 98 | greater size are stored in files. 99 | * `disk_pickle_protocol` (int) default highest Pickle protocol - the Pickle 100 | protocol to use for data types that are not natively supported. 101 | 102 | .. data:: diskcache.EVICTION_POLICY 103 | 104 | * `least-recently-stored` (default) - evict least recently stored keys first. 105 | * `least-recently-used` - evict least recently used keys first. 106 | * `least-frequently-used` - evict least frequently used keys first. 107 | * `none` - never evict keys. 108 | 109 | Disk 110 | ---- 111 | 112 | Read the :ref:`Disk tutorial ` for details. 113 | 114 | .. autoclass:: diskcache.Disk 115 | :members: 116 | :special-members: 117 | :exclude-members: __weakref__ 118 | 119 | JSONDisk 120 | -------- 121 | 122 | Read the :ref:`Disk tutorial ` for details. 123 | 124 | .. autoclass:: diskcache.JSONDisk 125 | :members: 126 | :special-members: 127 | :exclude-members: __weakref__ 128 | 129 | Timeout 130 | ------- 131 | 132 | .. autoexception:: diskcache.Timeout 133 | -------------------------------------------------------------------------------- /docs/cache-benchmarks.rst: -------------------------------------------------------------------------------- 1 | DiskCache Cache Benchmarks 2 | ========================== 3 | 4 | Accurately measuring performance is a difficult task. The benchmarks on this 5 | page are synthetic in the sense that they were designed to stress getting, 6 | setting, and deleting items repeatedly. Measurements in production systems are 7 | much harder to reproduce reliably. So take the following data with a `grain of 8 | salt`_. A stated feature of :doc:`DiskCache ` is performance so we would 9 | be remiss not to produce this page with comparisons. 10 | 11 | The source for all benchmarks can be found under the "tests" directory in the 12 | source code repository. Measurements are reported by percentile: median, 90th 13 | percentile, 99th percentile, and maximum along with total time and miss 14 | rate. The average is not reported as its less useful in response-time 15 | scenarios. Each process in the benchmark executes 100,000 operations with ten 16 | times as many sets as deletes and ten times as many gets as sets. 17 | 18 | Each comparison includes `Memcached`_ and `Redis`_ with default client and 19 | server settings. Note that these backends work differently as they communicate 20 | over the localhost network. The also require a server process running and 21 | maintained. All keys and values are short byte strings to reduce the network 22 | impact. 23 | 24 | .. _`grain of salt`: https://en.wikipedia.org/wiki/Grain_of_salt 25 | .. _`Memcached`: http://memcached.org/ 26 | .. _`Redis`: http://redis.io/ 27 | 28 | Single Access 29 | ------------- 30 | 31 | The single access workload starts one worker processes which performs all 32 | operations. No concurrent cache access occurs. 33 | 34 | Get 35 | ... 36 | 37 | .. image:: _static/core-p1-get.png 38 | 39 | Above displays cache access latency at three percentiles. Notice the 40 | performance of :doc:`DiskCache ` is faster than highly optimized 41 | memory-backed server solutions. 42 | 43 | Set 44 | ... 45 | 46 | .. image:: _static/core-p1-set.png 47 | 48 | Above displays cache store latency at three percentiles. The cost of writing to 49 | disk is higher but still sub-millisecond. All data in :doc:`DiskCache ` 50 | is persistent. 51 | 52 | Delete 53 | ...... 54 | 55 | .. image:: _static/core-p1-delete.png 56 | 57 | Above displays cache delete latency at three percentiles. As above, deletes 58 | require disk writes but latency is still sub-millisecond. 59 | 60 | Timing Data 61 | ........... 62 | 63 | Not all data is easily displayed in the graphs above. Miss rate, maximum 64 | latency and total latency is recorded below. 65 | 66 | ========= ========= ========= ========= ========= ========= ========= ========= 67 | Timings for diskcache.Cache 68 | ------------------------------------------------------------------------------- 69 | Action Count Miss Median P90 P99 Max Total 70 | ========= ========= ========= ========= ========= ========= ========= ========= 71 | get 88966 9705 12.159us 17.166us 28.849us 174.999us 1.206s 72 | set 9021 0 68.903us 93.937us 188.112us 10.297ms 875.907ms 73 | delete 1012 104 47.207us 66.042us 128.031us 7.160ms 89.599ms 74 | Total 98999 2.171s 75 | ========= ========= ========= ========= ========= ========= ========= ========= 76 | 77 | The generated workload includes a ~1% cache miss rate. All items were stored 78 | with no expiry. The miss rate is due entirely to gets after deletes. 79 | 80 | ========= ========= ========= ========= ========= ========= ========= ========= 81 | Timings for diskcache.FanoutCache(shards=4, timeout=1.0) 82 | ------------------------------------------------------------------------------- 83 | Action Count Miss Median P90 P99 Max Total 84 | ========= ========= ========= ========= ========= ========= ========= ========= 85 | get 88966 9705 15.020us 20.027us 33.855us 437.021us 1.425s 86 | set 9021 0 71.049us 100.136us 203.133us 9.186ms 892.262ms 87 | delete 1012 104 48.161us 69.141us 129.952us 5.216ms 87.294ms 88 | Total 98999 2.405s 89 | ========= ========= ========= ========= ========= ========= ========= ========= 90 | 91 | The high maximum store latency is likely an artifact of disk/OS interactions. 92 | 93 | ========= ========= ========= ========= ========= ========= ========= ========= 94 | Timings for diskcache.FanoutCache(shards=8, timeout=0.010) 95 | ------------------------------------------------------------------------------- 96 | Action Count Miss Median P90 P99 Max Total 97 | ========= ========= ========= ========= ========= ========= ========= ========= 98 | get 88966 9705 15.020us 20.027us 34.094us 627.995us 1.420s 99 | set 9021 0 72.956us 100.851us 203.133us 9.623ms 927.824ms 100 | delete 1012 104 50.783us 72.002us 132.084us 8.396ms 78.898ms 101 | Total 98999 2.426s 102 | ========= ========= ========= ========= ========= ========= ========= ========= 103 | 104 | Notice the low overhead of the :class:`FanoutCache 105 | `. Increasing the number of shards from four to eight 106 | has a negligible impact on performance. 107 | 108 | ========= ========= ========= ========= ========= ========= ========= ========= 109 | Timings for pylibmc.Client 110 | ------------------------------------------------------------------------------- 111 | Action Count Miss Median P90 P99 Max Total 112 | ========= ========= ========= ========= ========= ========= ========= ========= 113 | get 88966 9705 25.988us 29.802us 41.008us 139.952us 2.388s 114 | set 9021 0 27.895us 30.994us 40.054us 97.990us 254.248ms 115 | delete 1012 104 25.988us 29.087us 38.147us 89.169us 27.159ms 116 | Total 98999 2.669s 117 | ========= ========= ========= ========= ========= ========= ========= ========= 118 | 119 | Memcached performance is low latency and stable. 120 | 121 | ========= ========= ========= ========= ========= ========= ========= ========= 122 | Timings for redis.StrictRedis 123 | ------------------------------------------------------------------------------- 124 | Action Count Miss Median P90 P99 Max Total 125 | ========= ========= ========= ========= ========= ========= ========= ========= 126 | get 88966 9705 44.107us 54.121us 73.910us 204.086us 4.125s 127 | set 9021 0 45.061us 56.028us 75.102us 237.942us 427.197ms 128 | delete 1012 104 44.107us 54.836us 72.002us 126.839us 46.771ms 129 | Total 98999 4.599s 130 | ========= ========= ========= ========= ========= ========= ========= ========= 131 | 132 | Redis performance is roughly half that of Memcached. :doc:`DiskCache ` 133 | performs better than Redis for get operations through the Max percentile. 134 | 135 | Concurrent Access 136 | ----------------- 137 | 138 | The concurrent access workload starts eight worker processes each with 139 | different and interleaved operations. None of these benchmarks saturated all 140 | the processors. 141 | 142 | Get 143 | ... 144 | 145 | .. image:: _static/core-p8-get.png 146 | 147 | Under heavy load, :doc:`DiskCache ` gets are low latency. At the 90th 148 | percentile, they are less than half the latency of Memcached. 149 | 150 | Set 151 | ... 152 | 153 | .. image:: _static/core-p8-set.png 154 | 155 | Stores are much slower under load and benefit greatly from sharding. Not 156 | displayed are latencies in excess of five milliseconds. With one shard 157 | allocated per worker, latency is within a magnitude of memory-backed server 158 | solutions. 159 | 160 | Delete 161 | ...... 162 | 163 | .. image:: _static/core-p8-delete.png 164 | 165 | Again deletes require writes to disk. Only the :class:`FanoutCache 166 | ` performs well with one shard allocated per worker. 167 | 168 | Timing Data 169 | ........... 170 | 171 | Not all data is easily displayed in the graphs above. Miss rate, maximum 172 | latency and total latency is recorded below. 173 | 174 | ========= ========= ========= ========= ========= ========= ========= ========= 175 | Timings for diskcache.Cache 176 | ------------------------------------------------------------------------------- 177 | Action Count Miss Median P90 P99 Max Total 178 | ========= ========= ========= ========= ========= ========= ========= ========= 179 | get 712546 71214 15.974us 23.127us 40.054us 4.953ms 12.349s 180 | set 71530 0 94.891us 1.328ms 21.307ms 1.846s 131.728s 181 | delete 7916 807 65.088us 1.278ms 19.610ms 1.244s 13.811s 182 | Total 791992 157.888s 183 | ========= ========= ========= ========= ========= ========= ========= ========= 184 | 185 | Notice the unacceptably high maximum store and delete latency. Without 186 | sharding, cache writers block each other. By default :class:`Cache 187 | ` objects raise a timeout error after sixty seconds. 188 | 189 | ========= ========= ========= ========= ========= ========= ========= ========= 190 | Timings for diskcache.FanoutCache(shards=4, timeout=1.0) 191 | ------------------------------------------------------------------------------- 192 | Action Count Miss Median P90 P99 Max Total 193 | ========= ========= ========= ========= ========= ========= ========= ========= 194 | get 712546 71623 19.073us 35.048us 59.843us 12.980ms 16.849s 195 | set 71530 0 108.004us 1.313ms 9.176ms 333.361ms 50.821s 196 | delete 7916 767 73.195us 1.264ms 9.033ms 108.232ms 4.964s 197 | Total 791992 72.634s 198 | ========= ========= ========= ========= ========= ========= ========= ========= 199 | 200 | Here :class:`FanoutCache ` uses four shards to 201 | distribute writes. That reduces the maximum latency by a factor of ten. Note 202 | the miss rate is variable due to the interleaved operations of concurrent 203 | workers. 204 | 205 | ========= ========= ========= ========= ========= ========= ========= ========= 206 | Timings for diskcache.FanoutCache(shards=8, timeout=0.010) 207 | ------------------------------------------------------------------------------- 208 | Action Count Miss Median P90 P99 Max Total 209 | ========= ========= ========= ========= ========= ========= ========= ========= 210 | get 712546 71106 25.034us 47.922us 101.089us 9.015ms 22.336s 211 | set 71530 39 134.945us 1.324ms 5.763ms 16.027ms 33.347s 212 | delete 7916 775 88.930us 1.267ms 5.017ms 13.732ms 3.308s 213 | Total 791992 58.991s 214 | ========= ========= ========= ========= ========= ========= ========= ========= 215 | 216 | With one shard allocated per worker and a low timeout, the maximum latency is 217 | more reasonable and corresponds to the specified 10 millisecond timeout. Some 218 | set and delete operations were therefore canceled and recorded as cache 219 | misses. The miss rate due to timeout is about 0.01% so our success rate is 220 | four-nines or 99.99%. 221 | 222 | ========= ========= ========= ========= ========= ========= ========= ========= 223 | Timings for pylibmc.Client 224 | ------------------------------------------------------------------------------- 225 | Action Count Miss Median P90 P99 Max Total 226 | ========= ========= ========= ========= ========= ========= ========= ========= 227 | get 712546 72043 83.923us 107.050us 123.978us 617.027us 61.824s 228 | set 71530 0 84.877us 108.004us 124.931us 312.090us 6.283s 229 | delete 7916 796 82.970us 105.858us 123.024us 288.963us 680.970ms 230 | Total 791992 68.788s 231 | ========= ========= ========= ========= ========= ========= ========= ========= 232 | 233 | Memcached performance is low latency and stable even under heavy load. Notice 234 | that cache gets are three times slower in total as compared with 235 | :class:`FanoutCache `. The superior performance of get 236 | operations put the overall performance of :doc:`DiskCache ` ahead of 237 | Memcached. 238 | 239 | ========= ========= ========= ========= ========= ========= ========= ========= 240 | Timings for redis.StrictRedis 241 | ------------------------------------------------------------------------------- 242 | Action Count Miss Median P90 P99 Max Total 243 | ========= ========= ========= ========= ========= ========= ========= ========= 244 | get 712546 72093 138.044us 169.039us 212.908us 151.121ms 101.197s 245 | set 71530 0 138.998us 169.992us 216.007us 1.200ms 10.173s 246 | delete 7916 752 136.137us 167.847us 211.954us 1.059ms 1.106s 247 | Total 791992 112.476s 248 | ========= ========= ========= ========= ========= ========= ========= ========= 249 | 250 | Redis performance is roughly half that of Memcached. Beware the impact of 251 | persistence settings on your Redis performance. Depending on your use of 252 | logging and snapshotting, maximum latency may increase significantly. 253 | -------------------------------------------------------------------------------- /docs/case-study-landing-page-caching.rst: -------------------------------------------------------------------------------- 1 | Case Study: Landing Page Caching 2 | ================================ 3 | 4 | :doc:`DiskCache ` version 4 added recipes for cache stampede mitigation. 5 | Cache stampedes are a type of system overload that can occur when parallel 6 | computing systems using memoization come under heavy load. This behaviour is 7 | sometimes also called dog-piling, cache miss storm, cache choking, or the 8 | thundering herd problem. Let's look at how that applies to landing page 9 | caching. 10 | 11 | .. code-block:: python 12 | 13 | import time 14 | 15 | def generate_landing_page(): 16 | time.sleep(0.2) # Work really hard. 17 | # Return HTML response. 18 | 19 | Imagine a website under heavy load with a function used to generate the landing 20 | page. There are five processes each with two threads for a total of ten 21 | concurrent workers. The landing page is loaded constantly and takes about two 22 | hundred milliseconds to generate. 23 | 24 | .. image:: _static/no-caching.png 25 | 26 | When we look at the number of concurrent workers and the latency with no 27 | caching at all, the graph looks as above. Notice each worker constantly 28 | regenerates the page with a consistently slow latency. 29 | 30 | .. code-block:: python 31 | :emphasize-lines: 5 32 | 33 | import diskcache as dc 34 | 35 | cache = dc.Cache() 36 | 37 | @cache.memoize(expire=1) 38 | def generate_landing_page(): 39 | time.sleep(0.2) 40 | 41 | Assume the result of generating the landing page can be memoized for one 42 | second. Memoization supports a traditional caching strategy. After each second, 43 | the cached HTML expires and all ten workers rush to regenerate the result. 44 | 45 | .. image:: _static/traditional-caching.png 46 | 47 | There is a huge improvement in average latency now but some requests experience 48 | worse latency than before due to the added overhead of caching. The cache 49 | stampede is visible too as the spikes in the concurrency graph. If generating 50 | the landing page requires significant resources then the spikes may be 51 | prohibitive. 52 | 53 | To reduce the number of concurrent workers, a barrier can be used to 54 | synchronize generating the landing page. 55 | 56 | .. code-block:: python 57 | :emphasize-lines: 1,2,3 58 | 59 | @cache.memoize(expire=0) 60 | @dc.barrier(cache, dc.Lock) 61 | @cache.memoize(expire=1) 62 | def generate_landing_page(): 63 | time.sleep(0.2) 64 | 65 | The double-checked locking uses two memoization decorators to optimistically 66 | look up the cached result before locking. With `expire` set to zero, the 67 | cache's get-operation is performed but the set-operation is skipped. Only the 68 | inner-nested memoize decorator will update the cache. 69 | 70 | .. image:: _static/synchronized-locking.png 71 | 72 | The number of concurrent workers is now greatly improved. Rather than having 73 | ten workers all attempt to generate the same result, a single worker generates 74 | the result and the other ten benefit. The maximum latency has increased however 75 | as three layers of caching and locking wrap the function. 76 | 77 | Ideally, the system would anticipate the pending expiration of the cached item 78 | and would recompute the result in a separate thread of execution. Coordinating 79 | recomputation would be a function of the number of workers, the expiration 80 | time, and the duration of computation. Fortunately, Vattani, et al. published 81 | the solution in "Optimal Probabilistic Cache Stampede Prevention" in 2015. 82 | 83 | .. code-block:: python 84 | :emphasize-lines: 1 85 | 86 | @dc.memoize_stampede(cache, expire=1) 87 | def generate_landing_page(): 88 | time.sleep(0.2) 89 | 90 | Early probabilistic recomputation uses a random number generator to simulate a 91 | cache miss prior to expiration. The new result is then computed in a separate 92 | thread while the cached result is returned to the caller. When the cache item 93 | is missing, the result is computed and cached synchronously. 94 | 95 | .. image:: _static/early-recomputation.png 96 | 97 | The latency is now its theoretical best. An initial warmup execution takes two 98 | hundred milliseconds and the remaining calls all return immediately from the 99 | cache. Behind the scenes, separate threads of execution are recomputing the 100 | result of workers and updating the cache. The concurrency graph shows a nearly 101 | constant stream of workers recomputing the function's result. 102 | 103 | .. code-block:: python 104 | :emphasize-lines: 1 105 | 106 | @dc.memoize_stampede(cache, expire=1, beta=0.5) 107 | def generate_landing_page(): 108 | time.sleep(0.2) 109 | 110 | Vattani described an additional parameter, :math:`\beta`, which could be used 111 | to tune the eagerness of recomputation. As the number and frequency of 112 | concurrent worker calls increases, eagerness can be lessened by decreasing the 113 | :math:`\beta` parameter. The default value of :math:`\beta` is one, and above 114 | it is set to half. 115 | 116 | .. image:: _static/early-recomputation-05.png 117 | 118 | Latency is now still its theoretical best while the worker load has decreased 119 | significantly. The likelihood of simulated cache misses is now half what it was 120 | before. The value was determined through experimentation. 121 | 122 | .. code-block:: python 123 | :emphasize-lines: 1 124 | 125 | @dc.memoize_stampede(cache, expire=1, beta=0.3) 126 | def generate_landing_page(): 127 | time.sleep(0.2) 128 | 129 | Lets see what happens when :math:`\beta` is set too low. 130 | 131 | .. image:: _static/early-recomputation-03.png 132 | 133 | When set too low, the cache item expires before a new value is recomputed. The 134 | real cache miss then causes the workers to synchronously recompute the landing 135 | page and cache the result. With no barrier in place, eleven workers cause a 136 | cache stampede. The eleven workers are composed of ten synchronous workers and 137 | one in a background thread. The best way to customize :math:`\beta` is through 138 | experimentation, otherwise the default is reasonable. 139 | 140 | :doc:`DiskCache ` provides data types and recipes for memoization and 141 | mitigation of cache stampedes. The decorators provided are composable for a 142 | variety of scenarios. The best way to get started is with the :doc:`tutorial`. 143 | -------------------------------------------------------------------------------- /docs/case-study-web-crawler.rst: -------------------------------------------------------------------------------- 1 | Case Study: Web Crawler 2 | ======================= 3 | 4 | :doc:`DiskCache ` version 2.7 added a couple persistent data 5 | structures. Let's see how they're useful with a case study in crawling the 6 | web. Easy enough, right? We'll start with code to retrieve urls: 7 | 8 | >>> from time import sleep 9 | >>> def get(url): 10 | ... "Get data for url." 11 | ... sleep(url / 1000.0) 12 | ... return str(url) 13 | 14 | No, we're not actually crawling the web. Our urls are numbers and we'll simply 15 | go to sleep to simulate downloading a web page. 16 | 17 | >>> get(20) 18 | '20' 19 | 20 | Once we download some data, we'll need to parse it and extract the links. 21 | 22 | >>> from random import randrange, seed 23 | >>> def parse(data): 24 | ... "Parse data and return list of links." 25 | ... seed(int(data)) 26 | ... count = randrange(1, 10) 27 | ... return [randrange(100) for _ in range(count)] 28 | 29 | Again, we're not really parsing data. We're just returning a list of one to ten 30 | integers between zero and one hundred. In our imaginary web, urls are just 31 | integers. 32 | 33 | >>> parse('20') 34 | [68, 76, 90, 25, 63, 90, 87, 57, 16] 35 | 36 | Alright, this is a pretty basic pattern. The ``get`` function returns data and 37 | the ``parse`` function returns a list of more data to go get. We can use the 38 | deque data type from the standard library's collection module to crawl our web. 39 | 40 | >>> from collections import deque 41 | >>> def crawl(): 42 | ... urls = deque([0]) 43 | ... results = dict() 44 | ... 45 | ... while True: 46 | ... try: 47 | ... url = urls.popleft() 48 | ... except IndexError: 49 | ... break 50 | ... 51 | ... if url in results: 52 | ... continue 53 | ... 54 | ... data = get(url) 55 | ... 56 | ... for link in parse(data): 57 | ... urls.append(link) 58 | ... 59 | ... results[url] = data 60 | ... 61 | ... print('Results: %s' % len(results)) 62 | 63 | We're doing a breadth-first search crawl of the web. Our initial seed is zero 64 | and we use that to initialize our queue. All the results are stored in a 65 | dictionary mapping url to data. We then iterate by repeatedly popping the first 66 | url from our queue. If we've already visited the url then we continue, 67 | otherwise we get the corresponding data and parse it. The parsed results are 68 | appended to our queue. Finally we store the data in our results 69 | dictionary. 70 | 71 | >>> crawl() 72 | Results: 99 73 | 74 | The results of our current code are ephemeral. All results are lost once the 75 | program terminates. To make the results persistent, we can use :doc:`DiskCache 76 | ` data structures and store the results in the local file 77 | system. :doc:`DiskCache ` provides both :class:`Deque ` 78 | and :class:`Index ` data structures which can replace our urls 79 | and results variables. 80 | 81 | >>> from diskcache import Deque, Index 82 | >>> def crawl(): 83 | ... urls = Deque([0], 'data/urls') 84 | ... results = Index('data/results') 85 | ... 86 | ... while True: 87 | ... try: 88 | ... url = urls.popleft() 89 | ... except IndexError: 90 | ... break 91 | ... 92 | ... if url in results: 93 | ... continue 94 | ... 95 | ... data = get(url) 96 | ... 97 | ... for link in parse(data): 98 | ... urls.append(link) 99 | ... 100 | ... results[url] = data 101 | ... 102 | ... print('Results: %s' % len(results)) 103 | 104 | Look familiar? Only three lines changed. The import at the top changed so now 105 | we're using ``diskcache`` rather than the ``collections`` module. Then, when we 106 | initialize the urls and results objects, we pass relative paths to directories 107 | where we want the data stored. Again, let's try it out: 108 | 109 | >>> crawl() 110 | Results: 99 111 | 112 | Our results are now persistent. We can initialize our results index outside of 113 | the crawl function and query it. 114 | 115 | >>> results = Index('data/results') 116 | >>> len(results) 117 | 99 118 | 119 | As an added benefit, our code also now works in parallel. 120 | 121 | >>> results.clear() 122 | >>> from multiprocessing import Process 123 | >>> processes = [Process(target=crawl) for _ in range(4)] 124 | >>> for process in processes: 125 | ... process.start() 126 | >>> for process in processes: 127 | ... process.join() 128 | >>> len(results) 129 | 99 130 | 131 | Each of the processes uses the same deque and index to crawl our web. Work is 132 | automatically divided among the processes as they pop urls from the queue. If 133 | this were run as a script then multiple Python processes could be started and 134 | stopped as desired. 135 | 136 | Interesting, no? Three simple changes and our code goes from ephemeral and 137 | single-process to persistent and multi-process. Nothing truly new has happened 138 | here but the API is convenient and that makes a huge difference. We're also no 139 | longer constrained by memory. :doc:`DiskCache ` makes efficient use of 140 | your disk and you can customize how much memory is used. By default the maximum 141 | memory consumption of deque and index objects is only a few dozen 142 | megabytes. Now our simple script can efficiently process terabytes of data. 143 | 144 | Go forth and build and share! 145 | -------------------------------------------------------------------------------- /docs/conf.py: -------------------------------------------------------------------------------- 1 | # Configuration file for the Sphinx documentation builder. 2 | # 3 | # This file only contains a selection of the most common options. For a full 4 | # list see the documentation: 5 | # https://www.sphinx-doc.org/en/master/usage/configuration.html 6 | 7 | # -- Path setup -------------------------------------------------------------- 8 | 9 | # If extensions (or modules to document with autodoc) are in another directory, 10 | # add these directories to sys.path here. If the directory is relative to the 11 | # documentation root, use os.path.abspath to make it absolute, like shown here. 12 | 13 | import os 14 | import sys 15 | sys.path.insert(0, os.path.abspath('..')) 16 | 17 | import diskcache 18 | 19 | 20 | # -- Project information ----------------------------------------------------- 21 | 22 | project = 'DiskCache' 23 | copyright = '2023, Grant Jenks' 24 | author = 'Grant Jenks' 25 | 26 | # The full version, including alpha/beta/rc tags 27 | release = diskcache.__version__ 28 | 29 | 30 | # -- General configuration --------------------------------------------------- 31 | 32 | # Add any Sphinx extension module names here, as strings. They can be 33 | # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom 34 | # ones. 35 | extensions = [ 36 | 'sphinx.ext.autodoc', 37 | 'sphinx.ext.todo', 38 | ] 39 | 40 | # Add any paths that contain templates here, relative to this directory. 41 | templates_path = ['_templates'] 42 | 43 | # List of patterns, relative to source directory, that match files and 44 | # directories to ignore when looking for source files. 45 | # This pattern also affects html_static_path and html_extra_path. 46 | exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store'] 47 | 48 | 49 | # -- Options for HTML output ------------------------------------------------- 50 | 51 | # The theme to use for HTML and HTML Help pages. See the documentation for 52 | # a list of builtin themes. 53 | html_theme = 'alabaster' 54 | 55 | # Theme options are theme-specific and customize the look and feel of a theme 56 | # further. For a list of options available for each theme, see the 57 | # documentation. 58 | html_theme_options = { 59 | 'logo': 'gj-logo.png', 60 | 'logo_name': True, 61 | 'logo_text_align': 'center', 62 | 'analytics_id': 'UA-19364636-2', 63 | 'show_powered_by': False, 64 | 'show_related': True, 65 | 'github_user': 'grantjenks', 66 | 'github_repo': 'python-diskcache', 67 | 'github_type': 'star', 68 | } 69 | 70 | # Add any paths that contain custom static files (such as style sheets) here, 71 | # relative to this directory. They are copied after the builtin static files, 72 | # so a file named "default.css" will overwrite the builtin "default.css". 73 | html_static_path = ['_static'] 74 | 75 | # Custom sidebar templates, maps document names to template names. 76 | html_sidebars = { 77 | '**': [ 78 | 'about.html', 79 | 'gumroad.html', 80 | 'localtoc.html', 81 | 'relations.html', 82 | 'searchbox.html', 83 | ] 84 | } 85 | 86 | def setup(app): 87 | app.add_css_file('custom.css') 88 | -------------------------------------------------------------------------------- /docs/development.rst: -------------------------------------------------------------------------------- 1 | DiskCache Development 2 | ===================== 3 | 4 | :doc:`DiskCache ` development is lead by Grant Jenks 5 | . 6 | 7 | Collaborators Welcome 8 | --------------------- 9 | 10 | #. Search issues or open a new issue to start a discussion around a bug. 11 | #. Fork the `GitHub repository`_ and make your changes in a new branch. 12 | #. Write a test which shows the bug was fixed. 13 | #. Send a pull request and message the development lead until its merged and 14 | published. 15 | 16 | .. _`GitHub repository`: https://github.com/grantjenks/python-diskcache 17 | 18 | Requests for Contributions 19 | -------------------------- 20 | 21 | #. Command-line interface. Operations to support: get, set, store, delete, 22 | expire, evict, clear, path, check, stats. 23 | #. Django admin interface for cache stats and interaction. 24 | #. Cache stampede barrier (source prototype in repo). 25 | #. API Compatibility 26 | 27 | #. `Shelf interface `_ 28 | #. `DBM interface `_ 29 | 30 | #. Backend Compatibility 31 | 32 | #. `Flask-Caching `_ 33 | #. `Beaker `_ 34 | #. `dogpile.cache `_ 35 | 36 | Get the Code 37 | ------------ 38 | 39 | :doc:`DiskCache ` is actively developed in a `GitHub repository`_. 40 | 41 | You can either clone the public repository:: 42 | 43 | $ git clone https://github.com/grantjenks/python-diskcache.git 44 | 45 | Download the `tarball `_:: 46 | 47 | $ curl -OL https://github.com/grantjenks/python-diskcache/tarball/master 48 | 49 | Or, download the `zipball `_:: 50 | 51 | $ curl -OL https://github.com/grantjenks/python-diskcache/zipball/master 52 | 53 | Installing Dependencies 54 | ----------------------- 55 | 56 | Install development dependencies with `pip `_:: 57 | 58 | $ pip install -r requirements.txt 59 | 60 | All packages for running tests will be installed. 61 | 62 | Additional packages like ``pylibmc`` and ``redis`` along with their server 63 | counterparts are necessary for some benchmarks. 64 | 65 | Testing 66 | ------- 67 | 68 | :doc:`DiskCache ` currently tests against five versions of Python: 69 | 70 | * CPython 3.5 71 | * CPython 3.6 72 | * CPython 3.7 73 | * CPython 3.8 74 | 75 | Testing uses `tox `_. If you don't want to 76 | install all the development requirements, then, after downloading, you can 77 | simply run:: 78 | 79 | $ python setup.py test 80 | 81 | The test argument to setup.py will download a minimal testing infrastructure 82 | and run the tests. 83 | 84 | :: 85 | 86 | $ tox 87 | GLOB sdist-make: python-diskcache/setup.py 88 | py27 inst-nodeps: python-diskcache/.tox/dist/diskcache-0.9.0.zip 89 | py27 runtests: PYTHONHASHSEED='3527394681' 90 | py27 runtests: commands[0] | nosetests 91 | ......................................................................... 92 | ---------------------------------------------------------------------- 93 | Ran 98 tests in 29.404s 94 | 95 | OK 96 | py34 inst-nodeps: python-diskcache/.tox/dist/diskcache-0.9.0.zip 97 | py34 runtests: PYTHONHASHSEED='3527394681' 98 | py34 runtests: commands[0] | nosetests 99 | ......................................................................... 100 | ---------------------------------------------------------------------- 101 | Ran 98 tests in 22.841s 102 | 103 | OK 104 | py35 inst-nodeps: python-diskcache/.tox/dist/diskcache-0.9.0.zip 105 | py35 runtests: PYTHONHASHSEED='3527394681' 106 | py35 runtests: commands[0] | nosetests 107 | ......................................................................... 108 | ---------------------------------------------------------------------- 109 | Ran 98 tests in 23.803s 110 | 111 | OK 112 | ____________________ summary ____________________ 113 | py27: commands succeeded 114 | py34: commands succeeded 115 | py35: commands succeeded 116 | congratulations :) 117 | 118 | Coverage testing uses `nose `_: 119 | 120 | :: 121 | 122 | $ nosetests --cover-erase --with-coverage --cover-package diskcache 123 | ......................................................................... 124 | Name Stmts Miss Cover Missing 125 | -------------------------------------------------------- 126 | diskcache.py 13 2 85% 9-11 127 | diskcache/core.py 442 4 99% 22-25 128 | diskcache/djangocache.py 43 0 100% 129 | diskcache/fanout.py 66 0 100% 130 | -------------------------------------------------------- 131 | TOTAL 564 6 99% 132 | ---------------------------------------------------------------------- 133 | Ran 98 tests in 28.766s 134 | 135 | OK 136 | 137 | It's normal to not see 100% coverage. Some code is specific to the Python 138 | runtime. 139 | 140 | Stress testing is also based on nose but can be run independently as a 141 | module. Stress tests are kept in the tests directory and prefixed with 142 | ``stress_test_``. Stress tests accept many arguments. Read the help for 143 | details. 144 | 145 | :: 146 | 147 | $ python -m tests.stress_test_core --help 148 | usage: stress_test_core.py [-h] [-n OPERATIONS] [-g GET_AVERAGE] 149 | [-k KEY_COUNT] [-d DEL_CHANCE] [-w WARMUP] 150 | [-e EXPIRE] [-t THREADS] [-p PROCESSES] [-s SEED] 151 | [--no-create] [--no-delete] [-v EVICTION_POLICY] 152 | 153 | optional arguments: 154 | -h, --help show this help message and exit 155 | -n OPERATIONS, --operations OPERATIONS 156 | Number of operations to perform (default: 10000) 157 | -g GET_AVERAGE, --get-average GET_AVERAGE 158 | Expected value of exponential variate used for GET 159 | count (default: 100) 160 | -k KEY_COUNT, --key-count KEY_COUNT 161 | Number of unique keys (default: 10) 162 | -d DEL_CHANCE, --del-chance DEL_CHANCE 163 | Likelihood of a key deletion (default: 0.1) 164 | -w WARMUP, --warmup WARMUP 165 | Number of warmup operations before timings (default: 166 | 10) 167 | -e EXPIRE, --expire EXPIRE 168 | Number of seconds before key expires (default: None) 169 | -t THREADS, --threads THREADS 170 | Number of threads to start in each process (default: 171 | 1) 172 | -p PROCESSES, --processes PROCESSES 173 | Number of processes to start (default: 1) 174 | -s SEED, --seed SEED Random seed (default: 0) 175 | --no-create Do not create operations data (default: True) 176 | --no-delete Do not delete operations data (default: True) 177 | -v EVICTION_POLICY, --eviction-policy EVICTION_POLICY 178 | 179 | If stress exits normally then it worked successfully. Some stress is run by tox 180 | and nose but the iteration count is limited. More rigorous testing requires 181 | increasing the iteration count to millions. At that level, it's best to just 182 | let it run overnight. Stress testing will stop at the first failure. 183 | 184 | Running Benchmarks 185 | ------------------ 186 | 187 | Running and plotting benchmarks is a two step process. Each is a Python script 188 | in the tests directory. Benchmark scripts are prefixed with ``benchmark_``. For 189 | example: 190 | 191 | :: 192 | 193 | $ python tests/benchmark_core.py --help 194 | usage: benchmark_core.py [-h] [-p PROCESSES] [-n OPERATIONS] [-r RANGE] 195 | [-w WARMUP] 196 | 197 | optional arguments: 198 | -h, --help show this help message and exit 199 | -p PROCESSES, --processes PROCESSES 200 | Number of processes to start (default: 8) 201 | -n OPERATIONS, --operations OPERATIONS 202 | Number of operations to perform (default: 100000) 203 | -r RANGE, --range RANGE 204 | Range of keys (default: 100) 205 | -w WARMUP, --warmup WARMUP 206 | Number of warmup operations before timings (default: 207 | 1000) 208 | 209 | Benchmark output is stored in text files prefixed with ``timings_`` in the 210 | `tests` directory. Plotting the benchmarks is done by passing the timings file 211 | as an argument to ``plot.py``. 212 | -------------------------------------------------------------------------------- /docs/djangocache-benchmarks.rst: -------------------------------------------------------------------------------- 1 | DiskCache DjangoCache Benchmarks 2 | ================================ 3 | 4 | :doc:`DiskCache ` provides a Django-compatible cache API in 5 | :class:`diskcache.DjangoCache`. A discussion of its options and abilities are 6 | described in the :doc:`tutorial `. Here we try to assess its 7 | performance compared to other Django cache backends. 8 | 9 | Keys and Values 10 | --------------- 11 | 12 | A survey of repositories on Github showed a diversity of cached values. Among 13 | those observed values were: 14 | 15 | 1. Processed text, most commonly HTML. The average HTML page size in 2014 was 16 | 59KB. Javascript assets totalled an average of 295KB and images range 17 | dramatically but averaged 1.2MB. 18 | 2. QuerySets, the building blocks of the Django ORM. 19 | 3. Numbers, settings, and labels. Generally small values that vary in how often 20 | they change. 21 | 22 | The diversity of cached values presents unique challenges. Below, keys and 23 | values, are constrained simply to short byte strings. This is done to filter 24 | out overhead from pickling, etc. from the benchmarks. 25 | 26 | Backends 27 | -------- 28 | 29 | Django ships with four cache backends: Memcached, Database, Filesystem, and 30 | Local-memory. The Memcached backend uses the `PyLibMC`_ client backend. 31 | Included in the results below is also Redis provided by the `django-redis`_ 32 | project built atop `redis-py`_. 33 | 34 | Not included were four projects which were difficult to setup and so 35 | impractical for testing. 36 | 37 | 1. | uWSGI cache backend. 38 | | https://pypi.python.org/pypi/django-uwsgi-cache 39 | 2. | Amazon S3 backend. 40 | | https://pypi.python.org/pypi/django-s3-cache 41 | 3. | MongoDB cache backend. 42 | | https://pypi.python.org/pypi/django-mongodb-cash-backend 43 | 4. | Cacheops - incompatible filebased caching. 44 | | https://pypi.python.org/pypi/django-cacheops 45 | 46 | Other caching related projects worth mentioning: 47 | 48 | 5. | Request-specific in-memory cache. 49 | | http://pythonhosted.org/johnny-cache/localstore_cache.html 50 | 6. | Cacheback moves all cache store operations to background Celery tasks. 51 | | https://pypi.python.org/pypi/django-cacheback 52 | 7. | Newcache claims to improve Django's Memcached backend. 53 | | https://pypi.python.org/pypi/django-newcache 54 | 8. | Supports tagging cache entries. 55 | | https://pypi.python.org/pypi/cache-tagging 56 | 57 | There are also Django packages which automatically cache database queries by 58 | patching the ORM. `Cachalot`_ has a good comparison and discussion in its 59 | introduction. 60 | 61 | .. _`PyLibMC`: https://pypi.python.org/pypi/pylibmc 62 | .. _`django-redis`: https://pypi.python.org/pypi/django-redis 63 | .. _`redis-py`: https://pypi.python.org/pypi/redis 64 | .. _`Cachalot`: http://django-cachalot.readthedocs.org/en/latest/introduction.html 65 | 66 | Filebased 67 | --------- 68 | 69 | Django's filesystem cache backend has a severe drawback. Every `set` operation 70 | checks whether a cull operation is necessary. This check requires listing all 71 | the files in the directory. To do so a call to ``glob.glob1`` is made. As the 72 | directory size increases, the call slows linearly. 73 | 74 | ============ ============ 75 | Timings for glob.glob1 76 | ------------------------- 77 | Count Time 78 | ============ ============ 79 | 1 1.602ms 80 | 10 2.213ms 81 | 100 8.946ms 82 | 1000 65.869ms 83 | 10000 604.972ms 84 | 100000 6.450s 85 | ============ ============ 86 | 87 | Above, the count regards the number of files in the directory and the time is 88 | the duration of the function call. At only a hundred files, it takes more than 89 | five milliseconds to construct the list of files. 90 | 91 | Concurrent Access 92 | ----------------- 93 | 94 | The concurrent access workload starts eight worker processes each with 95 | different and interleaved operations. None of these benchmarks saturated all 96 | the processors. Operations used 1,100 unique keys and, where applicable, caches 97 | were limited to 1,000 keys. This was done to illustrate the impact of the 98 | culling strategy in ``locmem`` and ``filebased`` caches. 99 | 100 | Get 101 | ... 102 | 103 | .. image:: _static/djangocache-get.png 104 | 105 | Under heavy load, :class:`DjangoCache ` gets are low 106 | latency. At the 99th percentile they are on par with the Memcached cache 107 | backend. 108 | 109 | Set 110 | ... 111 | 112 | .. image:: _static/djangocache-set.png 113 | 114 | Not displayed above is the filebased cache backend. At all percentiles, the 115 | latency exceeded five milliseconds. Timing data is available below. Though 116 | :doc:`DiskCache ` is the slowest, its latency remains competitive. 117 | 118 | Delete 119 | ...... 120 | 121 | .. image:: _static/djangocache-delete.png 122 | 123 | Like sets, deletes require writes to disk. Though :class:`DjangoCache 124 | ` is the slowest, it remains competitive with latency 125 | less than five milliseconds. Remember that unlike Local-memory, Memached, and 126 | Redis, it persists all cached data. 127 | 128 | Timing Data 129 | ........... 130 | 131 | Not all data is easily displayed in the graphs above. Miss rate, maximum 132 | latency and total latency is recorded below. 133 | 134 | ========= ========= ========= ========= ========= ========= ========= ========= 135 | Timings for locmem 136 | ------------------------------------------------------------------------------- 137 | Action Count Miss Median P90 P99 Max Total 138 | ========= ========= ========= ========= ========= ========= ========= ========= 139 | get 712546 140750 36.001us 57.936us 60.081us 10.202ms 28.962s 140 | set 71530 0 36.955us 39.101us 45.061us 2.784ms 2.709s 141 | delete 7916 0 32.902us 35.048us 37.193us 1.524ms 265.399ms 142 | Total 791992 31.936s 143 | ========= ========= ========= ========= ========= ========= ========= ========= 144 | 145 | Notice the high cache miss rate. This reflects the isolation of local memory 146 | caches from each other. Also the culling strategy of local memory caches is 147 | random. 148 | 149 | ========= ========= ========= ========= ========= ========= ========= ========= 150 | Timings for memcached 151 | ------------------------------------------------------------------------------- 152 | Action Count Miss Median P90 P99 Max Total 153 | ========= ========= ========= ========= ========= ========= ========= ========= 154 | get 712546 69185 87.023us 99.182us 110.865us 576.973us 61.758s 155 | set 71530 0 89.169us 102.043us 114.202us 259.876us 6.395s 156 | delete 7916 0 85.115us 97.990us 108.957us 201.941us 672.212ms 157 | Total 791992 68.825s 158 | ========= ========= ========= ========= ========= ========= ========= ========= 159 | 160 | Memcached performance is low latency and stable. 161 | 162 | ========= ========= ========= ========= ========= ========= ========= ========= 163 | Timings for redis 164 | ------------------------------------------------------------------------------- 165 | Action Count Miss Median P90 P99 Max Total 166 | ========= ========= ========= ========= ========= ========= ========= ========= 167 | get 712546 69526 160.933us 195.980us 239.134us 1.365ms 116.816s 168 | set 71530 0 166.178us 200.987us 242.949us 587.940us 12.143s 169 | delete 7916 791 143.051us 177.860us 217.915us 330.925us 1.165s 170 | Total 791992 130.124s 171 | ========= ========= ========= ========= ========= ========= ========= ========= 172 | 173 | Redis performance is roughly half that of Memcached. Beware the impact of 174 | persistence settings on your Redis performance. Depending on your use of 175 | logging and snapshotting, maximum latency may increase significantly. 176 | 177 | ========= ========= ========= ========= ========= ========= ========= ========= 178 | Timings for diskcache 179 | ------------------------------------------------------------------------------- 180 | Action Count Miss Median P90 P99 Max Total 181 | ========= ========= ========= ========= ========= ========= ========= ========= 182 | get 712546 69509 33.855us 56.982us 79.155us 11.908ms 30.078s 183 | set 71530 0 178.814us 1.355ms 5.032ms 26.620ms 34.461s 184 | delete 7916 0 107.050us 1.280ms 4.738ms 17.217ms 3.303s 185 | Total 791992 67.842s 186 | ========= ========= ========= ========= ========= ========= ========= ========= 187 | 188 | :class:`DjangoCache ` defaults to using eight shards 189 | with a 10 millisecond timeout. Notice that cache get operations are in 190 | aggregate more than twice as fast as Memcached. And total cache time for all 191 | operations is comparable. The higher set and delete latencies are due to the 192 | retry behavior of :class:`DjangoCache ` objects. If 193 | lower latency is required then the retry behavior can be disabled. 194 | 195 | ========= ========= ========= ========= ========= ========= ========= ========= 196 | Timings for filebased 197 | ------------------------------------------------------------------------------- 198 | Action Count Miss Median P90 P99 Max Total 199 | ========= ========= ========= ========= ========= ========= ========= ========= 200 | get 712749 103843 112.772us 193.119us 423.908us 18.428ms 92.428s 201 | set 71431 0 8.893ms 11.742ms 14.790ms 44.201ms 646.879s 202 | delete 7812 0 223.875us 389.099us 679.016us 15.058ms 1.940s 203 | Total 791992 741.247s 204 | ========= ========= ========= ========= ========= ========= ========= ========= 205 | 206 | Notice the higher cache miss rate. That's a result of the cache's random 207 | culling strategy. Get and set operations also take three to twenty times longer 208 | in aggregate as compared with :class:`DjangoCache `. 209 | -------------------------------------------------------------------------------- /docs/index.rst: -------------------------------------------------------------------------------- 1 | .. include:: ../README.rst 2 | 3 | .. toctree:: 4 | :hidden: 5 | 6 | tutorial 7 | cache-benchmarks 8 | djangocache-benchmarks 9 | case-study-web-crawler 10 | case-study-landing-page-caching 11 | sf-python-2017-meetup-talk 12 | api 13 | development 14 | -------------------------------------------------------------------------------- /docs/make.bat: -------------------------------------------------------------------------------- 1 | @ECHO OFF 2 | 3 | pushd %~dp0 4 | 5 | REM Command file for Sphinx documentation 6 | 7 | if "%SPHINXBUILD%" == "" ( 8 | set SPHINXBUILD=sphinx-build 9 | ) 10 | set SOURCEDIR=. 11 | set BUILDDIR=_build 12 | 13 | if "%1" == "" goto help 14 | 15 | %SPHINXBUILD% >NUL 2>NUL 16 | if errorlevel 9009 ( 17 | echo. 18 | echo.The 'sphinx-build' command was not found. Make sure you have Sphinx 19 | echo.installed, then set the SPHINXBUILD environment variable to point 20 | echo.to the full path of the 'sphinx-build' executable. Alternatively you 21 | echo.may add the Sphinx directory to PATH. 22 | echo. 23 | echo.If you don't have Sphinx installed, grab it from 24 | echo.http://sphinx-doc.org/ 25 | exit /b 1 26 | ) 27 | 28 | %SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% 29 | goto end 30 | 31 | :help 32 | %SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% 33 | 34 | :end 35 | popd 36 | -------------------------------------------------------------------------------- /docs/sf-python-2017-meetup-talk.rst: -------------------------------------------------------------------------------- 1 | Talk: All Things Cached - SF Python 2017 Meetup 2 | =============================================== 3 | 4 | * `Python All Things Cached Slides`_ 5 | * Can we have some fun together in this talk? 6 | * Can I show you some code that I would not run in production? 7 | * Great talk by David Beazley at PyCon Israel this year. 8 | 9 | * Encourages us to scratch our itch under the code phrase: "It's just a 10 | prototype." Not a bad place to start. Often how it ends :) 11 | 12 | 13 | Landscape 14 | --------- 15 | 16 | * At face value, caches seem simple: get/set/delete. 17 | * But zoom in a little and you find just more and more detail. 18 | 19 | 20 | Backends 21 | -------- 22 | 23 | * Backends have different designs and tradeoffs. 24 | 25 | 26 | Frameworks 27 | ---------- 28 | 29 | * Caches have broad applications. 30 | * Web and scientific communities reach for them first. 31 | 32 | 33 | I can haz mor memory? 34 | --------------------- 35 | 36 | * Redis is great technology: free, open source, fast. 37 | * But another process to manage and more memory required. 38 | 39 | :: 40 | 41 | $ emacs talk/settings.py 42 | $ emacs talk/urls.py 43 | $ emacs talk/views.py 44 | 45 | :: 46 | 47 | $ gunicorn --reload talk.wsgi 48 | 49 | :: 50 | 51 | $ emacs benchmark.py 52 | 53 | :: 54 | 55 | $ python benchmark.py 56 | 57 | * I dislike benchmarks in general so don't copy this code. I kind of stole it 58 | from Beazley in another great talk he did on concurrency in Python. He said 59 | not to copy it so I'm telling you not to copy it. 60 | 61 | :: 62 | 63 | $ python manage.py shell 64 | 65 | .. code-block:: pycon 66 | 67 | >>> import time 68 | >>> from django.conf import settings 69 | >>> from django.core.cache import caches 70 | 71 | .. code-block:: pycon 72 | 73 | >>> for key in settings.CACHES.keys(): 74 | ... caches[key].clear() 75 | 76 | :: 77 | 78 | >>> while True: 79 | ... !ls /tmp/filebased | wc -l 80 | ... time.sleep(1) 81 | 82 | 83 | Fool me once, strike one. Feel me twice? Strike three. 84 | ------------------------------------------------------ 85 | 86 | * Filebased cache has two severe drawbacks. 87 | 88 | 1. Culling is random. 89 | 2. set() uses glob.glob1() which slows linearly with directory size. 90 | 91 | 92 | DiskCache 93 | --------- 94 | 95 | * Wanted to solve Django-filebased cache problems. 96 | * Felt like something was missing in the landscape. 97 | * Found an unlikely hero in SQLite. 98 | 99 | 100 | I'd rather drive a slow car fast than a fast car slow 101 | ----------------------------------------------------- 102 | 103 | * Story: driving down the Grapevine in SoCal in friend's 1960s VW Bug. 104 | 105 | 106 | Features 107 | -------- 108 | 109 | * Lot's of features. Maybe a few too many. Ex: never used the tag metadata and 110 | eviction feature. 111 | 112 | 113 | Use Case: Static file serving with read() 114 | ----------------------------------------- 115 | 116 | * Some fun features. Data is stored in files and web servers are good at 117 | serving files. 118 | 119 | 120 | Use Case: Analytics with incr()/pop() 121 | ------------------------------------- 122 | 123 | * Tried to create really functional APIs. 124 | * All write operations are atomic. 125 | 126 | 127 | Case Study: Baby Web Crawler 128 | ---------------------------- 129 | 130 | * Convert from ephemeral, single-process to persistent, multi-process. 131 | 132 | 133 | "get" Time vs Percentile 134 | ------------------------ 135 | 136 | * Tradeoff cache latency and miss-rate using timeout. 137 | 138 | 139 | "set" Time vs Percentile 140 | ------------------------ 141 | 142 | * Django-filebased cache so slow, can't plot. 143 | 144 | 145 | Design 146 | ------ 147 | 148 | * Cache is a single shard. FanoutCache uses multiple shards. Trick is 149 | cross-platform hash. 150 | * Pickle can actually be fast if you use a higher protocol. Default 0. Up to 4 151 | now. 152 | 153 | * Don't choose higher than 2 if you want to be portable between Python 2 154 | and 3. 155 | 156 | * Size limit really indicates when to start culling. Limit number of items 157 | deleted. 158 | 159 | 160 | SQLite 161 | ------ 162 | 163 | * Tradeoff cache latency and miss-rate using timeout. 164 | * SQLite supports 64-bit integers and floats, UTF-8 text and binary blobs. 165 | * Use a context manager for isolation level management. 166 | * Pragmas tune the behavior and performance of SQLite. 167 | 168 | * Default is robust and slow. 169 | * Use write-ahead-log so writers don't block readers. 170 | * Memory-map pages for fast lookups. 171 | 172 | 173 | Best way to make money in photography? Sell all your gear. 174 | ---------------------------------------------------------- 175 | 176 | * Who saw eclipse? Awesome, right? 177 | 178 | * Hard to really photograph the experience. 179 | * This is me, staring up at the sun, blinding myself as I hold my glasses and 180 | my phone to take a photo. Clearly lousy. 181 | 182 | * Software talks are hard to get right and I can't cover everything related to 183 | caching in 20 minutes. I hope you've learned something tonight or at least 184 | seen something interesting. 185 | 186 | 187 | Conclusion 188 | ---------- 189 | 190 | * Windows support mostly "just worked". 191 | 192 | * SQLite is truly cross-platform. 193 | * Filesystems are a little different. 194 | * AppVeyor was about half as fast as Travis. 195 | * check() to fix inconsistencies. 196 | 197 | * Caveats: 198 | 199 | * NFS and SQLite do not play nice. 200 | * Not well suited to queues (want read:write at 10:1 or higher). 201 | 202 | * Alternative databases: BerkeleyDB, LMDB, RocksDB, LevelDB, etc. 203 | * Engage with me on Github, find bugs, complain about performance. 204 | * If you like the project, star-it on Github and share it with friends. 205 | * Thanks for letting me share tonight. Questions? 206 | 207 | .. _`Python All Things Cached Slides`: http://bit.ly/dc-2017-slides 208 | -------------------------------------------------------------------------------- /mypy.ini: -------------------------------------------------------------------------------- 1 | [mypy] 2 | 3 | [mypy-django.*] 4 | ignore_missing_imports = True 5 | -------------------------------------------------------------------------------- /requirements-dev.txt: -------------------------------------------------------------------------------- 1 | -e . 2 | blue 3 | coverage 4 | django==4.2.* 5 | django_redis 6 | doc8 7 | flake8 8 | ipython 9 | jedi 10 | pickleDB 11 | pylibmc 12 | pylint 13 | pytest 14 | pytest-cov 15 | pytest-django 16 | pytest-env 17 | pytest-xdist 18 | rstcheck 19 | sphinx 20 | sqlitedict 21 | tox 22 | twine 23 | wheel 24 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | -e . 2 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | from setuptools import setup 2 | from setuptools.command.test import test as TestCommand 3 | 4 | import diskcache 5 | 6 | 7 | class Tox(TestCommand): 8 | def finalize_options(self): 9 | TestCommand.finalize_options(self) 10 | self.test_args = [] 11 | self.test_suite = True 12 | 13 | def run_tests(self): 14 | import tox 15 | 16 | errno = tox.cmdline(self.test_args) 17 | exit(errno) 18 | 19 | 20 | with open('README.rst', encoding='utf-8') as reader: 21 | readme = reader.read() 22 | 23 | setup( 24 | name=diskcache.__title__, 25 | version=diskcache.__version__, 26 | description='Disk Cache -- Disk and file backed persistent cache.', 27 | long_description=readme, 28 | author='Grant Jenks', 29 | author_email='contact@grantjenks.com', 30 | url='http://www.grantjenks.com/docs/diskcache/', 31 | project_urls={ 32 | 'Documentation': 'http://www.grantjenks.com/docs/diskcache/', 33 | 'Funding': 'https://gum.co/diskcache', 34 | 'Source': 'https://github.com/grantjenks/python-diskcache', 35 | 'Tracker': 'https://github.com/grantjenks/python-diskcache/issues', 36 | }, 37 | license='Apache 2.0', 38 | packages=['diskcache'], 39 | tests_require=['tox'], 40 | cmdclass={'test': Tox}, 41 | python_requires='>=3', 42 | install_requires=[], 43 | classifiers=( 44 | 'Development Status :: 5 - Production/Stable', 45 | 'Intended Audience :: Developers', 46 | 'License :: OSI Approved :: Apache Software License', 47 | 'Natural Language :: English', 48 | 'Programming Language :: Python', 49 | 'Programming Language :: Python :: 3', 50 | 'Programming Language :: Python :: 3.5', 51 | 'Programming Language :: Python :: 3.6', 52 | 'Programming Language :: Python :: 3.7', 53 | 'Programming Language :: Python :: 3.8', 54 | 'Programming Language :: Python :: 3.9', 55 | 'Programming Language :: Python :: Implementation :: CPython', 56 | ), 57 | ) 58 | -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/grantjenks/python-diskcache/ebfa37cd99d7ef716ec452ad8af4b4276a8e2233/tests/__init__.py -------------------------------------------------------------------------------- /tests/benchmark_core.py: -------------------------------------------------------------------------------- 1 | """Benchmark diskcache.Cache 2 | 3 | $ export PYTHONPATH=/Users/grantj/repos/python-diskcache 4 | $ python tests/benchmark_core.py -p 1 > tests/timings_core_p1.txt 5 | $ python tests/benchmark_core.py -p 8 > tests/timings_core_p8.txt 6 | """ 7 | 8 | import collections as co 9 | import multiprocessing as mp 10 | import os 11 | import pickle 12 | import random 13 | import shutil 14 | import time 15 | import warnings 16 | 17 | from utils import display 18 | 19 | PROCS = 8 20 | OPS = int(1e5) 21 | RANGE = 100 22 | WARMUP = int(1e3) 23 | 24 | caches = [] 25 | 26 | 27 | ############################################################################### 28 | # Disk Cache Benchmarks 29 | ############################################################################### 30 | 31 | import diskcache # noqa 32 | 33 | caches.append( 34 | ( 35 | 'diskcache.Cache', 36 | diskcache.Cache, 37 | ('tmp',), 38 | {}, 39 | ) 40 | ) 41 | caches.append( 42 | ( 43 | 'diskcache.FanoutCache(shards=4, timeout=1.0)', 44 | diskcache.FanoutCache, 45 | ('tmp',), 46 | {'shards': 4, 'timeout': 1.0}, 47 | ) 48 | ) 49 | caches.append( 50 | ( 51 | 'diskcache.FanoutCache(shards=8, timeout=0.010)', 52 | diskcache.FanoutCache, 53 | ('tmp',), 54 | {'shards': 8, 'timeout': 0.010}, 55 | ) 56 | ) 57 | 58 | 59 | ############################################################################### 60 | # PyLibMC Benchmarks 61 | ############################################################################### 62 | 63 | try: 64 | import pylibmc 65 | 66 | caches.append( 67 | ( 68 | 'pylibmc.Client', 69 | pylibmc.Client, 70 | (['127.0.0.1'],), 71 | { 72 | 'binary': True, 73 | 'behaviors': {'tcp_nodelay': True, 'ketama': True}, 74 | }, 75 | ) 76 | ) 77 | except ImportError: 78 | warnings.warn('skipping pylibmc') 79 | 80 | 81 | ############################################################################### 82 | # Redis Benchmarks 83 | ############################################################################### 84 | 85 | try: 86 | import redis 87 | 88 | caches.append( 89 | ( 90 | 'redis.StrictRedis', 91 | redis.StrictRedis, 92 | (), 93 | {'host': 'localhost', 'port': 6379, 'db': 0}, 94 | ) 95 | ) 96 | except ImportError: 97 | warnings.warn('skipping redis') 98 | 99 | 100 | def worker(num, kind, args, kwargs): 101 | random.seed(num) 102 | 103 | time.sleep(0.01) # Let other processes start. 104 | 105 | obj = kind(*args, **kwargs) 106 | 107 | timings = co.defaultdict(list) 108 | 109 | for count in range(OPS): 110 | key = str(random.randrange(RANGE)).encode('utf-8') 111 | value = str(count).encode('utf-8') * random.randrange(1, 100) 112 | choice = random.random() 113 | 114 | if choice < 0.900: 115 | start = time.time() 116 | result = obj.get(key) 117 | end = time.time() 118 | miss = result is None 119 | action = 'get' 120 | elif choice < 0.990: 121 | start = time.time() 122 | result = obj.set(key, value) 123 | end = time.time() 124 | miss = result is False 125 | action = 'set' 126 | else: 127 | start = time.time() 128 | result = obj.delete(key) 129 | end = time.time() 130 | miss = result is False 131 | action = 'delete' 132 | 133 | if count > WARMUP: 134 | delta = end - start 135 | timings[action].append(delta) 136 | if miss: 137 | timings[action + '-miss'].append(delta) 138 | 139 | with open('output-%d.pkl' % num, 'wb') as writer: 140 | pickle.dump(timings, writer, protocol=pickle.HIGHEST_PROTOCOL) 141 | 142 | 143 | def dispatch(): 144 | for name, kind, args, kwargs in caches: 145 | shutil.rmtree('tmp', ignore_errors=True) 146 | 147 | obj = kind(*args, **kwargs) 148 | 149 | for key in range(RANGE): 150 | key = str(key).encode('utf-8') 151 | obj.set(key, key) 152 | 153 | try: 154 | obj.close() 155 | except Exception: 156 | pass 157 | 158 | processes = [ 159 | mp.Process(target=worker, args=(value, kind, args, kwargs)) 160 | for value in range(PROCS) 161 | ] 162 | 163 | for process in processes: 164 | process.start() 165 | 166 | for process in processes: 167 | process.join() 168 | 169 | timings = co.defaultdict(list) 170 | 171 | for num in range(PROCS): 172 | filename = 'output-%d.pkl' % num 173 | 174 | with open(filename, 'rb') as reader: 175 | output = pickle.load(reader) 176 | 177 | for key in output: 178 | timings[key].extend(output[key]) 179 | 180 | os.remove(filename) 181 | 182 | display(name, timings) 183 | 184 | 185 | if __name__ == '__main__': 186 | import argparse 187 | 188 | parser = argparse.ArgumentParser( 189 | formatter_class=argparse.ArgumentDefaultsHelpFormatter, 190 | ) 191 | parser.add_argument( 192 | '-p', 193 | '--processes', 194 | type=int, 195 | default=PROCS, 196 | help='Number of processes to start', 197 | ) 198 | parser.add_argument( 199 | '-n', 200 | '--operations', 201 | type=float, 202 | default=OPS, 203 | help='Number of operations to perform', 204 | ) 205 | parser.add_argument( 206 | '-r', 207 | '--range', 208 | type=int, 209 | default=RANGE, 210 | help='Range of keys', 211 | ) 212 | parser.add_argument( 213 | '-w', 214 | '--warmup', 215 | type=float, 216 | default=WARMUP, 217 | help='Number of warmup operations before timings', 218 | ) 219 | 220 | args = parser.parse_args() 221 | 222 | PROCS = int(args.processes) 223 | OPS = int(args.operations) 224 | RANGE = int(args.range) 225 | WARMUP = int(args.warmup) 226 | 227 | dispatch() 228 | -------------------------------------------------------------------------------- /tests/benchmark_djangocache.py: -------------------------------------------------------------------------------- 1 | """Benchmark diskcache.DjangoCache 2 | 3 | $ export PYTHONPATH=/Users/grantj/repos/python-diskcache 4 | $ python tests/benchmark_djangocache.py > tests/timings_djangocache.txt 5 | """ 6 | 7 | import collections as co 8 | import multiprocessing as mp 9 | import os 10 | import pickle 11 | import random 12 | import shutil 13 | import time 14 | 15 | from utils import display 16 | 17 | PROCS = 8 18 | OPS = int(1e5) 19 | RANGE = int(1.1e3) 20 | WARMUP = int(1e3) 21 | 22 | 23 | def setup(): 24 | os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'tests.settings_benchmark') 25 | import django 26 | 27 | django.setup() 28 | 29 | 30 | def worker(num, name): 31 | setup() 32 | 33 | from django.core.cache import caches 34 | 35 | obj = caches[name] 36 | 37 | random.seed(num) 38 | 39 | timings = co.defaultdict(list) 40 | 41 | time.sleep(0.01) # Let other processes start. 42 | 43 | for count in range(OPS): 44 | key = str(random.randrange(RANGE)).encode('utf-8') 45 | value = str(count).encode('utf-8') * random.randrange(1, 100) 46 | choice = random.random() 47 | 48 | if choice < 0.900: 49 | start = time.time() 50 | result = obj.get(key) 51 | end = time.time() 52 | miss = result is None 53 | action = 'get' 54 | elif choice < 0.990: 55 | start = time.time() 56 | result = obj.set(key, value) 57 | end = time.time() 58 | miss = result is False 59 | action = 'set' 60 | else: 61 | start = time.time() 62 | result = obj.delete(key) 63 | end = time.time() 64 | miss = result is False 65 | action = 'delete' 66 | 67 | if count > WARMUP: 68 | delta = end - start 69 | timings[action].append(delta) 70 | if miss: 71 | timings[action + '-miss'].append(delta) 72 | 73 | with open('output-%d.pkl' % num, 'wb') as writer: 74 | pickle.dump(timings, writer, protocol=pickle.HIGHEST_PROTOCOL) 75 | 76 | 77 | def prepare(name): 78 | setup() 79 | 80 | from django.core.cache import caches 81 | 82 | obj = caches[name] 83 | 84 | for key in range(RANGE): 85 | key = str(key).encode('utf-8') 86 | obj.set(key, key) 87 | 88 | try: 89 | obj.close() 90 | except Exception: 91 | pass 92 | 93 | 94 | def dispatch(): 95 | setup() 96 | 97 | from django.core.cache import caches # noqa 98 | 99 | for name in ['locmem', 'memcached', 'redis', 'diskcache', 'filebased']: 100 | shutil.rmtree('tmp', ignore_errors=True) 101 | 102 | preparer = mp.Process(target=prepare, args=(name,)) 103 | preparer.start() 104 | preparer.join() 105 | 106 | processes = [ 107 | mp.Process(target=worker, args=(value, name)) 108 | for value in range(PROCS) 109 | ] 110 | 111 | for process in processes: 112 | process.start() 113 | 114 | for process in processes: 115 | process.join() 116 | 117 | timings = co.defaultdict(list) 118 | 119 | for num in range(PROCS): 120 | filename = 'output-%d.pkl' % num 121 | 122 | with open(filename, 'rb') as reader: 123 | output = pickle.load(reader) 124 | 125 | for key in output: 126 | timings[key].extend(output[key]) 127 | 128 | os.remove(filename) 129 | 130 | display(name, timings) 131 | 132 | 133 | if __name__ == '__main__': 134 | import argparse 135 | 136 | parser = argparse.ArgumentParser( 137 | formatter_class=argparse.ArgumentDefaultsHelpFormatter, 138 | ) 139 | parser.add_argument( 140 | '-p', 141 | '--processes', 142 | type=int, 143 | default=PROCS, 144 | help='Number of processes to start', 145 | ) 146 | parser.add_argument( 147 | '-n', 148 | '--operations', 149 | type=float, 150 | default=OPS, 151 | help='Number of operations to perform', 152 | ) 153 | parser.add_argument( 154 | '-r', 155 | '--range', 156 | type=int, 157 | default=RANGE, 158 | help='Range of keys', 159 | ) 160 | parser.add_argument( 161 | '-w', 162 | '--warmup', 163 | type=float, 164 | default=WARMUP, 165 | help='Number of warmup operations before timings', 166 | ) 167 | 168 | args = parser.parse_args() 169 | 170 | PROCS = int(args.processes) 171 | OPS = int(args.operations) 172 | RANGE = int(args.range) 173 | WARMUP = int(args.warmup) 174 | 175 | dispatch() 176 | -------------------------------------------------------------------------------- /tests/benchmark_glob.py: -------------------------------------------------------------------------------- 1 | """Benchmark glob.glob1 as used by django.core.cache.backends.filebased.""" 2 | 3 | import os 4 | import os.path as op 5 | import shutil 6 | import timeit 7 | 8 | from utils import secs 9 | 10 | shutil.rmtree('tmp', ignore_errors=True) 11 | 12 | os.mkdir('tmp') 13 | 14 | size = 12 15 | cols = ('Count', 'Time') 16 | template = ' '.join(['%' + str(size) + 's'] * len(cols)) 17 | 18 | print() 19 | print(' '.join(['=' * size] * len(cols))) 20 | print('Timings for glob.glob1') 21 | print('-'.join(['-' * size] * len(cols))) 22 | print(template % ('Count', 'Time')) 23 | print(' '.join(['=' * size] * len(cols))) 24 | 25 | for count in [10**exp for exp in range(6)]: 26 | for value in range(count): 27 | with open(op.join('tmp', '%s.tmp' % value), 'wb') as writer: 28 | pass 29 | 30 | delta = timeit.timeit( 31 | stmt="glob.glob1('tmp', '*.tmp')", setup='import glob', number=100 32 | ) 33 | 34 | print(template % (count, secs(delta))) 35 | 36 | print(' '.join(['=' * size] * len(cols))) 37 | 38 | shutil.rmtree('tmp', ignore_errors=True) 39 | -------------------------------------------------------------------------------- /tests/benchmark_incr.py: -------------------------------------------------------------------------------- 1 | """Benchmark cache.incr method. 2 | """ 3 | 4 | import json 5 | import multiprocessing as mp 6 | import shutil 7 | import time 8 | 9 | import diskcache as dc 10 | 11 | from .utils import secs 12 | 13 | COUNT = int(1e3) 14 | PROCS = 8 15 | 16 | 17 | def worker(num): 18 | """Rapidly increment key and time operation.""" 19 | time.sleep(0.1) # Let other workers start. 20 | 21 | cache = dc.Cache('tmp') 22 | values = [] 23 | 24 | for _ in range(COUNT): 25 | start = time.time() 26 | cache.incr(b'key') 27 | end = time.time() 28 | values.append(end - start) 29 | 30 | with open('output-%s.json' % num, 'w') as writer: 31 | json.dump(values, writer) 32 | 33 | 34 | def main(): 35 | """Run workers and print percentile results.""" 36 | shutil.rmtree('tmp', ignore_errors=True) 37 | 38 | processes = [ 39 | mp.Process(target=worker, args=(num,)) for num in range(PROCS) 40 | ] 41 | 42 | for process in processes: 43 | process.start() 44 | 45 | for process in processes: 46 | process.join() 47 | 48 | with dc.Cache('tmp') as cache: 49 | assert cache.get(b'key') == COUNT * PROCS 50 | 51 | for num in range(PROCS): 52 | values = [] 53 | with open('output-%s.json' % num) as reader: 54 | values += json.load(reader) 55 | 56 | values.sort() 57 | p50 = int(len(values) * 0.50) - 1 58 | p90 = int(len(values) * 0.90) - 1 59 | p99 = int(len(values) * 0.99) - 1 60 | p00 = len(values) - 1 61 | print(['{0:9s}'.format(val) for val in 'p50 p90 p99 max'.split()]) 62 | print([secs(values[pos]) for pos in [p50, p90, p99, p00]]) 63 | 64 | 65 | if __name__ == '__main__': 66 | main() 67 | -------------------------------------------------------------------------------- /tests/benchmark_kv_store.py: -------------------------------------------------------------------------------- 1 | """Benchmarking Key-Value Stores 2 | 3 | $ python -m IPython tests/benchmark_kv_store.py 4 | """ 5 | 6 | from IPython import get_ipython 7 | 8 | import diskcache 9 | 10 | ipython = get_ipython() 11 | assert ipython is not None, 'No IPython! Run with $ ipython ...' 12 | 13 | value = 'value' 14 | 15 | print('diskcache set') 16 | dc = diskcache.FanoutCache('/tmp/diskcache') 17 | ipython.magic("timeit -n 100 -r 7 dc['key'] = value") 18 | print('diskcache get') 19 | ipython.magic("timeit -n 100 -r 7 dc['key']") 20 | print('diskcache set/delete') 21 | ipython.magic("timeit -n 100 -r 7 dc['key'] = value; del dc['key']") 22 | 23 | try: 24 | import dbm.gnu # Only trust GNU DBM 25 | except ImportError: 26 | print('Error: Cannot import dbm.gnu') 27 | print('Error: Skipping import shelve') 28 | else: 29 | print('dbm set') 30 | d = dbm.gnu.open('/tmp/dbm', 'c') 31 | ipython.magic("timeit -n 100 -r 7 d['key'] = value; d.sync()") 32 | print('dbm get') 33 | ipython.magic("timeit -n 100 -r 7 d['key']") 34 | print('dbm set/delete') 35 | ipython.magic( 36 | "timeit -n 100 -r 7 d['key'] = value; d.sync(); del d['key']; d.sync()" 37 | ) 38 | 39 | import shelve 40 | 41 | print('shelve set') 42 | s = shelve.open('/tmp/shelve') 43 | ipython.magic("timeit -n 100 -r 7 s['key'] = value; s.sync()") 44 | print('shelve get') 45 | ipython.magic("timeit -n 100 -r 7 s['key']") 46 | print('shelve set/delete') 47 | ipython.magic( 48 | "timeit -n 100 -r 7 s['key'] = value; s.sync(); del s['key']; s.sync()" 49 | ) 50 | 51 | try: 52 | import sqlitedict 53 | except ImportError: 54 | print('Error: Cannot import sqlitedict') 55 | else: 56 | print('sqlitedict set') 57 | sd = sqlitedict.SqliteDict('/tmp/sqlitedict', autocommit=True) 58 | ipython.magic("timeit -n 100 -r 7 sd['key'] = value") 59 | print('sqlitedict get') 60 | ipython.magic("timeit -n 100 -r 7 sd['key']") 61 | print('sqlitedict set/delete') 62 | ipython.magic("timeit -n 100 -r 7 sd['key'] = value; del sd['key']") 63 | 64 | try: 65 | import pickledb 66 | except ImportError: 67 | print('Error: Cannot import pickledb') 68 | else: 69 | print('pickledb set') 70 | p = pickledb.load('/tmp/pickledb', True) 71 | ipython.magic("timeit -n 100 -r 7 p['key'] = value") 72 | print('pickledb get') 73 | ipython.magic( 74 | "timeit -n 100 -r 7 p = pickledb.load('/tmp/pickledb', True); p['key']" 75 | ) 76 | print('pickledb set/delete') 77 | ipython.magic("timeit -n 100 -r 7 p['key'] = value; del p['key']") 78 | -------------------------------------------------------------------------------- /tests/db.sqlite3: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/grantjenks/python-diskcache/ebfa37cd99d7ef716ec452ad8af4b4276a8e2233/tests/db.sqlite3 -------------------------------------------------------------------------------- /tests/issue_109.py: -------------------------------------------------------------------------------- 1 | """Benchmark for Issue #109 2 | """ 3 | 4 | import time 5 | 6 | import diskcache as dc 7 | 8 | 9 | def main(): 10 | import argparse 11 | 12 | parser = argparse.ArgumentParser() 13 | parser.add_argument('--cache-dir', default='/tmp/test') 14 | parser.add_argument('--iterations', type=int, default=100) 15 | parser.add_argument('--sleep', type=float, default=0.1) 16 | parser.add_argument('--size', type=int, default=25) 17 | args = parser.parse_args() 18 | 19 | data = dc.FanoutCache(args.cache_dir) 20 | delays = [] 21 | values = {str(num): num for num in range(args.size)} 22 | iterations = args.iterations 23 | 24 | for i in range(args.iterations): 25 | print(f'Iteration {i + 1}/{iterations}', end='\r') 26 | time.sleep(args.sleep) 27 | for key, value in values.items(): 28 | start = time.monotonic() 29 | data[key] = value 30 | stop = time.monotonic() 31 | diff = stop - start 32 | delays.append(diff) 33 | 34 | # Discard warmup delays, first two iterations. 35 | del delays[: (len(values) * 2)] 36 | 37 | # Convert seconds to microseconds. 38 | delays = sorted(delay * 1e6 for delay in delays) 39 | 40 | # Display performance. 41 | print() 42 | print(f'Total #: {len(delays)}') 43 | print(f'Min delay (us): {delays[0]:>8.3f}') 44 | print(f'50th %ile (us): {delays[int(len(delays) * 0.50)]:>8.3f}') 45 | print(f'90th %ile (us): {delays[int(len(delays) * 0.90)]:>8.3f}') 46 | print(f'99th %ile (us): {delays[int(len(delays) * 0.99)]:>8.3f}') 47 | print(f'Max delay (us): {delays[-1]:>8.3f}') 48 | 49 | 50 | if __name__ == '__main__': 51 | main() 52 | -------------------------------------------------------------------------------- /tests/issue_85.py: -------------------------------------------------------------------------------- 1 | """Test Script for Issue #85 2 | 3 | $ export PYTHONPATH=`pwd` 4 | $ python tests/issue_85.py 5 | """ 6 | 7 | import collections 8 | import os 9 | import random 10 | import shutil 11 | import sqlite3 12 | import threading 13 | import time 14 | 15 | import django 16 | 17 | 18 | def remove_cache_dir(): 19 | print('REMOVING CACHE DIRECTORY') 20 | shutil.rmtree('.cache', ignore_errors=True) 21 | 22 | 23 | def init_django(): 24 | global shard 25 | print('INITIALIZING DJANGO') 26 | os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'tests.settings') 27 | django.setup() 28 | from django.core.cache import cache 29 | 30 | shard = cache._cache._shards[0] 31 | 32 | 33 | def multi_threading_init_test(): 34 | print('RUNNING MULTI-THREADING INIT TEST') 35 | from django.core.cache import cache 36 | 37 | def run(): 38 | cache.get('key') 39 | 40 | threads = [threading.Thread(target=run) for _ in range(50)] 41 | _ = [thread.start() for thread in threads] 42 | _ = [thread.join() for thread in threads] 43 | 44 | 45 | def show_sqlite_compile_options(): 46 | print('SQLITE COMPILE OPTIONS') 47 | options = shard._sql('pragma compile_options').fetchall() 48 | print('\n'.join(val for val, in options)) 49 | 50 | 51 | def create_data_table(): 52 | print('CREATING DATA TABLE') 53 | shard._con.execute('create table data (x)') 54 | nums = [(num,) for num in range(1000)] 55 | shard._con.executemany('insert into data values (?)', nums) 56 | 57 | 58 | commands = { 59 | 'begin/read/write': [ 60 | 'BEGIN', 61 | 'SELECT MAX(x) FROM data', 62 | 'UPDATE data SET x = x + 1', 63 | 'COMMIT', 64 | ], 65 | 'begin/write/read': [ 66 | 'BEGIN', 67 | 'UPDATE data SET x = x + 1', 68 | 'SELECT MAX(x) FROM data', 69 | 'COMMIT', 70 | ], 71 | 'begin immediate/read/write': [ 72 | 'BEGIN IMMEDIATE', 73 | 'SELECT MAX(x) FROM data', 74 | 'UPDATE data SET x = x + 1', 75 | 'COMMIT', 76 | ], 77 | 'begin immediate/write/read': [ 78 | 'BEGIN IMMEDIATE', 79 | 'UPDATE data SET x = x + 1', 80 | 'SELECT MAX(x) FROM data', 81 | 'COMMIT', 82 | ], 83 | 'begin exclusive/read/write': [ 84 | 'BEGIN EXCLUSIVE', 85 | 'SELECT MAX(x) FROM data', 86 | 'UPDATE data SET x = x + 1', 87 | 'COMMIT', 88 | ], 89 | 'begin exclusive/write/read': [ 90 | 'BEGIN EXCLUSIVE', 91 | 'UPDATE data SET x = x + 1', 92 | 'SELECT MAX(x) FROM data', 93 | 'COMMIT', 94 | ], 95 | } 96 | 97 | 98 | values = collections.deque() 99 | 100 | 101 | def run(statements): 102 | ident = threading.get_ident() 103 | try: 104 | for index, statement in enumerate(statements): 105 | if index == (len(statements) - 1): 106 | values.append(('COMMIT', ident)) 107 | time.sleep(random.random() / 10.0) 108 | shard._sql(statement) 109 | if index == 0: 110 | values.append(('BEGIN', ident)) 111 | except sqlite3.OperationalError: 112 | values.append(('ERROR', ident)) 113 | 114 | 115 | def test_transaction_errors(): 116 | for key, statements in commands.items(): 117 | print(f'RUNNING {key}') 118 | values.clear() 119 | threads = [] 120 | for _ in range(100): 121 | thread = threading.Thread(target=run, args=(statements,)) 122 | threads.append(thread) 123 | _ = [thread.start() for thread in threads] 124 | _ = [thread.join() for thread in threads] 125 | errors = [pair for pair in values if pair[0] == 'ERROR'] 126 | begins = [pair for pair in values if pair[0] == 'BEGIN'] 127 | commits = [pair for pair in values if pair[0] == 'COMMIT'] 128 | print('Error count:', len(errors)) 129 | print('Begin count:', len(begins)) 130 | print('Commit count:', len(commits)) 131 | begin_idents = [ident for _, ident in begins] 132 | commit_idents = [ident for _, ident in commits] 133 | print('Serialized:', begin_idents == commit_idents) 134 | 135 | 136 | if __name__ == '__main__': 137 | remove_cache_dir() 138 | init_django() 139 | multi_threading_init_test() 140 | show_sqlite_compile_options() 141 | create_data_table() 142 | test_transaction_errors() 143 | -------------------------------------------------------------------------------- /tests/models.py: -------------------------------------------------------------------------------- 1 | from django.db import models 2 | from django.utils import timezone 3 | 4 | 5 | def expensive_calculation(): 6 | expensive_calculation.num_runs += 1 7 | return timezone.now() 8 | 9 | 10 | class Poll(models.Model): 11 | question = models.CharField(max_length=200) 12 | answer = models.CharField(max_length=200) 13 | pub_date = models.DateTimeField( 14 | 'date published', default=expensive_calculation 15 | ) 16 | -------------------------------------------------------------------------------- /tests/plot.py: -------------------------------------------------------------------------------- 1 | """Plot Benchmarks for docs 2 | 3 | $ export PYTHONPATH=/Users/grantj/repos/python-diskcache 4 | $ python tests/plot.py --show tests/timings_core_p1.txt 5 | """ 6 | 7 | import argparse 8 | import collections as co 9 | import re 10 | import sys 11 | 12 | import matplotlib.pyplot as plt 13 | 14 | 15 | def parse_timing(timing, limit): 16 | """Parse timing.""" 17 | if timing.endswith('ms'): 18 | value = float(timing[:-2]) * 1e-3 19 | elif timing.endswith('us'): 20 | value = float(timing[:-2]) * 1e-6 21 | else: 22 | assert timing.endswith('s') 23 | value = float(timing[:-1]) 24 | return 0.0 if value > limit else value * 1e6 25 | 26 | 27 | def parse_row(row, line): 28 | """Parse row.""" 29 | return [val.strip() for val in row.match(line).groups()] 30 | 31 | 32 | def parse_data(infile): 33 | """Parse data from `infile`.""" 34 | blocks = re.compile(' '.join(['=' * 9] * 8)) 35 | dashes = re.compile('^-{79}$') 36 | title = re.compile('^Timings for (.*)$') 37 | row = re.compile(' '.join(['(.{9})'] * 7) + ' (.{8,9})') 38 | 39 | lines = infile.readlines() 40 | 41 | data = co.OrderedDict() 42 | index = 0 43 | 44 | while index < len(lines): 45 | line = lines[index] 46 | 47 | if blocks.match(line): 48 | try: 49 | name = title.match(lines[index + 1]).group(1) 50 | except Exception: 51 | index += 1 52 | continue 53 | 54 | data[name] = {} 55 | 56 | assert dashes.match(lines[index + 2]) 57 | 58 | cols = parse_row(row, lines[index + 3]) 59 | 60 | assert blocks.match(lines[index + 4]) 61 | 62 | get_row = parse_row(row, lines[index + 5]) 63 | assert get_row[0] == 'get' 64 | 65 | set_row = parse_row(row, lines[index + 6]) 66 | assert set_row[0] == 'set' 67 | 68 | delete_row = parse_row(row, lines[index + 7]) 69 | assert delete_row[0] == 'delete' 70 | 71 | assert blocks.match(lines[index + 9]) 72 | 73 | data[name]['get'] = dict(zip(cols, get_row)) 74 | data[name]['set'] = dict(zip(cols, set_row)) 75 | data[name]['delete'] = dict(zip(cols, delete_row)) 76 | 77 | index += 10 78 | else: 79 | index += 1 80 | 81 | return data 82 | 83 | 84 | def make_plot(data, action, save=False, show=False, limit=0.005): 85 | """Make plot.""" 86 | fig, ax = plt.subplots(figsize=(8, 10)) 87 | colors = ['#ff7f00', '#377eb8', '#4daf4a', '#984ea3', '#e41a1c'] 88 | width = 0.15 89 | 90 | ticks = ('Median', 'P90', 'P99') 91 | index = (0, 1, 2) 92 | names = list(data) 93 | bars = [] 94 | 95 | for pos, (name, color) in enumerate(zip(names, colors)): 96 | bars.append( 97 | ax.bar( 98 | [val + pos * width for val in index], 99 | [ 100 | parse_timing(data[name][action][tick], limit) 101 | for tick in ticks 102 | ], 103 | width, 104 | color=color, 105 | ) 106 | ) 107 | 108 | ax.set_ylabel('Time (microseconds)') 109 | ax.set_title('"%s" Time vs Percentile' % action) 110 | ax.set_xticks([val + width * (len(data) / 2) for val in index]) 111 | ax.set_xticklabels(ticks) 112 | 113 | box = ax.get_position() 114 | ax.set_position( 115 | [box.x0, box.y0 + box.height * 0.2, box.width, box.height * 0.8] 116 | ) 117 | ax.legend( 118 | [bar[0] for bar in bars], 119 | names, 120 | loc='lower center', 121 | bbox_to_anchor=(0.5, -0.25), 122 | ) 123 | 124 | if show: 125 | plt.show() 126 | 127 | if save: 128 | plt.savefig('%s-%s.png' % (save, action), dpi=120, bbox_inches='tight') 129 | 130 | plt.close() 131 | 132 | 133 | def main(): 134 | parser = argparse.ArgumentParser() 135 | 136 | parser.add_argument( 137 | 'infile', 138 | type=argparse.FileType('r'), 139 | default=sys.stdin, 140 | ) 141 | parser.add_argument('-l', '--limit', type=float, default=0.005) 142 | parser.add_argument('-s', '--save') 143 | parser.add_argument('--show', action='store_true') 144 | 145 | args = parser.parse_args() 146 | 147 | data = parse_data(args.infile) 148 | 149 | for action in ['get', 'set', 'delete']: 150 | make_plot(data, action, args.save, args.show, args.limit) 151 | 152 | 153 | if __name__ == '__main__': 154 | main() 155 | -------------------------------------------------------------------------------- /tests/plot_early_recompute.py: -------------------------------------------------------------------------------- 1 | """Early Recomputation Measurements 2 | """ 3 | 4 | import functools as ft 5 | import multiprocessing.pool 6 | import shutil 7 | import threading 8 | import time 9 | 10 | import diskcache as dc 11 | 12 | 13 | def make_timer(times): 14 | """Make a decorator which accumulates (start, end) in `times` for function 15 | calls. 16 | 17 | """ 18 | lock = threading.Lock() 19 | 20 | def timer(func): 21 | @ft.wraps(func) 22 | def wrapper(*args, **kwargs): 23 | start = time.time() 24 | func(*args, **kwargs) 25 | pair = start, time.time() 26 | with lock: 27 | times.append(pair) 28 | 29 | return wrapper 30 | 31 | return timer 32 | 33 | 34 | def make_worker(times, delay=0.2): 35 | """Make a worker which accumulates (start, end) in `times` and sleeps for 36 | `delay` seconds. 37 | 38 | """ 39 | 40 | @make_timer(times) 41 | def worker(): 42 | time.sleep(delay) 43 | 44 | return worker 45 | 46 | 47 | def make_repeater(func, total=10, delay=0.01): 48 | """Make a repeater which calls `func` and sleeps for `delay` seconds 49 | repeatedly until `total` seconds have elapsed. 50 | 51 | """ 52 | 53 | def repeat(num): 54 | start = time.time() 55 | while time.time() - start < total: 56 | func() 57 | time.sleep(delay) 58 | 59 | return repeat 60 | 61 | 62 | def frange(start, stop, step=1e-3): 63 | """Generator for floating point values from `start` to `stop` by `step`.""" 64 | while start < stop: 65 | yield start 66 | start += step 67 | 68 | 69 | def plot(option, filename, cache_times, worker_times): 70 | """Plot concurrent workers and latency.""" 71 | import matplotlib.pyplot as plt 72 | 73 | fig, (workers, latency) = plt.subplots(2, sharex=True) 74 | 75 | fig.suptitle(option) 76 | 77 | changes = [(start, 1) for start, _ in worker_times] 78 | changes.extend((stop, -1) for _, stop in worker_times) 79 | changes.sort() 80 | start = (changes[0][0] - 1e-6, 0) 81 | counts = [start] 82 | 83 | for mark, diff in changes: 84 | # Re-sample between previous and current data point for a nicer-looking 85 | # line plot. 86 | 87 | for step in frange(counts[-1][0], mark): 88 | pair = (step, counts[-1][1]) 89 | counts.append(pair) 90 | 91 | pair = (mark, counts[-1][1] + diff) 92 | counts.append(pair) 93 | 94 | min_x = min(start for start, _ in cache_times) 95 | max_x = max(start for start, _ in cache_times) 96 | for step in frange(counts[-1][0], max_x): 97 | pair = (step, counts[-1][1]) 98 | counts.append(pair) 99 | 100 | x_counts = [x - min_x for x, y in counts] 101 | y_counts = [y for x, y in counts] 102 | 103 | workers.set_title('Concurrency') 104 | workers.set_ylabel('Workers') 105 | workers.set_ylim(0, 11) 106 | workers.plot(x_counts, y_counts) 107 | 108 | latency.set_title('Latency') 109 | latency.set_ylabel('Seconds') 110 | latency.set_ylim(0, 0.5) 111 | latency.set_xlabel('Time') 112 | x_latency = [start - min_x for start, _ in cache_times] 113 | y_latency = [stop - start for start, stop in cache_times] 114 | latency.scatter(x_latency, y_latency) 115 | 116 | plt.savefig(filename) 117 | 118 | 119 | def main(): 120 | shutil.rmtree('/tmp/cache') 121 | cache = dc.Cache('/tmp/cache') 122 | 123 | count = 10 124 | 125 | cache_times = [] 126 | timer = make_timer(cache_times) 127 | 128 | options = { 129 | ('No Caching', 'no-caching.png'): [ 130 | timer, 131 | ], 132 | ('Traditional Caching', 'traditional-caching.png'): [ 133 | timer, 134 | cache.memoize(expire=1), 135 | ], 136 | ('Synchronized Locking', 'synchronized-locking.png'): [ 137 | timer, 138 | cache.memoize(expire=0), 139 | dc.barrier(cache, dc.Lock), 140 | cache.memoize(expire=1), 141 | ], 142 | ('Early Recomputation', 'early-recomputation.png'): [ 143 | timer, 144 | dc.memoize_stampede(cache, expire=1), 145 | ], 146 | ('Early Recomputation (beta=0.5)', 'early-recomputation-05.png'): [ 147 | timer, 148 | dc.memoize_stampede(cache, expire=1, beta=0.5), 149 | ], 150 | ('Early Recomputation (beta=0.3)', 'early-recomputation-03.png'): [ 151 | timer, 152 | dc.memoize_stampede(cache, expire=1, beta=0.3), 153 | ], 154 | } 155 | 156 | for (option, filename), decorators in options.items(): 157 | print('Simulating:', option) 158 | worker_times = [] 159 | worker = make_worker(worker_times) 160 | for decorator in reversed(decorators): 161 | worker = decorator(worker) 162 | 163 | worker() 164 | repeater = make_repeater(worker) 165 | 166 | with multiprocessing.pool.ThreadPool(count) as pool: 167 | pool.map(repeater, [worker] * count) 168 | 169 | plot(option, filename, cache_times, worker_times) 170 | 171 | cache.clear() 172 | cache_times.clear() 173 | 174 | 175 | if __name__ == '__main__': 176 | main() 177 | -------------------------------------------------------------------------------- /tests/settings.py: -------------------------------------------------------------------------------- 1 | """ 2 | Django settings for tests project. 3 | 4 | Generated by 'django-admin startproject' using Django 1.9.1. 5 | 6 | For more information on this file, see 7 | https://docs.djangoproject.com/en/1.9/topics/settings/ 8 | 9 | For the full list of settings and their values, see 10 | https://docs.djangoproject.com/en/1.9/ref/settings/ 11 | """ 12 | 13 | import os 14 | 15 | # Build paths inside the project like this: os.path.join(BASE_DIR, ...) 16 | BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) 17 | 18 | 19 | # Quick-start development settings - unsuitable for production 20 | # See https://docs.djangoproject.com/en/1.9/howto/deployment/checklist/ 21 | 22 | # SECURITY WARNING: keep the secret key used in production secret! 23 | SECRET_KEY = '5bg%^f37a=%mh8(qkq1#)a$e*d-pt*dzox0_39-ywqh=@m(_ii' 24 | 25 | # SECURITY WARNING: don't run with debug turned on in production! 26 | DEBUG = True 27 | 28 | ALLOWED_HOSTS = ['testserver'] 29 | 30 | 31 | # Application definition 32 | 33 | INSTALLED_APPS = [ 34 | 'django.contrib.admin', 35 | 'django.contrib.auth', 36 | 'django.contrib.contenttypes', 37 | 'django.contrib.sessions', 38 | 'django.contrib.messages', 39 | 'django.contrib.staticfiles', 40 | 'tests', 41 | ] 42 | 43 | MIDDLEWARE_CLASSES = [ 44 | 'django.middleware.security.SecurityMiddleware', 45 | 'django.contrib.sessions.middleware.SessionMiddleware', 46 | 'django.middleware.common.CommonMiddleware', 47 | 'django.middleware.csrf.CsrfViewMiddleware', 48 | 'django.contrib.auth.middleware.AuthenticationMiddleware', 49 | 'django.contrib.auth.middleware.SessionAuthenticationMiddleware', 50 | 'django.contrib.messages.middleware.MessageMiddleware', 51 | 'django.middleware.clickjacking.XFrameOptionsMiddleware', 52 | ] 53 | 54 | ROOT_URLCONF = 'project.urls' 55 | 56 | TEMPLATES = [ 57 | { 58 | 'BACKEND': 'django.template.backends.django.DjangoTemplates', 59 | 'DIRS': [], 60 | 'APP_DIRS': True, 61 | 'OPTIONS': { 62 | 'context_processors': [ 63 | 'django.template.context_processors.debug', 64 | 'django.template.context_processors.request', 65 | 'django.contrib.auth.context_processors.auth', 66 | 'django.contrib.messages.context_processors.messages', 67 | ], 68 | }, 69 | }, 70 | ] 71 | 72 | WSGI_APPLICATION = 'project.wsgi.application' 73 | 74 | 75 | # Database 76 | # https://docs.djangoproject.com/en/1.9/ref/settings/#databases 77 | 78 | DATABASES = { 79 | 'default': { 80 | 'ENGINE': 'django.db.backends.sqlite3', 81 | 'NAME': os.path.join(BASE_DIR, 'tests', 'db.sqlite3'), 82 | } 83 | } 84 | 85 | 86 | # Password validation 87 | # https://docs.djangoproject.com/en/1.9/ref/settings/#auth-password-validators 88 | 89 | AUTH_PASSWORD_VALIDATORS = [ 90 | { 91 | 'NAME': 'django.contrib.auth.password_validation.UserAttributeSimilarityValidator', 92 | }, 93 | { 94 | 'NAME': 'django.contrib.auth.password_validation.MinimumLengthValidator', 95 | }, 96 | { 97 | 'NAME': 'django.contrib.auth.password_validation.CommonPasswordValidator', 98 | }, 99 | { 100 | 'NAME': 'django.contrib.auth.password_validation.NumericPasswordValidator', 101 | }, 102 | ] 103 | 104 | 105 | # Internationalization 106 | # https://docs.djangoproject.com/en/1.9/topics/i18n/ 107 | 108 | LANGUAGE_CODE = 'en' 109 | 110 | TIME_ZONE = 'UTC' 111 | 112 | USE_I18N = True 113 | 114 | USE_L10N = False 115 | 116 | USE_TZ = False 117 | 118 | 119 | # Static files (CSS, JavaScript, Images) 120 | # https://docs.djangoproject.com/en/1.9/howto/static-files/ 121 | 122 | STATIC_URL = '/static/' 123 | 124 | 125 | # Caching 126 | 127 | CACHE_DIR = os.path.join(BASE_DIR, '.cache') 128 | 129 | CACHES = { 130 | 'default': { 131 | 'BACKEND': 'diskcache.DjangoCache', 132 | 'LOCATION': CACHE_DIR, 133 | }, 134 | } 135 | -------------------------------------------------------------------------------- /tests/settings_benchmark.py: -------------------------------------------------------------------------------- 1 | from .settings import * # noqa 2 | 3 | CACHES = { 4 | 'default': { 5 | 'BACKEND': 'diskcache.DjangoCache', 6 | 'LOCATION': CACHE_DIR, # noqa 7 | }, 8 | 'memcached': { 9 | 'BACKEND': 'django.core.cache.backends.memcached.PyLibMCCache', 10 | 'LOCATION': '127.0.0.1:11211', 11 | }, 12 | 'redis': { 13 | 'BACKEND': 'django_redis.cache.RedisCache', 14 | 'LOCATION': 'redis://127.0.0.1:6379/1', 15 | 'OPTIONS': { 16 | 'CLIENT_CLASS': 'django_redis.client.DefaultClient', 17 | }, 18 | }, 19 | 'filebased': { 20 | 'BACKEND': 'django.core.cache.backends.filebased.FileBasedCache', 21 | 'LOCATION': '/tmp/django_cache', 22 | 'OPTIONS': { 23 | 'CULL_FREQUENCY': 10, 24 | 'MAX_ENTRIES': 1000, 25 | }, 26 | }, 27 | 'locmem': { 28 | 'BACKEND': 'django.core.cache.backends.locmem.LocMemCache', 29 | 'LOCATION': 'diskcache', 30 | 'OPTIONS': { 31 | 'CULL_FREQUENCY': 10, 32 | 'MAX_ENTRIES': 1000, 33 | }, 34 | }, 35 | 'diskcache': { 36 | 'BACKEND': 'diskcache.DjangoCache', 37 | 'LOCATION': 'tmp', 38 | }, 39 | } 40 | -------------------------------------------------------------------------------- /tests/stress_test_core.py: -------------------------------------------------------------------------------- 1 | """Stress test diskcache.core.Cache.""" 2 | 3 | import collections as co 4 | import multiprocessing as mp 5 | import os 6 | import pickle 7 | import queue 8 | import random 9 | import shutil 10 | import threading 11 | import time 12 | import warnings 13 | 14 | from diskcache import Cache, EmptyDirWarning, Timeout, UnknownFileWarning 15 | 16 | from .utils import display 17 | 18 | OPERATIONS = int(1e4) 19 | GET_AVERAGE = 100 20 | KEY_COUNT = 10 21 | DEL_CHANCE = 0.1 22 | WARMUP = 10 23 | EXPIRE = None 24 | 25 | 26 | def make_keys(): 27 | def make_int(): 28 | return random.randrange(int(1e9)) 29 | 30 | def make_long(): 31 | value = random.randrange(int(1e9)) 32 | return value << 64 33 | 34 | def make_unicode(): 35 | word_size = random.randint(1, 26) 36 | word = ''.join(random.sample('abcdefghijklmnopqrstuvwxyz', word_size)) 37 | size = random.randint(1, int(200 / 13)) 38 | return word * size 39 | 40 | def make_bytes(): 41 | word_size = random.randint(1, 26) 42 | word = ''.join( 43 | random.sample('abcdefghijklmnopqrstuvwxyz', word_size) 44 | ).encode('utf-8') 45 | size = random.randint(1, int(200 / 13)) 46 | return word * size 47 | 48 | def make_float(): 49 | return random.random() 50 | 51 | def make_object(): 52 | return (make_float(),) * random.randint(1, 20) 53 | 54 | funcs = [ 55 | make_int, 56 | make_long, 57 | make_unicode, 58 | make_bytes, 59 | make_float, 60 | make_object, 61 | ] 62 | 63 | while True: 64 | func = random.choice(funcs) 65 | yield func() 66 | 67 | 68 | def make_vals(): 69 | def make_int(): 70 | return random.randrange(int(1e9)) 71 | 72 | def make_long(): 73 | value = random.randrange(int(1e9)) 74 | return value << 64 75 | 76 | def make_unicode(): 77 | word_size = random.randint(1, 26) 78 | word = ''.join(random.sample('abcdefghijklmnopqrstuvwxyz', word_size)) 79 | size = random.randint(1, int(2**16 / 13)) 80 | return word * size 81 | 82 | def make_bytes(): 83 | word_size = random.randint(1, 26) 84 | word = ''.join( 85 | random.sample('abcdefghijklmnopqrstuvwxyz', word_size) 86 | ).encode('utf-8') 87 | size = random.randint(1, int(2**16 / 13)) 88 | return word * size 89 | 90 | def make_float(): 91 | return random.random() 92 | 93 | def make_object(): 94 | return [make_float()] * random.randint(1, int(2e3)) 95 | 96 | funcs = [ 97 | make_int, 98 | make_long, 99 | make_unicode, 100 | make_bytes, 101 | make_float, 102 | make_object, 103 | ] 104 | 105 | while True: 106 | func = random.choice(funcs) 107 | yield func() 108 | 109 | 110 | def key_ops(): 111 | keys = make_keys() 112 | vals = make_vals() 113 | 114 | key = next(keys) 115 | 116 | while True: 117 | value = next(vals) 118 | yield 'set', key, value 119 | for _ in range(int(random.expovariate(1.0 / GET_AVERAGE))): 120 | yield 'get', key, value 121 | if random.random() < DEL_CHANCE: 122 | yield 'delete', key, None 123 | 124 | 125 | def all_ops(): 126 | keys = [key_ops() for _ in range(KEY_COUNT)] 127 | 128 | for _ in range(OPERATIONS): 129 | ops = random.choice(keys) 130 | yield next(ops) 131 | 132 | 133 | def worker(queue, eviction_policy, processes, threads): 134 | timings = co.defaultdict(list) 135 | cache = Cache('tmp', eviction_policy=eviction_policy) 136 | 137 | for index, (action, key, value) in enumerate(iter(queue.get, None)): 138 | start = time.time() 139 | 140 | try: 141 | if action == 'set': 142 | cache.set(key, value, expire=EXPIRE) 143 | elif action == 'get': 144 | result = cache.get(key) 145 | else: 146 | assert action == 'delete' 147 | cache.delete(key) 148 | except Timeout: 149 | miss = True 150 | else: 151 | miss = False 152 | 153 | stop = time.time() 154 | 155 | if ( 156 | action == 'get' 157 | and processes == 1 158 | and threads == 1 159 | and EXPIRE is None 160 | ): 161 | assert result == value 162 | 163 | if index > WARMUP: 164 | delta = stop - start 165 | timings[action].append(delta) 166 | if miss: 167 | timings[action + '-miss'].append(delta) 168 | 169 | queue.put(timings) 170 | 171 | cache.close() 172 | 173 | 174 | def dispatch(num, eviction_policy, processes, threads): 175 | with open('input-%s.pkl' % num, 'rb') as reader: 176 | process_queue = pickle.load(reader) 177 | 178 | thread_queues = [queue.Queue() for _ in range(threads)] 179 | subthreads = [ 180 | threading.Thread( 181 | target=worker, 182 | args=(thread_queue, eviction_policy, processes, threads), 183 | ) 184 | for thread_queue in thread_queues 185 | ] 186 | 187 | for index, triplet in enumerate(process_queue): 188 | thread_queue = thread_queues[index % threads] 189 | thread_queue.put(triplet) 190 | 191 | for thread_queue in thread_queues: 192 | thread_queue.put(None) 193 | 194 | # start = time.time() 195 | 196 | for thread in subthreads: 197 | thread.start() 198 | 199 | for thread in subthreads: 200 | thread.join() 201 | 202 | # stop = time.time() 203 | 204 | timings = co.defaultdict(list) 205 | 206 | for thread_queue in thread_queues: 207 | data = thread_queue.get() 208 | for key in data: 209 | timings[key].extend(data[key]) 210 | 211 | with open('output-%s.pkl' % num, 'wb') as writer: 212 | pickle.dump(timings, writer, protocol=2) 213 | 214 | 215 | def percentile(sequence, percent): 216 | if not sequence: 217 | return None 218 | 219 | values = sorted(sequence) 220 | 221 | if percent == 0: 222 | return values[0] 223 | 224 | pos = int(len(values) * percent) - 1 225 | 226 | return values[pos] 227 | 228 | 229 | def stress_test( 230 | create=True, 231 | delete=True, 232 | eviction_policy='least-recently-stored', 233 | processes=1, 234 | threads=1, 235 | ): 236 | shutil.rmtree('tmp', ignore_errors=True) 237 | 238 | if processes == 1: 239 | # Use threads. 240 | func = threading.Thread 241 | else: 242 | func = mp.Process 243 | 244 | subprocs = [ 245 | func(target=dispatch, args=(num, eviction_policy, processes, threads)) 246 | for num in range(processes) 247 | ] 248 | 249 | if create: 250 | operations = list(all_ops()) 251 | process_queue = [[] for _ in range(processes)] 252 | 253 | for index, ops in enumerate(operations): 254 | process_queue[index % processes].append(ops) 255 | 256 | for num in range(processes): 257 | with open('input-%s.pkl' % num, 'wb') as writer: 258 | pickle.dump(process_queue[num], writer, protocol=2) 259 | 260 | for process in subprocs: 261 | process.start() 262 | 263 | for process in subprocs: 264 | process.join() 265 | 266 | with Cache('tmp') as cache: 267 | warnings.simplefilter('error') 268 | warnings.simplefilter('ignore', category=UnknownFileWarning) 269 | warnings.simplefilter('ignore', category=EmptyDirWarning) 270 | cache.check() 271 | 272 | timings = co.defaultdict(list) 273 | 274 | for num in range(processes): 275 | with open('output-%s.pkl' % num, 'rb') as reader: 276 | data = pickle.load(reader) 277 | for key in data: 278 | timings[key] += data[key] 279 | 280 | if delete: 281 | for num in range(processes): 282 | os.remove('input-%s.pkl' % num) 283 | os.remove('output-%s.pkl' % num) 284 | 285 | display(eviction_policy, timings) 286 | 287 | shutil.rmtree('tmp', ignore_errors=True) 288 | 289 | 290 | def stress_test_lru(): 291 | """Stress test least-recently-used eviction policy.""" 292 | stress_test(eviction_policy='least-recently-used') 293 | 294 | 295 | def stress_test_lfu(): 296 | """Stress test least-frequently-used eviction policy.""" 297 | stress_test(eviction_policy='least-frequently-used') 298 | 299 | 300 | def stress_test_none(): 301 | """Stress test 'none' eviction policy.""" 302 | stress_test(eviction_policy='none') 303 | 304 | 305 | def stress_test_mp(): 306 | """Stress test multiple threads and processes.""" 307 | stress_test(processes=4, threads=4) 308 | 309 | 310 | if __name__ == '__main__': 311 | import argparse 312 | 313 | parser = argparse.ArgumentParser( 314 | formatter_class=argparse.ArgumentDefaultsHelpFormatter, 315 | ) 316 | parser.add_argument( 317 | '-n', 318 | '--operations', 319 | type=float, 320 | default=OPERATIONS, 321 | help='Number of operations to perform', 322 | ) 323 | parser.add_argument( 324 | '-g', 325 | '--get-average', 326 | type=float, 327 | default=GET_AVERAGE, 328 | help='Expected value of exponential variate used for GET count', 329 | ) 330 | parser.add_argument( 331 | '-k', 332 | '--key-count', 333 | type=float, 334 | default=KEY_COUNT, 335 | help='Number of unique keys', 336 | ) 337 | parser.add_argument( 338 | '-d', 339 | '--del-chance', 340 | type=float, 341 | default=DEL_CHANCE, 342 | help='Likelihood of a key deletion', 343 | ) 344 | parser.add_argument( 345 | '-w', 346 | '--warmup', 347 | type=float, 348 | default=WARMUP, 349 | help='Number of warmup operations before timings', 350 | ) 351 | parser.add_argument( 352 | '-e', 353 | '--expire', 354 | type=float, 355 | default=EXPIRE, 356 | help='Number of seconds before key expires', 357 | ) 358 | parser.add_argument( 359 | '-t', 360 | '--threads', 361 | type=int, 362 | default=1, 363 | help='Number of threads to start in each process', 364 | ) 365 | parser.add_argument( 366 | '-p', 367 | '--processes', 368 | type=int, 369 | default=1, 370 | help='Number of processes to start', 371 | ) 372 | parser.add_argument( 373 | '-s', 374 | '--seed', 375 | type=int, 376 | default=0, 377 | help='Random seed', 378 | ) 379 | parser.add_argument( 380 | '--no-create', 381 | action='store_false', 382 | dest='create', 383 | help='Do not create operations data', 384 | ) 385 | parser.add_argument( 386 | '--no-delete', 387 | action='store_false', 388 | dest='delete', 389 | help='Do not delete operations data', 390 | ) 391 | parser.add_argument( 392 | '-v', 393 | '--eviction-policy', 394 | type=str, 395 | default='least-recently-stored', 396 | ) 397 | 398 | args = parser.parse_args() 399 | 400 | OPERATIONS = int(args.operations) 401 | GET_AVERAGE = int(args.get_average) 402 | KEY_COUNT = int(args.key_count) 403 | DEL_CHANCE = args.del_chance 404 | WARMUP = int(args.warmup) 405 | EXPIRE = args.expire 406 | 407 | random.seed(args.seed) 408 | 409 | start = time.time() 410 | stress_test( 411 | create=args.create, 412 | delete=args.delete, 413 | eviction_policy=args.eviction_policy, 414 | processes=args.processes, 415 | threads=args.threads, 416 | ) 417 | end = time.time() 418 | print('Total wall clock time: %.3f seconds' % (end - start)) 419 | -------------------------------------------------------------------------------- /tests/stress_test_deque.py: -------------------------------------------------------------------------------- 1 | """Stress test diskcache.persistent.Deque.""" 2 | 3 | import collections as co 4 | import functools as ft 5 | import random 6 | 7 | import diskcache as dc 8 | 9 | OPERATIONS = 1000 10 | SEED = 0 11 | SIZE = 10 12 | 13 | functions = [] 14 | 15 | 16 | def register(function): 17 | functions.append(function) 18 | return function 19 | 20 | 21 | def lencheck(function): 22 | @ft.wraps(function) 23 | def wrapper(sequence, deque): 24 | assert len(sequence) == len(deque) 25 | 26 | if not deque: 27 | return 28 | 29 | function(sequence, deque) 30 | 31 | return wrapper 32 | 33 | 34 | @register 35 | @lencheck 36 | def stress_get(sequence, deque): 37 | index = random.randrange(len(sequence)) 38 | assert sequence[index] == deque[index] 39 | 40 | 41 | @register 42 | @lencheck 43 | def stress_set(sequence, deque): 44 | index = random.randrange(len(sequence)) 45 | value = random.random() 46 | sequence[index] = value 47 | deque[index] = value 48 | 49 | 50 | @register 51 | @lencheck 52 | def stress_del(sequence, deque): 53 | index = random.randrange(len(sequence)) 54 | del sequence[index] 55 | del deque[index] 56 | 57 | 58 | @register 59 | def stress_iadd(sequence, deque): 60 | values = [random.random() for _ in range(5)] 61 | sequence += values 62 | deque += values 63 | 64 | 65 | @register 66 | def stress_iter(sequence, deque): 67 | assert all(alpha == beta for alpha, beta in zip(sequence, deque)) 68 | 69 | 70 | @register 71 | def stress_reversed(sequence, deque): 72 | reversed_sequence = reversed(sequence) 73 | reversed_deque = reversed(deque) 74 | pairs = zip(reversed_sequence, reversed_deque) 75 | assert all(alpha == beta for alpha, beta in pairs) 76 | 77 | 78 | @register 79 | def stress_append(sequence, deque): 80 | value = random.random() 81 | sequence.append(value) 82 | deque.append(value) 83 | 84 | 85 | @register 86 | def stress_appendleft(sequence, deque): 87 | value = random.random() 88 | sequence.appendleft(value) 89 | deque.appendleft(value) 90 | 91 | 92 | @register 93 | @lencheck 94 | def stress_pop(sequence, deque): 95 | assert sequence.pop() == deque.pop() 96 | 97 | 98 | register(stress_pop) 99 | register(stress_pop) 100 | 101 | 102 | @register 103 | @lencheck 104 | def stress_popleft(sequence, deque): 105 | assert sequence.popleft() == deque.popleft() 106 | 107 | 108 | register(stress_popleft) 109 | register(stress_popleft) 110 | 111 | 112 | @register 113 | def stress_reverse(sequence, deque): 114 | sequence.reverse() 115 | deque.reverse() 116 | assert all(alpha == beta for alpha, beta in zip(sequence, deque)) 117 | 118 | 119 | @register 120 | @lencheck 121 | def stress_rotate(sequence, deque): 122 | assert len(sequence) == len(deque) 123 | steps = random.randrange(len(deque)) 124 | sequence.rotate(steps) 125 | deque.rotate(steps) 126 | assert all(alpha == beta for alpha, beta in zip(sequence, deque)) 127 | 128 | 129 | def stress(sequence, deque): 130 | for count in range(OPERATIONS): 131 | function = random.choice(functions) 132 | function(sequence, deque) 133 | 134 | if count % 100 == 0: 135 | print('\r', len(sequence), ' ' * 7, end='') 136 | 137 | print() 138 | 139 | 140 | def test(): 141 | random.seed(SEED) 142 | sequence = co.deque(range(SIZE)) 143 | deque = dc.Deque(range(SIZE)) 144 | stress(sequence, deque) 145 | assert all(alpha == beta for alpha, beta in zip(sequence, deque)) 146 | 147 | 148 | if __name__ == '__main__': 149 | test() 150 | -------------------------------------------------------------------------------- /tests/stress_test_deque_mp.py: -------------------------------------------------------------------------------- 1 | """Stress test diskcache.persistent.Deque.""" 2 | 3 | import itertools as it 4 | import multiprocessing as mp 5 | import random 6 | import time 7 | 8 | import diskcache as dc 9 | 10 | OPERATIONS = 1000 11 | SEED = 0 12 | SIZE = 10 13 | 14 | functions = [] 15 | 16 | 17 | def register(function): 18 | functions.append(function) 19 | return function 20 | 21 | 22 | @register 23 | def stress_get(deque): 24 | index = random.randrange(max(1, len(deque))) 25 | 26 | try: 27 | deque[index] 28 | except IndexError: 29 | pass 30 | 31 | 32 | @register 33 | def stress_set(deque): 34 | index = random.randrange(max(1, len(deque))) 35 | value = random.random() 36 | 37 | try: 38 | deque[index] = value 39 | except IndexError: 40 | pass 41 | 42 | 43 | @register 44 | def stress_del(deque): 45 | index = random.randrange(max(1, len(deque))) 46 | 47 | try: 48 | del deque[index] 49 | except IndexError: 50 | pass 51 | 52 | 53 | @register 54 | def stress_iadd(deque): 55 | values = [random.random() for _ in range(5)] 56 | deque += values 57 | 58 | 59 | @register 60 | def stress_append(deque): 61 | value = random.random() 62 | deque.append(value) 63 | 64 | 65 | @register 66 | def stress_appendleft(deque): 67 | value = random.random() 68 | deque.appendleft(value) 69 | 70 | 71 | @register 72 | def stress_pop(deque): 73 | try: 74 | deque.pop() 75 | except IndexError: 76 | pass 77 | 78 | 79 | @register 80 | def stress_popleft(deque): 81 | try: 82 | deque.popleft() 83 | except IndexError: 84 | pass 85 | 86 | 87 | @register 88 | def stress_reverse(deque): 89 | deque.reverse() 90 | 91 | 92 | @register 93 | def stress_rotate(deque): 94 | steps = random.randrange(max(1, len(deque))) 95 | deque.rotate(steps) 96 | 97 | 98 | def stress(seed, deque): 99 | random.seed(seed) 100 | for count in range(OPERATIONS): 101 | if len(deque) > 100: 102 | function = random.choice([stress_pop, stress_popleft]) 103 | else: 104 | function = random.choice(functions) 105 | function(deque) 106 | 107 | 108 | def test(status=False): 109 | random.seed(SEED) 110 | deque = dc.Deque(range(SIZE)) 111 | processes = [] 112 | 113 | for count in range(8): 114 | process = mp.Process(target=stress, args=(SEED + count, deque)) 115 | process.start() 116 | processes.append(process) 117 | 118 | for value in it.count(): 119 | time.sleep(1) 120 | 121 | if status: 122 | print('\r', value, 's', len(deque), 'items', ' ' * 20, end='') 123 | 124 | if all(not process.is_alive() for process in processes): 125 | break 126 | 127 | if status: 128 | print('') 129 | 130 | assert all(process.exitcode == 0 for process in processes) 131 | 132 | 133 | if __name__ == '__main__': 134 | test(status=True) 135 | -------------------------------------------------------------------------------- /tests/stress_test_fanout.py: -------------------------------------------------------------------------------- 1 | """Stress test diskcache.core.Cache.""" 2 | 3 | import multiprocessing as mp 4 | import os 5 | import pickle 6 | import queue 7 | import random 8 | import shutil 9 | import threading 10 | import time 11 | import warnings 12 | 13 | from diskcache import EmptyDirWarning, FanoutCache, UnknownFileWarning 14 | 15 | from .utils import display 16 | 17 | OPERATIONS = int(1e4) 18 | GET_AVERAGE = 100 19 | KEY_COUNT = 10 20 | DEL_CHANCE = 0.1 21 | WARMUP = 10 22 | EXPIRE = None 23 | 24 | 25 | def make_keys(): 26 | def make_int(): 27 | return random.randrange(int(1e9)) 28 | 29 | def make_long(): 30 | value = random.randrange(int(1e9)) 31 | return value << 64 32 | 33 | def make_unicode(): 34 | word_size = random.randint(1, 26) 35 | word = ''.join(random.sample('abcdefghijklmnopqrstuvwxyz', word_size)) 36 | size = random.randint(1, int(200 / 13)) 37 | return word * size 38 | 39 | def make_bytes(): 40 | word_size = random.randint(1, 26) 41 | word = ''.join( 42 | random.sample('abcdefghijklmnopqrstuvwxyz', word_size) 43 | ).encode('utf-8') 44 | size = random.randint(1, int(200 / 13)) 45 | return word * size 46 | 47 | def make_float(): 48 | return random.random() 49 | 50 | def make_object(): 51 | return (make_float(),) * random.randint(1, 20) 52 | 53 | funcs = [ 54 | make_int, 55 | make_long, 56 | make_unicode, 57 | make_bytes, 58 | make_float, 59 | make_object, 60 | ] 61 | 62 | while True: 63 | func = random.choice(funcs) 64 | yield func() 65 | 66 | 67 | def make_vals(): 68 | def make_int(): 69 | return random.randrange(int(1e9)) 70 | 71 | def make_long(): 72 | value = random.randrange(int(1e9)) 73 | return value << 64 74 | 75 | def make_unicode(): 76 | word_size = random.randint(1, 26) 77 | word = ''.join(random.sample('abcdefghijklmnopqrstuvwxyz', word_size)) 78 | size = random.randint(1, int(2**16 / 13)) 79 | return word * size 80 | 81 | def make_bytes(): 82 | word_size = random.randint(1, 26) 83 | word = ''.join( 84 | random.sample('abcdefghijklmnopqrstuvwxyz', word_size) 85 | ).encode('utf-8') 86 | size = random.randint(1, int(2**16 / 13)) 87 | return word * size 88 | 89 | def make_float(): 90 | return random.random() 91 | 92 | def make_object(): 93 | return [make_float()] * random.randint(1, int(2e3)) 94 | 95 | funcs = [ 96 | make_int, 97 | make_long, 98 | make_unicode, 99 | make_bytes, 100 | make_float, 101 | make_object, 102 | ] 103 | 104 | while True: 105 | func = random.choice(funcs) 106 | yield func() 107 | 108 | 109 | def key_ops(): 110 | keys = make_keys() 111 | vals = make_vals() 112 | 113 | key = next(keys) 114 | 115 | while True: 116 | value = next(vals) 117 | yield 'set', key, value 118 | for _ in range(int(random.expovariate(1.0 / GET_AVERAGE))): 119 | yield 'get', key, value 120 | if random.random() < DEL_CHANCE: 121 | yield 'delete', key, None 122 | 123 | 124 | def all_ops(): 125 | keys = [key_ops() for _ in range(KEY_COUNT)] 126 | 127 | for _ in range(OPERATIONS): 128 | ops = random.choice(keys) 129 | yield next(ops) 130 | 131 | 132 | def worker(queue, eviction_policy, processes, threads): 133 | timings = {'get': [], 'set': [], 'delete': []} 134 | cache = FanoutCache('tmp', eviction_policy=eviction_policy) 135 | 136 | for index, (action, key, value) in enumerate(iter(queue.get, None)): 137 | start = time.time() 138 | 139 | if action == 'set': 140 | cache.set(key, value, expire=EXPIRE) 141 | elif action == 'get': 142 | result = cache.get(key) 143 | else: 144 | assert action == 'delete' 145 | cache.delete(key) 146 | 147 | stop = time.time() 148 | 149 | if ( 150 | action == 'get' 151 | and processes == 1 152 | and threads == 1 153 | and EXPIRE is None 154 | ): 155 | assert result == value 156 | 157 | if index > WARMUP: 158 | timings[action].append(stop - start) 159 | 160 | queue.put(timings) 161 | 162 | cache.close() 163 | 164 | 165 | def dispatch(num, eviction_policy, processes, threads): 166 | with open('input-%s.pkl' % num, 'rb') as reader: 167 | process_queue = pickle.load(reader) 168 | 169 | thread_queues = [queue.Queue() for _ in range(threads)] 170 | subthreads = [ 171 | threading.Thread( 172 | target=worker, 173 | args=(thread_queue, eviction_policy, processes, threads), 174 | ) 175 | for thread_queue in thread_queues 176 | ] 177 | 178 | for index, triplet in enumerate(process_queue): 179 | thread_queue = thread_queues[index % threads] 180 | thread_queue.put(triplet) 181 | 182 | for thread_queue in thread_queues: 183 | thread_queue.put(None) 184 | 185 | start = time.time() 186 | 187 | for thread in subthreads: 188 | thread.start() 189 | 190 | for thread in subthreads: 191 | thread.join() 192 | 193 | stop = time.time() 194 | 195 | timings = {'get': [], 'set': [], 'delete': [], 'self': (stop - start)} 196 | 197 | for thread_queue in thread_queues: 198 | data = thread_queue.get() 199 | for key in data: 200 | timings[key].extend(data[key]) 201 | 202 | with open('output-%s.pkl' % num, 'wb') as writer: 203 | pickle.dump(timings, writer, protocol=2) 204 | 205 | 206 | def percentile(sequence, percent): 207 | if not sequence: 208 | return None 209 | 210 | values = sorted(sequence) 211 | 212 | if percent == 0: 213 | return values[0] 214 | 215 | pos = int(len(values) * percent) - 1 216 | 217 | return values[pos] 218 | 219 | 220 | def stress_test( 221 | create=True, 222 | delete=True, 223 | eviction_policy='least-recently-stored', 224 | processes=1, 225 | threads=1, 226 | ): 227 | shutil.rmtree('tmp', ignore_errors=True) 228 | 229 | if processes == 1: 230 | # Use threads. 231 | func = threading.Thread 232 | else: 233 | func = mp.Process 234 | 235 | subprocs = [ 236 | func(target=dispatch, args=(num, eviction_policy, processes, threads)) 237 | for num in range(processes) 238 | ] 239 | 240 | if create: 241 | operations = list(all_ops()) 242 | process_queue = [[] for _ in range(processes)] 243 | 244 | for index, ops in enumerate(operations): 245 | process_queue[index % processes].append(ops) 246 | 247 | for num in range(processes): 248 | with open('input-%s.pkl' % num, 'wb') as writer: 249 | pickle.dump(process_queue[num], writer, protocol=2) 250 | 251 | for process in subprocs: 252 | process.start() 253 | 254 | for process in subprocs: 255 | process.join() 256 | 257 | with FanoutCache('tmp') as cache: 258 | warnings.simplefilter('error') 259 | warnings.simplefilter('ignore', category=UnknownFileWarning) 260 | warnings.simplefilter('ignore', category=EmptyDirWarning) 261 | cache.check() 262 | 263 | timings = {'get': [], 'set': [], 'delete': [], 'self': 0.0} 264 | 265 | for num in range(processes): 266 | with open('output-%s.pkl' % num, 'rb') as reader: 267 | data = pickle.load(reader) 268 | for key in data: 269 | timings[key] += data[key] 270 | 271 | if delete: 272 | for num in range(processes): 273 | os.remove('input-%s.pkl' % num) 274 | os.remove('output-%s.pkl' % num) 275 | 276 | display(eviction_policy, timings) 277 | 278 | shutil.rmtree('tmp', ignore_errors=True) 279 | 280 | 281 | def stress_test_lru(): 282 | """Stress test least-recently-used eviction policy.""" 283 | stress_test(eviction_policy='least-recently-used') 284 | 285 | 286 | def stress_test_lfu(): 287 | """Stress test least-frequently-used eviction policy.""" 288 | stress_test(eviction_policy='least-frequently-used') 289 | 290 | 291 | def stress_test_none(): 292 | """Stress test 'none' eviction policy.""" 293 | stress_test(eviction_policy='none') 294 | 295 | 296 | def stress_test_mp(): 297 | """Stress test multiple threads and processes.""" 298 | stress_test(processes=4, threads=4) 299 | 300 | 301 | if __name__ == '__main__': 302 | import argparse 303 | 304 | parser = argparse.ArgumentParser( 305 | formatter_class=argparse.ArgumentDefaultsHelpFormatter, 306 | ) 307 | parser.add_argument( 308 | '-n', 309 | '--operations', 310 | type=float, 311 | default=OPERATIONS, 312 | help='Number of operations to perform', 313 | ) 314 | parser.add_argument( 315 | '-g', 316 | '--get-average', 317 | type=float, 318 | default=GET_AVERAGE, 319 | help='Expected value of exponential variate used for GET count', 320 | ) 321 | parser.add_argument( 322 | '-k', 323 | '--key-count', 324 | type=float, 325 | default=KEY_COUNT, 326 | help='Number of unique keys', 327 | ) 328 | parser.add_argument( 329 | '-d', 330 | '--del-chance', 331 | type=float, 332 | default=DEL_CHANCE, 333 | help='Likelihood of a key deletion', 334 | ) 335 | parser.add_argument( 336 | '-w', 337 | '--warmup', 338 | type=float, 339 | default=WARMUP, 340 | help='Number of warmup operations before timings', 341 | ) 342 | parser.add_argument( 343 | '-e', 344 | '--expire', 345 | type=float, 346 | default=EXPIRE, 347 | help='Number of seconds before key expires', 348 | ) 349 | parser.add_argument( 350 | '-t', 351 | '--threads', 352 | type=int, 353 | default=1, 354 | help='Number of threads to start in each process', 355 | ) 356 | parser.add_argument( 357 | '-p', 358 | '--processes', 359 | type=int, 360 | default=1, 361 | help='Number of processes to start', 362 | ) 363 | parser.add_argument( 364 | '-s', 365 | '--seed', 366 | type=int, 367 | default=0, 368 | help='Random seed', 369 | ) 370 | parser.add_argument( 371 | '--no-create', 372 | action='store_false', 373 | dest='create', 374 | help='Do not create operations data', 375 | ) 376 | parser.add_argument( 377 | '--no-delete', 378 | action='store_false', 379 | dest='delete', 380 | help='Do not delete operations data', 381 | ) 382 | parser.add_argument( 383 | '-v', 384 | '--eviction-policy', 385 | type=str, 386 | default='least-recently-stored', 387 | ) 388 | 389 | args = parser.parse_args() 390 | 391 | OPERATIONS = int(args.operations) 392 | GET_AVERAGE = int(args.get_average) 393 | KEY_COUNT = int(args.key_count) 394 | DEL_CHANCE = args.del_chance 395 | WARMUP = int(args.warmup) 396 | EXPIRE = args.expire 397 | 398 | random.seed(args.seed) 399 | 400 | start = time.time() 401 | stress_test( 402 | create=args.create, 403 | delete=args.delete, 404 | eviction_policy=args.eviction_policy, 405 | processes=args.processes, 406 | threads=args.threads, 407 | ) 408 | end = time.time() 409 | print('Total wall clock time: %.3f seconds' % (end - start)) 410 | -------------------------------------------------------------------------------- /tests/stress_test_index.py: -------------------------------------------------------------------------------- 1 | """Stress test diskcache.persistent.Index.""" 2 | 3 | import collections as co 4 | import itertools as it 5 | import random 6 | 7 | import diskcache as dc 8 | 9 | KEYS = 100 10 | OPERATIONS = 25000 11 | SEED = 0 12 | 13 | functions = [] 14 | 15 | 16 | def register(function): 17 | functions.append(function) 18 | return function 19 | 20 | 21 | @register 22 | def stress_get(mapping, index): 23 | key = random.randrange(KEYS) 24 | assert mapping.get(key, None) == index.get(key, None) 25 | 26 | 27 | @register 28 | def stress_set(mapping, index): 29 | key = random.randrange(KEYS) 30 | value = random.random() 31 | mapping[key] = value 32 | index[key] = value 33 | 34 | 35 | register(stress_set) 36 | register(stress_set) 37 | register(stress_set) 38 | 39 | 40 | @register 41 | def stress_pop(mapping, index): 42 | key = random.randrange(KEYS) 43 | assert mapping.pop(key, None) == index.pop(key, None) 44 | 45 | 46 | @register 47 | def stress_popitem(mapping, index): 48 | if len(mapping) == len(index) == 0: 49 | return 50 | elif random.randrange(2): 51 | assert mapping.popitem() == index.popitem() 52 | else: 53 | assert mapping.popitem(last=False) == index.popitem(last=False) 54 | 55 | 56 | @register 57 | def stress_iter(mapping, index): 58 | iterator = it.islice(zip(mapping, index), 5) 59 | assert all(alpha == beta for alpha, beta in iterator) 60 | 61 | 62 | @register 63 | def stress_reversed(mapping, index): 64 | reversed_mapping = reversed(mapping) 65 | reversed_index = reversed(index) 66 | pairs = it.islice(zip(reversed_mapping, reversed_index), 5) 67 | assert all(alpha == beta for alpha, beta in pairs) 68 | 69 | 70 | @register 71 | def stress_len(mapping, index): 72 | assert len(mapping) == len(index) 73 | 74 | 75 | def stress(mapping, index): 76 | for count in range(OPERATIONS): 77 | function = random.choice(functions) 78 | function(mapping, index) 79 | 80 | if count % 1000 == 0: 81 | print('\r', len(mapping), ' ' * 7, end='') 82 | 83 | print() 84 | 85 | 86 | def test(): 87 | random.seed(SEED) 88 | mapping = co.OrderedDict(enumerate(range(KEYS))) 89 | index = dc.Index(enumerate(range(KEYS))) 90 | stress(mapping, index) 91 | assert mapping == index 92 | 93 | 94 | if __name__ == '__main__': 95 | test() 96 | -------------------------------------------------------------------------------- /tests/stress_test_index_mp.py: -------------------------------------------------------------------------------- 1 | """Stress test diskcache.persistent.Index.""" 2 | 3 | import itertools as it 4 | import multiprocessing as mp 5 | import random 6 | import time 7 | 8 | import diskcache as dc 9 | 10 | KEYS = 100 11 | OPERATIONS = 10000 12 | SEED = 0 13 | 14 | functions = [] 15 | 16 | 17 | def register(function): 18 | functions.append(function) 19 | return function 20 | 21 | 22 | @register 23 | def stress_get(index): 24 | key = random.randrange(KEYS) 25 | index.get(key, None) 26 | 27 | 28 | @register 29 | def stress_set(index): 30 | key = random.randrange(KEYS) 31 | value = random.random() 32 | index[key] = value 33 | 34 | 35 | register(stress_set) 36 | register(stress_set) 37 | register(stress_set) 38 | 39 | 40 | @register 41 | def stress_del(index): 42 | key = random.randrange(KEYS) 43 | 44 | try: 45 | del index[key] 46 | except KeyError: 47 | pass 48 | 49 | 50 | @register 51 | def stress_pop(index): 52 | key = random.randrange(KEYS) 53 | index.pop(key, None) 54 | 55 | 56 | @register 57 | def stress_popitem(index): 58 | try: 59 | if random.randrange(2): 60 | index.popitem() 61 | else: 62 | index.popitem(last=False) 63 | except KeyError: 64 | pass 65 | 66 | 67 | @register 68 | def stress_iter(index): 69 | iterator = it.islice(index, 5) 70 | 71 | for key in iterator: 72 | pass 73 | 74 | 75 | @register 76 | def stress_reversed(index): 77 | iterator = it.islice(reversed(index), 5) 78 | 79 | for key in iterator: 80 | pass 81 | 82 | 83 | @register 84 | def stress_len(index): 85 | len(index) 86 | 87 | 88 | def stress(seed, index): 89 | random.seed(seed) 90 | for count in range(OPERATIONS): 91 | function = random.choice(functions) 92 | function(index) 93 | 94 | 95 | def test(status=False): 96 | random.seed(SEED) 97 | index = dc.Index(enumerate(range(KEYS))) 98 | processes = [] 99 | 100 | for count in range(8): 101 | process = mp.Process(target=stress, args=(SEED + count, index)) 102 | process.start() 103 | processes.append(process) 104 | 105 | for value in it.count(): 106 | time.sleep(1) 107 | 108 | if status: 109 | print('\r', value, 's', len(index), 'keys', ' ' * 20, end='') 110 | 111 | if all(not process.is_alive() for process in processes): 112 | break 113 | 114 | if status: 115 | print('') 116 | 117 | assert all(process.exitcode == 0 for process in processes) 118 | 119 | 120 | if __name__ == '__main__': 121 | test(status=True) 122 | -------------------------------------------------------------------------------- /tests/test_deque.py: -------------------------------------------------------------------------------- 1 | """Test diskcache.persistent.Deque.""" 2 | 3 | import pickle 4 | import shutil 5 | import tempfile 6 | from unittest import mock 7 | 8 | import pytest 9 | 10 | import diskcache as dc 11 | from diskcache.core import ENOVAL 12 | 13 | 14 | def rmdir(directory): 15 | try: 16 | shutil.rmtree(directory) 17 | except OSError: 18 | pass 19 | 20 | 21 | @pytest.fixture 22 | def deque(): 23 | deque = dc.Deque() 24 | yield deque 25 | rmdir(deque.directory) 26 | 27 | 28 | def test_init(): 29 | directory = tempfile.mkdtemp() 30 | sequence = list('abcde') 31 | deque = dc.Deque(sequence, None) 32 | 33 | assert deque == sequence 34 | 35 | rmdir(deque.directory) 36 | del deque 37 | 38 | rmdir(directory) 39 | deque = dc.Deque(sequence, directory) 40 | 41 | assert deque.directory == directory 42 | assert deque == sequence 43 | 44 | other = dc.Deque(directory=directory) 45 | 46 | assert other == deque 47 | 48 | del deque 49 | del other 50 | rmdir(directory) 51 | 52 | 53 | def test_getsetdel(deque): 54 | sequence = list('abcde') 55 | assert len(deque) == 0 56 | 57 | for key in sequence: 58 | deque.append(key) 59 | 60 | assert len(deque) == len(sequence) 61 | 62 | for index in range(len(sequence)): 63 | assert deque[index] == sequence[index] 64 | 65 | for index in range(len(sequence)): 66 | deque[index] = index 67 | 68 | for index in range(len(sequence)): 69 | assert deque[index] == index 70 | 71 | for index in range(len(sequence)): 72 | if index % 2: 73 | del deque[-1] 74 | else: 75 | del deque[0] 76 | 77 | assert len(deque) == 0 78 | 79 | 80 | def test_append(deque): 81 | deque.maxlen = 3 82 | for item in 'abcde': 83 | deque.append(item) 84 | assert deque == 'cde' 85 | 86 | 87 | def test_appendleft(deque): 88 | deque.maxlen = 3 89 | for item in 'abcde': 90 | deque.appendleft(item) 91 | assert deque == 'edc' 92 | 93 | 94 | def test_index_positive(deque): 95 | cache = mock.MagicMock() 96 | cache.__len__.return_value = 3 97 | cache.iterkeys.return_value = ['a', 'b', 'c'] 98 | cache.__getitem__.side_effect = [KeyError, 101, 102] 99 | with mock.patch.object(deque, '_cache', cache): 100 | assert deque[0] == 101 101 | 102 | 103 | def test_index_negative(deque): 104 | cache = mock.MagicMock() 105 | cache.__len__.return_value = 3 106 | cache.iterkeys.return_value = ['c', 'b', 'a'] 107 | cache.__getitem__.side_effect = [KeyError, 101, 100] 108 | with mock.patch.object(deque, '_cache', cache): 109 | assert deque[-1] == 101 110 | 111 | 112 | def test_index_out_of_range(deque): 113 | cache = mock.MagicMock() 114 | cache.__len__.return_value = 3 115 | cache.iterkeys.return_value = ['a', 'b', 'c'] 116 | cache.__getitem__.side_effect = [KeyError] * 3 117 | with mock.patch.object(deque, '_cache', cache): 118 | with pytest.raises(IndexError): 119 | deque[0] 120 | 121 | 122 | def test_iter_keyerror(deque): 123 | cache = mock.MagicMock() 124 | cache.iterkeys.return_value = ['a', 'b', 'c'] 125 | cache.__getitem__.side_effect = [KeyError, 101, 102] 126 | with mock.patch.object(deque, '_cache', cache): 127 | assert list(iter(deque)) == [101, 102] 128 | 129 | 130 | def test_reversed(deque): 131 | sequence = list('abcde') 132 | deque += sequence 133 | assert list(reversed(deque)) == list(reversed(sequence)) 134 | 135 | 136 | def test_reversed_keyerror(deque): 137 | cache = mock.MagicMock() 138 | cache.iterkeys.return_value = ['c', 'b', 'a'] 139 | cache.__getitem__.side_effect = [KeyError, 101, 100] 140 | with mock.patch.object(deque, '_cache', cache): 141 | assert list(reversed(deque)) == [101, 100] 142 | 143 | 144 | def test_state(deque): 145 | sequence = list('abcde') 146 | deque.extend(sequence) 147 | assert deque == sequence 148 | deque.maxlen = 3 149 | assert list(deque) == sequence[-3:] 150 | state = pickle.dumps(deque) 151 | values = pickle.loads(state) 152 | assert values == sequence[-3:] 153 | assert values.maxlen == 3 154 | 155 | 156 | def test_compare(deque): 157 | assert not (deque == {}) 158 | assert not (deque == [0]) 159 | assert deque != [1] 160 | deque.append(0) 161 | assert deque <= [0] 162 | assert deque <= [1] 163 | 164 | 165 | def test_indexerror_negative(deque): 166 | with pytest.raises(IndexError): 167 | deque[-1] 168 | 169 | 170 | def test_indexerror(deque): 171 | with pytest.raises(IndexError): 172 | deque[0] 173 | 174 | 175 | def test_repr(): 176 | directory = tempfile.mkdtemp() 177 | deque = dc.Deque(directory=directory) 178 | assert repr(deque) == 'Deque(directory=%r)' % directory 179 | 180 | 181 | def test_copy(deque): 182 | sequence = list('abcde') 183 | deque.extend(sequence) 184 | temp = deque.copy() 185 | assert deque == sequence 186 | assert temp == sequence 187 | 188 | 189 | def test_count(deque): 190 | deque += 'abbcccddddeeeee' 191 | 192 | for index, value in enumerate('abcde', 1): 193 | assert deque.count(value) == index 194 | 195 | 196 | def test_extend(deque): 197 | sequence = list('abcde') 198 | deque.extend(sequence) 199 | assert deque == sequence 200 | 201 | 202 | def test_extendleft(deque): 203 | sequence = list('abcde') 204 | deque.extendleft(sequence) 205 | assert deque == list(reversed(sequence)) 206 | 207 | 208 | def test_pop(deque): 209 | sequence = list('abcde') 210 | deque.extend(sequence) 211 | 212 | while sequence: 213 | assert deque.pop() == sequence.pop() 214 | 215 | 216 | def test_pop_indexerror(deque): 217 | with pytest.raises(IndexError): 218 | deque.pop() 219 | 220 | 221 | def test_popleft(deque): 222 | sequence = list('abcde') 223 | deque.extend(sequence) 224 | 225 | while sequence: 226 | value = sequence[0] 227 | assert deque.popleft() == value 228 | del sequence[0] 229 | 230 | 231 | def test_popleft_indexerror(deque): 232 | with pytest.raises(IndexError): 233 | deque.popleft() 234 | 235 | 236 | def test_remove(deque): 237 | deque.extend('abaca') 238 | deque.remove('a') 239 | assert deque == 'baca' 240 | deque.remove('a') 241 | assert deque == 'bca' 242 | deque.remove('a') 243 | assert deque == 'bc' 244 | 245 | 246 | def test_remove_valueerror(deque): 247 | with pytest.raises(ValueError): 248 | deque.remove(0) 249 | 250 | 251 | def test_remove_keyerror(deque): 252 | cache = mock.MagicMock() 253 | cache.iterkeys.return_value = ['a', 'b', 'c'] 254 | cache.__getitem__.side_effect = [KeyError, 100, 100] 255 | cache.__delitem__.side_effect = [KeyError, None] 256 | with mock.patch.object(deque, '_cache', cache): 257 | deque.remove(100) 258 | 259 | 260 | def test_reverse(deque): 261 | deque += 'abcde' 262 | deque.reverse() 263 | assert deque == 'edcba' 264 | 265 | 266 | def test_rotate_typeerror(deque): 267 | with pytest.raises(TypeError): 268 | deque.rotate(0.5) 269 | 270 | 271 | def test_rotate(deque): 272 | deque.rotate(1) 273 | deque.rotate(-1) 274 | deque += 'abcde' 275 | deque.rotate(3) 276 | assert deque == 'cdeab' 277 | 278 | 279 | def test_rotate_negative(deque): 280 | deque += 'abcde' 281 | deque.rotate(-2) 282 | assert deque == 'cdeab' 283 | 284 | 285 | def test_rotate_indexerror(deque): 286 | deque += 'abc' 287 | 288 | cache = mock.MagicMock() 289 | cache.__len__.return_value = 3 290 | cache.pull.side_effect = [(None, ENOVAL)] 291 | 292 | with mock.patch.object(deque, '_cache', cache): 293 | deque.rotate(1) 294 | 295 | 296 | def test_rotate_indexerror_negative(deque): 297 | deque += 'abc' 298 | 299 | cache = mock.MagicMock() 300 | cache.__len__.return_value = 3 301 | cache.pull.side_effect = [(None, ENOVAL)] 302 | 303 | with mock.patch.object(deque, '_cache', cache): 304 | deque.rotate(-1) 305 | 306 | 307 | def test_peek(deque): 308 | value = b'x' * 100_000 309 | deque.append(value) 310 | assert len(deque) == 1 311 | assert deque.peek() == value 312 | assert len(deque) == 1 313 | assert deque.peek() == value 314 | assert len(deque) == 1 315 | -------------------------------------------------------------------------------- /tests/test_doctest.py: -------------------------------------------------------------------------------- 1 | import doctest 2 | 3 | import diskcache.core 4 | import diskcache.djangocache 5 | import diskcache.fanout 6 | import diskcache.persistent 7 | import diskcache.recipes 8 | 9 | 10 | def test_core(): 11 | failures, _ = doctest.testmod(diskcache.core) 12 | assert failures == 0 13 | 14 | 15 | def test_djangocache(): 16 | failures, _ = doctest.testmod(diskcache.djangocache) 17 | assert failures == 0 18 | 19 | 20 | def test_fanout(): 21 | failures, _ = doctest.testmod(diskcache.fanout) 22 | assert failures == 0 23 | 24 | 25 | def test_persistent(): 26 | failures, _ = doctest.testmod(diskcache.persistent) 27 | assert failures == 0 28 | 29 | 30 | def test_recipes(): 31 | failures, _ = doctest.testmod(diskcache.recipes) 32 | assert failures == 0 33 | 34 | 35 | def test_tutorial(): 36 | failures, _ = doctest.testfile('../docs/tutorial.rst') 37 | assert failures == 0 38 | -------------------------------------------------------------------------------- /tests/test_index.py: -------------------------------------------------------------------------------- 1 | """Test diskcache.persistent.Index.""" 2 | 3 | import pickle 4 | import shutil 5 | import tempfile 6 | 7 | import pytest 8 | 9 | import diskcache as dc 10 | 11 | 12 | def rmdir(directory): 13 | try: 14 | shutil.rmtree(directory) 15 | except OSError: 16 | pass 17 | 18 | 19 | @pytest.fixture 20 | def index(): 21 | index = dc.Index() 22 | yield index 23 | rmdir(index.directory) 24 | 25 | 26 | def test_init(): 27 | directory = tempfile.mkdtemp() 28 | mapping = {'a': 5, 'b': 4, 'c': 3, 'd': 2, 'e': 1} 29 | index = dc.Index(None, mapping) 30 | 31 | assert index == mapping 32 | 33 | rmdir(index.directory) 34 | del index 35 | 36 | rmdir(directory) 37 | index = dc.Index(directory, mapping) 38 | 39 | assert index.directory == directory 40 | assert index == mapping 41 | 42 | other = dc.Index(directory) 43 | 44 | assert other == index 45 | 46 | del index 47 | del other 48 | rmdir(directory) 49 | index = dc.Index(directory, mapping.items()) 50 | 51 | assert index == mapping 52 | 53 | del index 54 | rmdir(directory) 55 | index = dc.Index(directory, a=5, b=4, c=3, d=2, e=1) 56 | 57 | assert index == mapping 58 | 59 | 60 | def test_getsetdel(index): 61 | letters = 'abcde' 62 | assert len(index) == 0 63 | 64 | for num, key in enumerate(letters): 65 | index[key] = num 66 | 67 | for num, key in enumerate(letters): 68 | assert index[key] == num 69 | 70 | for key in letters: 71 | del index[key] 72 | 73 | assert len(index) == 0 74 | 75 | 76 | def test_pop(index): 77 | letters = 'abcde' 78 | assert len(index) == 0 79 | 80 | for num, key in enumerate(letters): 81 | index[key] = num 82 | 83 | assert index.pop('a') == 0 84 | assert index.pop('c') == 2 85 | assert index.pop('e') == 4 86 | assert index.pop('b') == 1 87 | assert index.pop('d') == 3 88 | assert len(index) == 0 89 | 90 | 91 | def test_pop_keyerror(index): 92 | with pytest.raises(KeyError): 93 | index.pop('a') 94 | 95 | 96 | def test_popitem(index): 97 | letters = 'abcde' 98 | 99 | for num, key in enumerate(letters): 100 | index[key] = num 101 | 102 | assert index.popitem() == ('e', 4) 103 | assert index.popitem(last=True) == ('d', 3) 104 | assert index.popitem(last=False) == ('a', 0) 105 | assert len(index) == 2 106 | 107 | 108 | def test_popitem_keyerror(index): 109 | with pytest.raises(KeyError): 110 | index.popitem() 111 | 112 | 113 | def test_setdefault(index): 114 | assert index.setdefault('a', 0) == 0 115 | assert index.setdefault('a', 1) == 0 116 | 117 | 118 | def test_iter(index): 119 | letters = 'abcde' 120 | 121 | for num, key in enumerate(letters): 122 | index[key] = num 123 | 124 | for num, key in enumerate(index): 125 | assert index[key] == num 126 | 127 | 128 | def test_reversed(index): 129 | letters = 'abcde' 130 | 131 | for num, key in enumerate(letters): 132 | index[key] = num 133 | 134 | for num, key in enumerate(reversed(index)): 135 | assert index[key] == (len(letters) - num - 1) 136 | 137 | 138 | def test_state(index): 139 | mapping = {'a': 5, 'b': 4, 'c': 3, 'd': 2, 'e': 1} 140 | index.update(mapping) 141 | assert index == mapping 142 | state = pickle.dumps(index) 143 | values = pickle.loads(state) 144 | assert values == mapping 145 | 146 | 147 | def test_memoize(index): 148 | count = 1000 149 | 150 | def fibiter(num): 151 | alpha, beta = 0, 1 152 | 153 | for _ in range(num): 154 | alpha, beta = beta, alpha + beta 155 | 156 | return alpha 157 | 158 | @index.memoize() 159 | def fibrec(num): 160 | if num == 0: 161 | return 0 162 | elif num == 1: 163 | return 1 164 | else: 165 | return fibrec(num - 1) + fibrec(num - 2) 166 | 167 | index._cache.stats(enable=True) 168 | 169 | for value in range(count): 170 | assert fibrec(value) == fibiter(value) 171 | 172 | hits1, misses1 = index._cache.stats() 173 | 174 | for value in range(count): 175 | assert fibrec(value) == fibiter(value) 176 | 177 | hits2, misses2 = index._cache.stats() 178 | 179 | assert hits2 == (hits1 + count) 180 | assert misses2 == misses1 181 | 182 | 183 | def test_repr(index): 184 | assert repr(index).startswith('Index(') 185 | -------------------------------------------------------------------------------- /tests/test_recipes.py: -------------------------------------------------------------------------------- 1 | """Test diskcache.recipes.""" 2 | 3 | import shutil 4 | import threading 5 | import time 6 | 7 | import pytest 8 | 9 | import diskcache as dc 10 | 11 | 12 | @pytest.fixture 13 | def cache(): 14 | with dc.Cache() as cache: 15 | yield cache 16 | shutil.rmtree(cache.directory, ignore_errors=True) 17 | 18 | 19 | def test_averager(cache): 20 | nums = dc.Averager(cache, 'nums') 21 | for i in range(10): 22 | nums.add(i) 23 | assert nums.get() == 4.5 24 | assert nums.pop() == 4.5 25 | for i in range(20): 26 | nums.add(i) 27 | assert nums.get() == 9.5 28 | assert nums.pop() == 9.5 29 | 30 | 31 | def test_lock(cache): 32 | state = {'num': 0} 33 | lock = dc.Lock(cache, 'demo') 34 | 35 | def worker(): 36 | state['num'] += 1 37 | with lock: 38 | assert lock.locked() 39 | state['num'] += 1 40 | time.sleep(0.1) 41 | 42 | with lock: 43 | thread = threading.Thread(target=worker) 44 | thread.start() 45 | time.sleep(0.1) 46 | assert state['num'] == 1 47 | thread.join() 48 | assert state['num'] == 2 49 | 50 | 51 | def test_rlock(cache): 52 | state = {'num': 0} 53 | rlock = dc.RLock(cache, 'demo') 54 | 55 | def worker(): 56 | state['num'] += 1 57 | with rlock: 58 | with rlock: 59 | state['num'] += 1 60 | time.sleep(0.1) 61 | 62 | with rlock: 63 | thread = threading.Thread(target=worker) 64 | thread.start() 65 | time.sleep(0.1) 66 | assert state['num'] == 1 67 | thread.join() 68 | assert state['num'] == 2 69 | 70 | 71 | def test_semaphore(cache): 72 | state = {'num': 0} 73 | semaphore = dc.BoundedSemaphore(cache, 'demo', value=3) 74 | 75 | def worker(): 76 | state['num'] += 1 77 | with semaphore: 78 | state['num'] += 1 79 | time.sleep(0.1) 80 | 81 | semaphore.acquire() 82 | semaphore.acquire() 83 | with semaphore: 84 | thread = threading.Thread(target=worker) 85 | thread.start() 86 | time.sleep(0.1) 87 | assert state['num'] == 1 88 | thread.join() 89 | assert state['num'] == 2 90 | semaphore.release() 91 | semaphore.release() 92 | 93 | 94 | def test_memoize_stampede(cache): 95 | state = {'num': 0} 96 | 97 | @dc.memoize_stampede(cache, 0.1) 98 | def worker(num): 99 | time.sleep(0.01) 100 | state['num'] += 1 101 | return num 102 | 103 | start = time.time() 104 | while (time.time() - start) < 1: 105 | worker(100) 106 | assert state['num'] > 0 107 | -------------------------------------------------------------------------------- /tests/timings_core_p1.txt: -------------------------------------------------------------------------------- 1 | 2 | ========= ========= ========= ========= ========= ========= ========= ========= 3 | Timings for diskcache.Cache 4 | ------------------------------------------------------------------------------- 5 | Action Count Miss Median P90 P99 Max Total 6 | ========= ========= ========= ========= ========= ========= ========= ========= 7 | get 89115 8714 19.073us 25.749us 32.902us 115.395us 1.800s 8 | set 8941 0 114.918us 137.091us 241.041us 4.946ms 1.242s 9 | delete 943 111 87.976us 149.202us 219.824us 4.795ms 120.738ms 10 | Total 98999 3.163s 11 | ========= ========= ========= ========= ========= ========= ========= ========= 12 | 13 | 14 | ========= ========= ========= ========= ========= ========= ========= ========= 15 | Timings for diskcache.FanoutCache(shards=4, timeout=1.0) 16 | ------------------------------------------------------------------------------- 17 | Action Count Miss Median P90 P99 Max Total 18 | ========= ========= ========= ========= ========= ========= ========= ========= 19 | get 89115 8714 21.935us 27.180us 36.001us 129.938us 2.028s 20 | set 8941 0 118.017us 170.946us 270.844us 5.129ms 1.307s 21 | delete 943 111 91.791us 153.780us 231.981us 4.883ms 119.732ms 22 | Total 98999 3.455s 23 | ========= ========= ========= ========= ========= ========= ========= ========= 24 | 25 | 26 | ========= ========= ========= ========= ========= ========= ========= ========= 27 | Timings for diskcache.FanoutCache(shards=8, timeout=0.010) 28 | ------------------------------------------------------------------------------- 29 | Action Count Miss Median P90 P99 Max Total 30 | ========= ========= ========= ========= ========= ========= ========= ========= 31 | get 89115 8714 20.981us 27.180us 35.286us 128.031us 2.023s 32 | set 8941 0 116.825us 175.953us 269.175us 5.248ms 1.367s 33 | delete 943 111 91.791us 158.787us 235.345us 4.634ms 106.991ms 34 | Total 98999 3.496s 35 | ========= ========= ========= ========= ========= ========= ========= ========= 36 | 37 | 38 | ========= ========= ========= ========= ========= ========= ========= ========= 39 | Timings for pylibmc.Client 40 | ------------------------------------------------------------------------------- 41 | Action Count Miss Median P90 P99 Max Total 42 | ========= ========= ========= ========= ========= ========= ========= ========= 43 | get 89115 8714 42.915us 62.227us 79.155us 166.178us 3.826s 44 | set 8941 0 44.107us 63.896us 82.254us 121.832us 396.247ms 45 | delete 943 111 41.962us 60.797us 75.817us 92.983us 39.570ms 46 | Total 98999 4.262s 47 | ========= ========= ========= ========= ========= ========= ========= ========= 48 | 49 | 50 | ========= ========= ========= ========= ========= ========= ========= ========= 51 | Timings for redis.StrictRedis 52 | ------------------------------------------------------------------------------- 53 | Action Count Miss Median P90 P99 Max Total 54 | ========= ========= ========= ========= ========= ========= ========= ========= 55 | get 89115 8714 86.069us 101.089us 144.005us 805.140us 7.722s 56 | set 8941 0 89.169us 104.189us 146.866us 408.173us 800.963ms 57 | delete 943 111 86.069us 99.182us 149.012us 327.826us 80.976ms 58 | Total 98999 8.604s 59 | ========= ========= ========= ========= ========= ========= ========= ========= 60 | -------------------------------------------------------------------------------- /tests/timings_core_p8.txt: -------------------------------------------------------------------------------- 1 | 2 | ========= ========= ========= ========= ========= ========= ========= ========= 3 | Timings for diskcache.Cache 4 | ------------------------------------------------------------------------------- 5 | Action Count Miss Median P90 P99 Max Total 6 | ========= ========= ========= ========= ========= ========= ========= ========= 7 | get 712612 69147 20.027us 28.133us 45.061us 2.792ms 15.838s 8 | set 71464 0 129.700us 1.388ms 35.831ms 1.342s 160.708s 9 | delete 7916 769 97.036us 1.340ms 21.605ms 837.003ms 13.551s 10 | Total 791992 194.943s 11 | ========= ========= ========= ========= ========= ========= ========= ========= 12 | 13 | 14 | ========= ========= ========= ========= ========= ========= ========= ========= 15 | Timings for diskcache.FanoutCache(shards=4, timeout=1.0) 16 | ------------------------------------------------------------------------------- 17 | Action Count Miss Median P90 P99 Max Total 18 | ========= ========= ========= ========= ========= ========= ========= ========= 19 | get 712612 70432 27.895us 48.876us 77.963us 12.945ms 25.443s 20 | set 71464 0 176.907us 1.416ms 9.385ms 183.997ms 65.606s 21 | delete 7916 747 132.084us 1.354ms 9.272ms 86.189ms 6.576s 22 | Total 791992 98.248s 23 | ========= ========= ========= ========= ========= ========= ========= ========= 24 | 25 | 26 | ========= ========= ========= ========= ========= ========= ========= ========= 27 | Timings for diskcache.FanoutCache(shards=8, timeout=0.010) 28 | ------------------------------------------------------------------------------- 29 | Action Count Miss Median P90 P99 Max Total 30 | ========= ========= ========= ========= ========= ========= ========= ========= 31 | get 712612 69622 41.962us 71.049us 96.083us 16.896ms 36.145s 32 | set 71464 39 257.969us 1.456ms 7.132ms 19.774ms 46.160s 33 | delete 7916 773 190.020us 1.377ms 5.927ms 12.939ms 4.442s 34 | Total 791992 86.799s 35 | ========= ========= ========= ========= ========= ========= ========= ========= 36 | 37 | 38 | ========= ========= ========= ========= ========= ========= ========= ========= 39 | Timings for pylibmc.Client 40 | ------------------------------------------------------------------------------- 41 | Action Count Miss Median P90 P99 Max Total 42 | ========= ========= ========= ========= ========= ========= ========= ========= 43 | get 712612 70517 95.844us 113.010us 131.130us 604.153us 69.024s 44 | set 71464 0 97.036us 114.918us 136.137us 608.921us 7.024s 45 | delete 7916 817 94.891us 112.057us 132.084us 604.153us 760.844ms 46 | Total 791992 76.809s 47 | ========= ========= ========= ========= ========= ========= ========= ========= 48 | 49 | 50 | ========= ========= ========= ========= ========= ========= ========= ========= 51 | Timings for redis.StrictRedis 52 | ------------------------------------------------------------------------------- 53 | Action Count Miss Median P90 P99 Max Total 54 | ========= ========= ========= ========= ========= ========= ========= ========= 55 | get 712612 70540 187.874us 244.141us 305.891us 1.416ms 138.516s 56 | set 71464 0 192.881us 249.147us 311.136us 1.363ms 14.246s 57 | delete 7916 825 185.966us 242.949us 305.176us 519.276us 1.525s 58 | Total 791992 154.287s 59 | ========= ========= ========= ========= ========= ========= ========= ========= 60 | -------------------------------------------------------------------------------- /tests/timings_djangocache.txt: -------------------------------------------------------------------------------- 1 | 2 | ========= ========= ========= ========= ========= ========= ========= ========= 3 | Timings for locmem 4 | ------------------------------------------------------------------------------- 5 | Action Count Miss Median P90 P99 Max Total 6 | ========= ========= ========= ========= ========= ========= ========= ========= 7 | get 712770 141094 34.809us 47.922us 55.075us 15.140ms 26.159s 8 | set 71249 0 38.862us 41.008us 59.843us 8.094ms 2.725s 9 | delete 7973 0 32.902us 35.048us 51.260us 2.963ms 257.951ms 10 | Total 791992 29.142s 11 | ========= ========= ========= ========= ========= ========= ========= ========= 12 | 13 | 14 | ========= ========= ========= ========= ========= ========= ========= ========= 15 | Timings for memcached 16 | ------------------------------------------------------------------------------- 17 | Action Count Miss Median P90 P99 Max Total 18 | ========= ========= ========= ========= ========= ========= ========= ========= 19 | get 712770 71873 102.043us 118.017us 182.867us 2.054ms 73.453s 20 | set 71249 0 104.904us 123.978us 182.152us 836.849us 7.592s 21 | delete 7973 0 98.944us 114.918us 176.191us 473.261us 795.398ms 22 | Total 791992 81.841s 23 | ========= ========= ========= ========= ========= ========= ========= ========= 24 | 25 | 26 | ========= ========= ========= ========= ========= ========= ========= ========= 27 | Timings for redis 28 | ------------------------------------------------------------------------------- 29 | Action Count Miss Median P90 P99 Max Total 30 | ========= ========= ========= ========= ========= ========= ========= ========= 31 | get 712770 71694 214.100us 267.982us 358.820us 1.556ms 155.709s 32 | set 71249 0 230.789us 284.195us 377.178us 1.462ms 16.764s 33 | delete 7973 790 195.742us 251.770us 345.945us 1.105ms 1.596s 34 | Total 791992 174.069s 35 | ========= ========= ========= ========= ========= ========= ========= ========= 36 | 37 | 38 | ========= ========= ========= ========= ========= ========= ========= ========= 39 | Timings for diskcache 40 | ------------------------------------------------------------------------------- 41 | Action Count Miss Median P90 P99 Max Total 42 | ========= ========= ========= ========= ========= ========= ========= ========= 43 | get 712770 70909 55.075us 82.016us 106.096us 36.816ms 44.088s 44 | set 71249 0 303.984us 1.489ms 6.499ms 39.687ms 49.088s 45 | delete 7973 0 228.882us 1.409ms 5.769ms 24.750ms 4.755s 46 | Total 791992 98.465s 47 | ========= ========= ========= ========= ========= ========= ========= ========= 48 | 49 | 50 | ========= ========= ========= ========= ========= ========= ========= ========= 51 | Timings for filebased 52 | ------------------------------------------------------------------------------- 53 | Action Count Miss Median P90 P99 Max Total 54 | ========= ========= ========= ========= ========= ========= ========= ========= 55 | get 712792 112290 114.918us 161.171us 444.889us 61.068ms 94.438s 56 | set 71268 0 11.289ms 13.278ms 16.653ms 108.282ms 809.448s 57 | delete 7977 0 432.014us 675.917us 5.785ms 55.249ms 3.652s 58 | Total 791992 907.537s 59 | ========= ========= ========= ========= ========= ========= ========= ========= 60 | -------------------------------------------------------------------------------- /tests/timings_glob.txt: -------------------------------------------------------------------------------- 1 | 2 | ============ ============ 3 | Timings for glob.glob1 4 | ------------------------- 5 | Count Time 6 | ============ ============ 7 | 1 1.602ms 8 | 10 2.213ms 9 | 100 8.946ms 10 | 1000 65.869ms 11 | 10000 604.972ms 12 | 100000 6.450s 13 | ============ ============ 14 | -------------------------------------------------------------------------------- /tests/utils.py: -------------------------------------------------------------------------------- 1 | import os 2 | import subprocess as sp 3 | 4 | 5 | def percentile(sequence, percent): 6 | if not sequence: 7 | return None 8 | 9 | values = sorted(sequence) 10 | 11 | if percent == 0: 12 | return values[0] 13 | 14 | pos = int(len(values) * percent) - 1 15 | 16 | return values[pos] 17 | 18 | 19 | def secs(value): 20 | units = ['s ', 'ms', 'us', 'ns'] 21 | 22 | if value is None: 23 | return ' 0.000ns' 24 | elif value == 0: 25 | return ' 0.000ns' 26 | else: 27 | for unit in units: 28 | if value > 1: 29 | return '%7.3f' % value + unit 30 | else: 31 | value *= 1000 32 | 33 | 34 | def run(*args): 35 | """Run command, print output, and return output.""" 36 | print('utils$', *args) 37 | result = sp.check_output(args) 38 | print(result) 39 | return result.strip() 40 | 41 | 42 | def mount_ramdisk(size, path): 43 | """Mount RAM disk at `path` with `size` in bytes.""" 44 | sectors = size / 512 45 | 46 | os.makedirs(path) 47 | 48 | dev_path = run('hdid', '-nomount', 'ram://%d' % sectors) 49 | run('newfs_hfs', '-v', 'RAMdisk', dev_path) 50 | run('mount', '-o', 'noatime', '-t', 'hfs', dev_path, path) 51 | 52 | return dev_path 53 | 54 | 55 | def unmount_ramdisk(dev_path, path): 56 | """Unmount RAM disk with `dev_path` and `path`.""" 57 | run('umount', path) 58 | run('diskutil', 'eject', dev_path) 59 | run('rm', '-r', path) 60 | 61 | 62 | def display(name, timings): 63 | cols = ('Action', 'Count', 'Miss', 'Median', 'P90', 'P99', 'Max', 'Total') 64 | template = ' '.join(['%9s'] * len(cols)) 65 | 66 | print() 67 | print(' '.join(['=' * 9] * len(cols))) 68 | print('Timings for %s' % name) 69 | print('-'.join(['-' * 9] * len(cols))) 70 | print(template % cols) 71 | print(' '.join(['=' * 9] * len(cols))) 72 | 73 | len_total = sum_total = 0 74 | 75 | for action in ['get', 'set', 'delete']: 76 | values = timings[action] 77 | len_total += len(values) 78 | sum_total += sum(values) 79 | 80 | print( 81 | template 82 | % ( 83 | action, 84 | len(values), 85 | len(timings.get(action + '-miss', [])), 86 | secs(percentile(values, 0.5)), 87 | secs(percentile(values, 0.9)), 88 | secs(percentile(values, 0.99)), 89 | secs(percentile(values, 1.0)), 90 | secs(sum(values)), 91 | ) 92 | ) 93 | 94 | totals = ('Total', len_total, '', '', '', '', '', secs(sum_total)) 95 | print(template % totals) 96 | print(' '.join(['=' * 9] * len(cols))) 97 | print() 98 | -------------------------------------------------------------------------------- /tox.ini: -------------------------------------------------------------------------------- 1 | [tox] 2 | envlist=bluecheck,doc8,docs,isortcheck,flake8,mypy,pylint,rstcheck,py38,py39,py310,py311 3 | skip_missing_interpreters=True 4 | 5 | [testenv] 6 | commands=pytest 7 | deps= 8 | django==4.2.* 9 | pytest 10 | pytest-cov 11 | pytest-django 12 | pytest-xdist 13 | setenv= 14 | DJANGO_SETTINGS_MODULE=tests.settings 15 | PYTHONPATH={toxinidir} 16 | 17 | [testenv:blue] 18 | commands=blue {toxinidir}/setup.py {toxinidir}/diskcache {toxinidir}/tests 19 | deps=blue 20 | 21 | [testenv:bluecheck] 22 | commands=blue --check {toxinidir}/setup.py {toxinidir}/diskcache {toxinidir}/tests 23 | deps=blue 24 | 25 | [testenv:doc8] 26 | commands=doc8 docs --ignore-path docs/_build 27 | deps=doc8 28 | 29 | [testenv:docs] 30 | allowlist_externals=make 31 | changedir=docs 32 | commands=make html 33 | deps= 34 | django==4.2.* 35 | sphinx 36 | 37 | [testenv:flake8] 38 | commands=flake8 {toxinidir}/setup.py {toxinidir}/diskcache {toxinidir}/tests 39 | deps=flake8 40 | 41 | [testenv:isort] 42 | commands=isort {toxinidir}/setup.py {toxinidir}/diskcache {toxinidir}/tests 43 | deps=isort 44 | 45 | [testenv:isortcheck] 46 | commands=isort --check {toxinidir}/setup.py {toxinidir}/diskcache {toxinidir}/tests 47 | deps=isort 48 | 49 | [testenv:mypy] 50 | commands=mypy {toxinidir}/diskcache 51 | deps=mypy 52 | 53 | [testenv:pylint] 54 | commands=pylint {toxinidir}/diskcache 55 | deps= 56 | django==4.2.* 57 | pylint 58 | 59 | [testenv:rstcheck] 60 | commands=rstcheck {toxinidir}/README.rst 61 | deps=rstcheck 62 | 63 | [testenv:uploaddocs] 64 | allowlist_externals=rsync 65 | changedir=docs 66 | commands= 67 | rsync --rsync-path 'sudo -u herokuish rsync' -azP --stats --delete \ 68 | _build/html/ \ 69 | grantjenks:/srv/www/grantjenks.com/public/docs/diskcache/ 70 | 71 | [isort] 72 | multi_line_output = 3 73 | include_trailing_comma = True 74 | force_grid_wrap = 0 75 | use_parentheses = True 76 | ensure_newline_before_comments = True 77 | line_length = 79 78 | 79 | [pytest] 80 | addopts= 81 | -n auto 82 | --cov-branch 83 | --cov-fail-under=98 84 | --cov-report=term-missing 85 | --cov=diskcache 86 | --doctest-glob="*.rst" 87 | --ignore docs/case-study-web-crawler.rst 88 | --ignore docs/sf-python-2017-meetup-talk.rst 89 | --ignore tests/benchmark_core.py 90 | --ignore tests/benchmark_djangocache.py 91 | --ignore tests/benchmark_glob.py 92 | --ignore tests/issue_85.py 93 | --ignore tests/plot.py 94 | 95 | [doc8] 96 | # ignore=D000 97 | 98 | [flake8] 99 | exclude=tests/test_djangocache.py 100 | extend-ignore=E203 101 | max-line-length=120 102 | --------------------------------------------------------------------------------