├── fancy_tests ├── __init__.py └── tests │ ├── __init__.py │ ├── models.py │ ├── templates │ └── home.html │ ├── urls.py │ ├── test_utils.py │ ├── test_command.py │ ├── settings.py │ ├── views.py │ ├── test_memory.py │ └── test_views.py ├── example ├── example │ ├── __init__.py │ ├── app │ │ ├── __init__.py │ │ ├── models.py │ │ ├── templates │ │ │ ├── page.html │ │ │ └── home.html │ │ ├── urls.py │ │ ├── tests.py │ │ └── views.py │ ├── urls.py │ ├── wsgi.py │ └── settings.py └── manage.py ├── fancy_cache ├── management │ ├── __init__.py │ └── commands │ │ ├── __init__.py │ │ └── fancy-urls.py ├── constants.py ├── __init__.py ├── urls.py ├── cache_page.py ├── views.py ├── utils.py ├── templates │ └── fancy-cache │ │ └── home.html ├── memory.py └── middleware.py ├── pyproject.toml ├── requirements.txt ├── .gitignore ├── deploy.sh ├── docs ├── changelog.rst ├── index.rst ├── gettingstarted.rst ├── stats.rst ├── Makefile ├── conf.py └── gettingfancy.rst ├── MANIFEST.in ├── .github └── workflows │ ├── black.yml │ └── ci.yml ├── runtests.py ├── fabfile.py ├── tox.ini ├── LICENSE ├── setup.py └── README.rst /fancy_tests/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /example/example/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /example/example/app/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /fancy_tests/tests/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /fancy_cache/management/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /fancy_cache/management/commands/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /fancy_tests/tests/models.py: -------------------------------------------------------------------------------- 1 | # lonely in here 2 | -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [tool.black] 2 | line-length = 80 3 | -------------------------------------------------------------------------------- /fancy_tests/tests/templates/home.html: -------------------------------------------------------------------------------- 1 | 2 | Random:{{ random_string }} 3 | 4 | -------------------------------------------------------------------------------- /example/example/app/models.py: -------------------------------------------------------------------------------- 1 | from django.db import models 2 | 3 | # Create your models here. 4 | -------------------------------------------------------------------------------- /fancy_cache/constants.py: -------------------------------------------------------------------------------- 1 | REMEMBERED_URLS_KEY = "fancy-urls" 2 | LONG_TIME = 60 * 60 * 24 * 30 3 | -------------------------------------------------------------------------------- /fancy_cache/__init__.py: -------------------------------------------------------------------------------- 1 | from .cache_page import cache_page # NOQA 2 | 3 | __version__ = "1.3.1" 4 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | # Tests 2 | Django>=3.2.0,<4.1 3 | django-nose 4 | fabric 5 | pylibmc 6 | sphinx 7 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.pyc 2 | build/ 3 | fancy_cache.egg-info/ 4 | dist/ 5 | django_fancy_cache.egg-info/ 6 | cover/ 7 | docs/_build/ 8 | .tox/ 9 | -------------------------------------------------------------------------------- /deploy.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -e 3 | 4 | rm -fr dist/* 5 | python setup.py sdist bdist_wheel 6 | twine check dist/* 7 | twine upload dist/* 8 | -------------------------------------------------------------------------------- /docs/changelog.rst: -------------------------------------------------------------------------------- 1 | .. index:: changelog 2 | 3 | .. _changelog-chapter: 4 | 5 | See https://github.com/peterbe/django-fancy-cache#changelog 6 | -------------------------------------------------------------------------------- /fancy_tests/tests/urls.py: -------------------------------------------------------------------------------- 1 | from django.urls import path 2 | from . import views 3 | 4 | 5 | urlpatterns = [path("", views.home, name="home")] 6 | -------------------------------------------------------------------------------- /fancy_cache/urls.py: -------------------------------------------------------------------------------- 1 | from django.urls import path 2 | from . import views 3 | 4 | 5 | urlpatterns = [ 6 | path("", views.home, name="home"), 7 | ] 8 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include LICENSE 2 | include README.rst 3 | recursive-include fancy_cache/templates/fancy-cache *.html 4 | recursive-include fancy_tests/tests/templates *.html 5 | -------------------------------------------------------------------------------- /example/example/urls.py: -------------------------------------------------------------------------------- 1 | from django.urls import include, path 2 | 3 | 4 | urlpatterns = [ 5 | path("", include("example.app.urls")), 6 | path("fancy-cache", include("fancy_cache.urls")), 7 | ] 8 | -------------------------------------------------------------------------------- /example/example/app/templates/page.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | Fancy! Page {{ page }} 6 | 7 | 8 | 9 |

Hello world!

10 |

Just so you know, the sum of the first 25,000,000 is {{ result }}

11 | 12 | 13 | -------------------------------------------------------------------------------- /.github/workflows/black.yml: -------------------------------------------------------------------------------- 1 | --- 2 | name: Run black 3 | 4 | on: [push, pull_request] 5 | 6 | jobs: 7 | black: 8 | runs-on: ubuntu-latest 9 | steps: 10 | - uses: actions/checkout@v2 11 | - uses: actions/setup-python@v2 12 | with: 13 | python-version: '3.x' 14 | - name: install black 15 | run: pip install black==22.3.0 16 | - name: Black 17 | run: black . --check 18 | -------------------------------------------------------------------------------- /example/example/app/urls.py: -------------------------------------------------------------------------------- 1 | from django.urls import path 2 | from . import views 3 | 4 | urlpatterns = [ 5 | path("", views.home, name="home"), 6 | path("page1.html", views.page1, name="page1"), 7 | path("page2.html", views.page2, name="page2"), 8 | path("page3.html", views.page3, name="page3"), 9 | path("page4.html", views.page4, name="page4"), 10 | path("page5.html", views.page5, name="page5"), 11 | ] 12 | -------------------------------------------------------------------------------- /example/example/app/tests.py: -------------------------------------------------------------------------------- 1 | """ 2 | This file demonstrates writing tests using the unittest module. These will pass 3 | when you run "manage.py test". 4 | 5 | Replace this with more appropriate tests for your application. 6 | """ 7 | 8 | from django.test import TestCase 9 | 10 | 11 | class SimpleTest(TestCase): 12 | def test_basic_addition(self): 13 | """ 14 | Tests that 1 + 1 always equals 2. 15 | """ 16 | self.assertEqual(1 + 1, 2) 17 | -------------------------------------------------------------------------------- /example/manage.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | import os 3 | import sys 4 | 5 | # make sure we're running the fancy_cache here and not anything installed 6 | parent = os.path.normpath(os.path.join(__file__, "../..")) 7 | sys.path.insert(0, parent) 8 | 9 | if __name__ == "__main__": 10 | os.environ.setdefault("DJANGO_SETTINGS_MODULE", "example.settings") 11 | 12 | from django.core.management import execute_from_command_line 13 | 14 | execute_from_command_line(sys.argv) 15 | -------------------------------------------------------------------------------- /fancy_cache/cache_page.py: -------------------------------------------------------------------------------- 1 | import typing 2 | 3 | from django.utils.decorators import decorator_from_middleware_with_args 4 | 5 | from .middleware import FancyCacheMiddleware 6 | 7 | 8 | def cache_page( 9 | timeout: typing.Optional[float], 10 | *, 11 | cache: str = None, 12 | key_prefix: str = None, 13 | **kwargs 14 | ) -> typing.Callable: 15 | return decorator_from_middleware_with_args(FancyCacheMiddleware)( 16 | page_timeout=timeout, cache_alias=cache, key_prefix=key_prefix, **kwargs 17 | ) 18 | -------------------------------------------------------------------------------- /fancy_cache/views.py: -------------------------------------------------------------------------------- 1 | from django.shortcuts import render 2 | from django.conf import settings 3 | 4 | from fancy_cache.memory import find_urls 5 | 6 | 7 | def home(request): 8 | data = { 9 | "found": find_urls([]), 10 | "remember_all_urls_setting": getattr( 11 | settings, "FANCY_REMEMBER_ALL_URLS", False 12 | ), 13 | "remember_stats_all_urls_setting": getattr( 14 | settings, "FANCY_REMEMBER_STATS_ALL_URLS", False 15 | ), 16 | } 17 | return render(request, "fancy-cache/home.html", data) 18 | -------------------------------------------------------------------------------- /docs/index.rst: -------------------------------------------------------------------------------- 1 | .. django-fancy-cache documentation master file, created by 2 | sphinx-quickstart on Sun Feb 10 16:18:37 2013. 3 | You can adapt this file completely to your liking, but it should at least 4 | contain the root `toctree` directive. 5 | 6 | Welcome to django-fancy-cache's documentation! 7 | ============================================== 8 | 9 | Contents: 10 | 11 | .. toctree:: 12 | :maxdepth: 2 13 | 14 | gettingstarted 15 | gettingfancy 16 | stats 17 | changelog 18 | 19 | 20 | 21 | 22 | Indices and tables 23 | ================== 24 | 25 | * :ref:`genindex` 26 | * :ref:`modindex` 27 | * :ref:`search` 28 | -------------------------------------------------------------------------------- /runtests.py: -------------------------------------------------------------------------------- 1 | import os 2 | import sys 3 | 4 | import django 5 | from django.test.utils import get_runner 6 | from django.conf import settings 7 | 8 | 9 | def runtests(): 10 | test_dir = os.path.join(os.path.dirname(__file__), "fancy_tests/tests") 11 | sys.path.insert(0, test_dir) 12 | 13 | os.environ["DJANGO_SETTINGS_MODULE"] = "settings" 14 | django.setup() 15 | 16 | TestRunner = get_runner(settings) 17 | test_runner = TestRunner(interactive=False, failfast=False) 18 | failures = test_runner.run_tests(["fancy_tests.tests"]) 19 | 20 | sys.exit(bool(failures)) 21 | 22 | 23 | if __name__ == "__main__": 24 | runtests() 25 | -------------------------------------------------------------------------------- /fancy_cache/utils.py: -------------------------------------------------------------------------------- 1 | import hashlib 2 | import time 3 | import typing 4 | 5 | 6 | def md5(x) -> str: 7 | return hashlib.md5(x.encode("utf-8")).hexdigest() 8 | 9 | 10 | def filter_remembered_urls( 11 | remembered_urls: typing.Dict[str, typing.Tuple[str, int]], 12 | ) -> typing.Dict[str, typing.Tuple[str, int]]: 13 | """ 14 | Filter out any expired URLs from Fancy Cache's remembered urls. 15 | """ 16 | now = int(time.time()) 17 | # TODO: Remove the check for tuple in a future release as it will 18 | # no longer be needed once the new dictionary structure {url: (cache_key, expiration_time)} 19 | # has been implemented. 20 | remembered_urls = { 21 | key: value 22 | for key, value in remembered_urls.items() 23 | if isinstance(value, tuple) and value[1] > now 24 | } 25 | return remembered_urls 26 | -------------------------------------------------------------------------------- /fabfile.py: -------------------------------------------------------------------------------- 1 | """ 2 | This Source Code Form is subject to the terms of the Mozilla Public 3 | License, v. 2.0. If a copy of the MPL was not distributed with this 4 | file, You can obtain one at http://mozilla.org/MPL/2.0/. 5 | """ 6 | import os 7 | 8 | from fabric.api import local 9 | 10 | 11 | ROOT = os.path.abspath(os.path.dirname(__file__)) 12 | os.environ["PYTHONPATH"] = ROOT 13 | 14 | 15 | def _test(extra_args): 16 | """Run test suite.""" 17 | os.environ["DJANGO_SETTINGS_MODULE"] = "fancy_tests.tests.settings" 18 | os.environ["REUSE_DB"] = "0" 19 | 20 | # Add tables and flush DB 21 | local("django-admin.py syncdb --noinput") 22 | local("django-admin.py flush --noinput") 23 | 24 | local("django-admin.py test %s" % extra_args) 25 | 26 | 27 | def test(): 28 | _test("-s") 29 | 30 | 31 | def coverage(): 32 | _test( 33 | "-s --with-coverage --cover-erase --cover-html " 34 | "--cover-package=fancy_cache" 35 | ) 36 | -------------------------------------------------------------------------------- /.github/workflows/ci.yml: -------------------------------------------------------------------------------- 1 | # Based on 2 | # https://pypi.org/project/tox-gh-actions/ 3 | 4 | --- 5 | name: Python 6 | 7 | on: [push, pull_request] 8 | 9 | jobs: 10 | build: 11 | runs-on: ubuntu-latest 12 | strategy: 13 | matrix: 14 | # PyPy disabled per issue #69: 15 | # https://github.com/peterbe/django-fancy-cache/issues/69 16 | # python-version: [3.7, 3.8, 3.9, "3.10", "pypy-3.8"] 17 | python-version: [3.7, 3.8, 3.9, "3.10", "3.11"] 18 | 19 | steps: 20 | - name: apt update 21 | run: sudo apt update 22 | - uses: niden/actions-memcached@v7 23 | - uses: actions/checkout@v2 24 | - name: Set up Python ${{ matrix.python-version }} 25 | uses: actions/setup-python@v2 26 | with: 27 | python-version: ${{ matrix.python-version }} 28 | - name: Install dependencies 29 | run: | 30 | python -m pip install --upgrade pip 31 | pip install tox tox-gh-actions 32 | sudo apt install libmemcached-dev 33 | - name: Test with tox 34 | run: tox -v 35 | -------------------------------------------------------------------------------- /fancy_tests/tests/test_utils.py: -------------------------------------------------------------------------------- 1 | import time 2 | import unittest 3 | 4 | from django.core.cache import cache 5 | 6 | from fancy_cache.constants import REMEMBERED_URLS_KEY 7 | from fancy_cache.utils import filter_remembered_urls 8 | 9 | 10 | class TestUtils(unittest.TestCase): 11 | def setUp(self): 12 | expiration_time = int(time.time()) + 5 13 | self.urls = { 14 | "/page1.html": ("key1", expiration_time), 15 | "/page2.html": ("key2", expiration_time), 16 | "/page3.html?foo=bar": ("key3", expiration_time), 17 | "/page3.html?foo=else": ("key4", expiration_time), 18 | } 19 | for key, value in self.urls.items(): 20 | cache.set(value[0], key) 21 | cache.set(REMEMBERED_URLS_KEY, self.urls, 5) 22 | 23 | def tearDown(self): 24 | cache.clear() 25 | 26 | def test_filter_remembered_urls(self): 27 | remembered_urls = filter_remembered_urls(self.urls) 28 | self.assertDictEqual(remembered_urls, self.urls) 29 | 30 | url = "/page1.html" 31 | self.urls[url] = ("key1", int(time.time()) - 1) 32 | remembered_urls = filter_remembered_urls(self.urls) 33 | self.assertEqual(len(remembered_urls.keys()), len(self.urls.keys()) - 1) 34 | self.assertNotIn(url, remembered_urls.keys()) 35 | -------------------------------------------------------------------------------- /tox.ini: -------------------------------------------------------------------------------- 1 | # Tox (http://tox.testrun.org/) is a tool for running tests 2 | # in multiple virtualenvs. This configuration file will run the 3 | # test suite on all supported python versions. To use it, "pip install tox" 4 | # and then run "tox" from this directory. 5 | 6 | [tox] 7 | envlist = py38-dj32, 8 | py39-dj32, 9 | py310-dj32, 10 | py311-dj32, 11 | py38-dj41, 12 | py39-dj41, 13 | py310-dj41, 14 | py38-dj42, 15 | py39-dj42, 16 | py310-dj42, 17 | py311-dj42 18 | # PyPy disabled per issue #69: 19 | # https://github.com/peterbe/django-fancy-cache/issues/69 20 | # pypy-3-dj22, 21 | # pypy-3-dj31, 22 | # pypy-3-dj32, 23 | # pypy-3-dj40 24 | 25 | [gh-actions] 26 | python = 27 | 3.8: py38 28 | 3.9: py39 29 | 3.10: py310 30 | 3.11: py311 31 | # pypy-3.8: pypy-3 32 | 33 | [testenv] 34 | basepython = 35 | py38: python3.8 36 | py39: python3.9 37 | py310: python3.10 38 | py311: python3.11 39 | # pypy-3: pypy3 40 | 41 | deps = 42 | dj32: Django>=3.2.0, <4.0.0 43 | dj41: Django>=4.1.0, <4.2.0 44 | dj42: Django>=4.2.0, <4.3.0 45 | mock 46 | 47 | usedevelop = true 48 | 49 | commands = pip install -r requirements.txt 50 | {envpython} setup.py test 51 | -------------------------------------------------------------------------------- /example/example/wsgi.py: -------------------------------------------------------------------------------- 1 | """ 2 | WSGI config for example project. 3 | 4 | This module contains the WSGI application used by Django's development server 5 | and any production WSGI deployments. It should expose a module-level variable 6 | named ``application``. Django's ``runserver`` and ``runfcgi`` commands discover 7 | this application via the ``WSGI_APPLICATION`` setting. 8 | 9 | Usually you will have the standard Django WSGI application here, but it also 10 | might make sense to replace the whole Django WSGI application with a custom one 11 | that later delegates to the Django one. For example, you could introduce WSGI 12 | middleware here, or combine a Django application with an application of another 13 | framework. 14 | 15 | """ 16 | import os 17 | 18 | # We defer to a DJANGO_SETTINGS_MODULE already in the environment. This breaks 19 | # if running multiple sites in the same mod_wsgi process. To fix this, use 20 | # mod_wsgi daemon mode with each site in its own daemon process, or use 21 | # os.environ["DJANGO_SETTINGS_MODULE"] = "example.settings" 22 | os.environ.setdefault("DJANGO_SETTINGS_MODULE", "example.settings") 23 | 24 | # This application object is used by any WSGI server configured to use this 25 | # file. This includes Django's development server, if the WSGI_APPLICATION 26 | # setting points here. 27 | from django.core.wsgi import get_wsgi_application 28 | 29 | application = get_wsgi_application() 30 | 31 | # Apply WSGI middleware here. 32 | # from helloworld.wsgi import HelloWorldApplication 33 | # application = HelloWorldApplication(application) 34 | -------------------------------------------------------------------------------- /docs/gettingstarted.rst: -------------------------------------------------------------------------------- 1 | .. index:: gettingstarted 2 | 3 | .. _gettingstarted-chapter: 4 | 5 | Getting started 6 | =============== 7 | 8 | The minimal you need to do is install ``django-fancy-cache`` and add 9 | it to a view. 10 | 11 | Installing is easy:: 12 | 13 | $ pip install django-fancy-cache 14 | 15 | Now, let's assume you have a ``views.py`` that looks like this:: 16 | 17 | from django.shortcuts import render 18 | 19 | def my_view(request): 20 | something_really_slow... 21 | return render(request, 'template.html') 22 | 23 | 24 | What you add is this:: 25 | 26 | from django.shortcuts import render 27 | from fancy_cache import cache_page 28 | 29 | @cache_page(3600) 30 | def my_view(request): 31 | something_really_slow... 32 | return render(request, 'template.html') 33 | 34 | 35 | Fancy cache also works for Class-based Views with the help of [Django's `method_decorator`](https://docs.djangoproject.com/en/dev/topics/class-based-views/intro/#decorating-the-class):: 36 | 37 | from django.utils.decorators import method_decorator 38 | from django.views.generic import TemplateView 39 | from fancy_cache import cache_page 40 | 41 | 42 | class MyView(TemplateView): 43 | template_name = "template.html" 44 | 45 | @method_decorator(cache_page(3600)) 46 | def get(self, request, *args, **kwargs): 47 | return super().get(request, *args, **kwargs) 48 | 49 | 50 | Getting fancy 51 | ------------- 52 | 53 | The above stuff isn't particularly fancy. The next steps is to 54 | start :ref:`gettingfancy-chapter`. 55 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) Peter Bengtsson. 2 | All rights reserved. 3 | 4 | Redistribution and use in source and binary forms, with or without modification, 5 | are permitted provided that the following conditions are met: 6 | 7 | 1. Redistributions of source code must retain the above copyright notice, 8 | this list of conditions and the following disclaimer. 9 | 10 | 2. Redistributions in binary form must reproduce the above copyright 11 | notice, this list of conditions and the following disclaimer in the 12 | documentation and/or other materials provided with the distribution. 13 | 14 | 3. Neither the name of django-fancy-cache nor the names of its contributors 15 | may be used to endorse or promote products derived from this software 16 | without specific prior written permission. 17 | 18 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND 19 | ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 20 | WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 21 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR 22 | ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 23 | (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 24 | LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON 25 | ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 26 | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 27 | SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 | -------------------------------------------------------------------------------- /fancy_tests/tests/test_command.py: -------------------------------------------------------------------------------- 1 | import time 2 | 3 | from django.test import TestCase 4 | from nose.tools import ok_ 5 | from django.core.cache import cache 6 | from django.core.management import call_command 7 | from io import StringIO 8 | from fancy_cache.constants import REMEMBERED_URLS_KEY 9 | 10 | 11 | class TestBaseCommand(TestCase): 12 | def setUp(self): 13 | expiration_time = int(time.time()) + 5 14 | self.urls = { 15 | "/page1.html": ( 16 | "key1", 17 | expiration_time, 18 | ), 19 | "/page2.html": ( 20 | "key2", 21 | expiration_time, 22 | ), 23 | "/page3.html?foo=bar": ( 24 | "key3", 25 | expiration_time, 26 | ), 27 | "/page3.html?foo=else": ( 28 | "key4", 29 | expiration_time, 30 | ), 31 | } 32 | for key, value in self.urls.items(): 33 | cache.set(value[0], key) 34 | cache.set(REMEMBERED_URLS_KEY, self.urls, 5) 35 | 36 | def tearDown(self): 37 | cache.clear() 38 | 39 | def test_fancyurls_command(self): 40 | out = StringIO() 41 | call_command("fancy-urls", verbosity=3, stdout=out) 42 | self.assertIn("4 URLs cached", out.getvalue()) 43 | 44 | def test_purge_command(self): 45 | out = StringIO() 46 | # Note: first call will show 4 URLs, so call again to confirm deletion 47 | call_command("fancy-urls", "--purge") 48 | call_command("fancy-urls", verbosity=3, stdout=out) 49 | self.assertIn("0 URLs cached", out.getvalue()) 50 | -------------------------------------------------------------------------------- /fancy_cache/templates/fancy-cache/home.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 8 | 9 | 10 |

11 | Stats about 12 | django-fancy-cache 13 | usage 14 |

15 | 16 | {% if not remember_all_urls_setting %} 17 |

18 | Note! 19 | You do not have FANCY_REMEMBER_ALL_URLS set in your 20 | settings so cached URLs will not be recorded unless explicitely 21 | set on the cache_page decorator. 22 |

23 | {% else %} 24 | {% if not remember_stats_all_urls_setting %} 25 |

26 | Note! 27 | You do not have FANCY_REMEMBER_STATS_ALL_URLS set in your 28 | settings so statistics about hits and misses will not be recorded. 29 |

30 | {% endif %} 31 | 32 | {% endif %} 33 | 34 | {% for url, cache_key, stats in found %} 35 | {% if forloop.first %} 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | {% endif %} 44 | 45 | 46 | 47 | {% if stats %} 48 | 49 | 50 | {% else %} 51 | 52 | 53 | {% endif %} 54 | 55 | {% if forloop.last %} 56 |
PathCache keyHitsMisses
{{ url }}{{ cache_key }}{{ stats.hits }}{{ stats.misses }}--
57 | {% endif %} 58 | {% endfor %} 59 | 60 | {% if not found %} 61 |

No found URLs in the cache at the moment. 62 |

63 | {% endif %} 64 | 65 | -------------------------------------------------------------------------------- /example/example/app/templates/home.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | Fancy! 6 | 7 | 8 | 9 |

Open each of these pages. You'll notice that they're initially 10 | very slow. That's because they're expensive view functions. 11 | After loading them a second time (within 10 seconds) you'll notice 12 | that they're incredibly fast. 13 |

14 | 15 | 22 |
23 |

This is just for the uber-curious about what's going on. These are the current 24 | remembered URLs:

25 | 26 | {% for url, key, stats in remembered_urls %} 27 | {% if forloop.first %} 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | {% endif %} 36 | 37 | 38 | 39 | {% if stats %} 40 | 41 | 42 | {% else %} 43 | 46 | {% if forloop.last %} 47 |
URLCache keyHitsMisses
{{ url }}{{ key }}{{ stats.hits }}{{ stats.misses }}n/a 44 | {% endif %} 45 |
48 | {% endif %} 49 | {% endfor %} 50 | 51 | 52 | -------------------------------------------------------------------------------- /fancy_tests/tests/settings.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | BASE_DIR = os.path.dirname(os.path.dirname(__file__)) 4 | 5 | SECRET_KEY = "anything" 6 | 7 | ALLOWED_HOSTS = ("testserver",) 8 | 9 | DATABASES = { 10 | "default": { 11 | "NAME": ":memory:", 12 | "ENGINE": "django.db.backends.sqlite3", 13 | } 14 | } 15 | 16 | CACHES = { 17 | "default": { 18 | "BACKEND": "django.core.cache.backends.locmem.LocMemCache", 19 | "LOCATION": "unique-snowflake", 20 | }, 21 | "second_backend": { 22 | "BACKEND": "django.core.cache.backends.locmem.LocMemCache", 23 | "LOCATION": "unique-snowflake", 24 | }, 25 | "dummy_backend": { 26 | "BACKEND": "django.core.cache.backends.dummy.DummyCache", 27 | "LOCATION": "unique-snowflake", 28 | }, 29 | "memcached_backend": { 30 | "BACKEND": "django.core.cache.backends.memcached.PyLibMCCache", 31 | "LOCATION": "127.0.0.1:11211", 32 | "OPTIONS": {"behaviors": {"cas": True}}, 33 | }, 34 | } 35 | 36 | INSTALLED_APPS = [ 37 | "fancy_cache", 38 | "fancy_tests.tests", 39 | "django.contrib.auth", 40 | "django.contrib.contenttypes", 41 | ] 42 | 43 | 44 | ROOT_URLCONF = "fancy_tests.tests.urls" 45 | 46 | 47 | TEMPLATES = [ 48 | { 49 | "BACKEND": "django.template.backends.django.DjangoTemplates", 50 | "DIRS": [], 51 | "APP_DIRS": True, 52 | "OPTIONS": { 53 | "context_processors": [ 54 | "django.template.context_processors.debug", 55 | "django.template.context_processors.request", 56 | "django.contrib.auth.context_processors.auth", 57 | "django.contrib.messages.context_processors.messages", 58 | ], 59 | }, 60 | }, 61 | ] 62 | -------------------------------------------------------------------------------- /fancy_tests/tests/views.py: -------------------------------------------------------------------------------- 1 | import uuid 2 | from django.shortcuts import render 3 | from django.views.decorators.cache import never_cache 4 | from fancy_cache import cache_page 5 | 6 | 7 | def _view(request): 8 | random_string = uuid.uuid4().hex 9 | return render(request, "home.html", dict(random_string=random_string)) 10 | 11 | 12 | @cache_page(60) 13 | def home(request): 14 | return _view(request) 15 | 16 | 17 | def prefixer1(request): 18 | if request.META.get("AUTH_USER"): 19 | # disable 20 | return None 21 | return "a_key" 22 | 23 | 24 | @cache_page(60, key_prefix=prefixer1) 25 | def home2(request): 26 | return _view(request) 27 | 28 | 29 | def post_processor1(response, request): 30 | assert "In your HTML" not in response.content.decode("utf8") 31 | response.content += ("In your HTML:%s" % uuid.uuid4().hex).encode("utf8") 32 | return response 33 | 34 | 35 | @cache_page(60, key_prefix=prefixer1, post_process_response=post_processor1) 36 | def home3(request): 37 | return _view(request) 38 | 39 | 40 | @cache_page( 41 | 60, key_prefix=prefixer1, post_process_response_always=post_processor1 42 | ) 43 | def home4(request): 44 | return _view(request) 45 | 46 | 47 | @cache_page(60, only_get_keys=["foo", "bar"]) 48 | def home5(request): 49 | return _view(request) 50 | 51 | 52 | @cache_page(60, forget_get_keys=["bar"]) 53 | def home5bis(request): 54 | return _view(request) 55 | 56 | 57 | @cache_page(60, remember_stats_all_urls=True, remember_all_urls=True) 58 | def home6(request): 59 | return _view(request) 60 | 61 | 62 | @cache_page(60, cache="second_backend") 63 | def home7(request): 64 | return _view(request) 65 | 66 | 67 | @cache_page(60, cache="dummy_backend", remember_all_urls=True) 68 | def home8(request): 69 | return _view(request) 70 | 71 | 72 | @never_cache 73 | @cache_page(60, remember_stats_all_urls=True, remember_all_urls=True) 74 | def home9(request): 75 | return _view(request) 76 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | import codecs 2 | import os 3 | import re 4 | 5 | 6 | # Prevent spurious errors during `python setup.py test`, a la 7 | # http://www.eby-sarna.com/pipermail/peak/2010-May/003357.html: 8 | try: 9 | import multiprocessing 10 | except ImportError: 11 | pass 12 | 13 | from setuptools import setup, find_packages 14 | 15 | 16 | def read(*parts): 17 | content = codecs.open( 18 | os.path.join(os.path.dirname(__file__), *parts) 19 | ).read() 20 | try: 21 | return content.decode("utf8") 22 | except AttributeError: 23 | # python 3 24 | return content 25 | 26 | 27 | def find_version(*file_paths): 28 | version_file = read(*file_paths) 29 | version_match = re.search( 30 | r"^__version__ = ['\"]([^'\"]*)['\"]", version_file, re.M 31 | ) 32 | if version_match: 33 | return version_match.group(1) 34 | raise RuntimeError("Unable to find version string.") 35 | 36 | 37 | setup( 38 | name="django-fancy-cache", 39 | version=find_version("fancy_cache/__init__.py"), 40 | description="A Django 'cache_page' decorator on steroids", 41 | long_description=read("README.rst"), 42 | long_description_content_type="text/x-rst", 43 | author="Peter Bengtsson", 44 | author_email="mail@peterbe.com", 45 | license="BSD", 46 | packages=find_packages(), 47 | include_package_data=True, 48 | zip_safe=False, 49 | classifiers=[ 50 | "Development Status :: 4 - Beta", 51 | "Intended Audience :: Developers", 52 | "License :: OSI Approved :: BSD License", 53 | "Operating System :: OS Independent", 54 | "Programming Language :: Python", 55 | "Programming Language :: Python :: 3.8", 56 | "Programming Language :: Python :: 3.9", 57 | "Programming Language :: Python :: 3.10", 58 | "Programming Language :: Python :: 3.11", 59 | ], 60 | tests_require=["nose"], 61 | test_suite="runtests.runtests", 62 | url="https://github.com/peterbe/django-fancy-cache", 63 | ) 64 | -------------------------------------------------------------------------------- /docs/stats.rst: -------------------------------------------------------------------------------- 1 | .. index:: stats 2 | 3 | .. _stats-chapter: 4 | 5 | Stats - hits and misses 6 | ======================= 7 | 8 | In :ref:`gettingfancy-chapter` we went through various ways of using 9 | ``django-fancy-cache`` that all have a functional difference. 10 | 11 | What you can do is enable stats so you can get an insight into what 12 | ``django-fancy-cache`` does for you. 13 | 14 | Setting up 15 | ---------- 16 | 17 | The first step is to switch on a setting. Put this in your 18 | ``settings``:: 19 | 20 | FANCY_REMEMBER_ALL_URLS = True 21 | FANCY_REMEMBER_STATS_ALL_URLS = True 22 | 23 | The other thing you need to do is to add ``fancy_cache`` to 24 | ``INSTALLED_APPS`` like this:: 25 | 26 | INSTALLED_APPS += ('fancy_cache',) 27 | 28 | The first one is to tell ``django-fancy-cache`` to remember every URL 29 | it caches and the second one is to keep track of how many hits and 30 | misses you have per URL. 31 | 32 | This will enable stats collections on all uses of the ``cache_page`` 33 | decorator. Alternatively you can do it explicitely on just one view. 34 | Like this:: 35 | 36 | from django.shortcuts import render 37 | from fancy_cache import cache_page 38 | 39 | @cache_page(3600, 40 | remember_all_urls=True, 41 | remember_stats_all_urls=True) 42 | def my_view(request): 43 | something_really_slow... 44 | return render(request, 'template.html') 45 | 46 | 47 | Now, run your view a couple of times and then you can use the 48 | management command to get an output of this:: 49 | 50 | $ ./manage.py fancy-urls 51 | /page5.html HITS 62 MISSES 5 52 | /page4.html HITS 4 MISSES 1 53 | 54 | Another way add ``django-fancy-cache`` to your root urls.py like this:: 55 | 56 | urlpatterns = [ 57 | ...your other stuff..., 58 | path('fancy-cache', include('fancy_cache.urls')), 59 | ] 60 | 61 | 62 | Now you can visit ``http://localhost:8000/fancy-cache``. It'll give 63 | you a very basic table with all cache keys, their hits and misses. 64 | -------------------------------------------------------------------------------- /fancy_cache/management/commands/fancy-urls.py: -------------------------------------------------------------------------------- 1 | from __future__ import print_function 2 | import os 3 | 4 | _this_wo_ext = os.path.basename(__file__).rsplit(".", 1)[0] 5 | 6 | __doc__ = """ 7 | If you enable `FANCY_REMEMBER_ALL_URLS` then every URL take is turned 8 | into a cache key for cache_page() to remember is recorded. 9 | You can use this to do statistics or to do invalidation by URL. 10 | 11 | To use: simply add the URL patterns after like this:: 12 | 13 | $ ./manage.py %(this_file)s /path1.html /path3/*/*.json 14 | 15 | To show all cached URLs simply run it with no pattern like this:: 16 | 17 | $ ./manage.py %(this_file)s 18 | 19 | Equally the ``--purge`` switch can always be added. For example, 20 | running this will purge all cached URLs:: 21 | 22 | $ ./manage.py %(this_file)s --purge 23 | 24 | If you enable `FANCY_REMEMBER_STATS_ALL_URLS` you can get a tally for each 25 | URL how many cache HITS and MISSES it has had. 26 | 27 | """ % dict( 28 | this_file=_this_wo_ext 29 | ) 30 | 31 | from optparse import make_option 32 | 33 | from django.core.management.base import BaseCommand 34 | 35 | from fancy_cache.memory import find_urls 36 | 37 | 38 | class Command(BaseCommand): 39 | help = __doc__.strip() 40 | 41 | def add_arguments(self, parser): 42 | parser.add_argument( 43 | "-p", 44 | "--purge", 45 | dest="purge", 46 | action="store_true", 47 | help="Purge found URLs", 48 | ) 49 | 50 | args = "urls" 51 | 52 | def handle(self, *urls, **options): 53 | verbose = int(options["verbosity"]) > 1 54 | _count = 0 55 | for url, cache_key, stats in find_urls(urls, purge=options["purge"]): 56 | _count += 1 57 | if stats: 58 | self.stdout.write(url[:70].ljust(65)), 59 | self.stdout.write("HITS", str(stats["hits"]).ljust(5)), 60 | self.stdout.write("MISSES", str(stats["misses"]).ljust(5)) 61 | 62 | else: 63 | self.stdout.write(url) 64 | 65 | if verbose: 66 | self.stdout.write("-- %s URLs cached --" % _count) 67 | -------------------------------------------------------------------------------- /example/example/app/views.py: -------------------------------------------------------------------------------- 1 | import time 2 | 3 | from django.shortcuts import render, redirect 4 | 5 | from fancy_cache import cache_page 6 | from fancy_cache.memory import find_urls 7 | 8 | 9 | def home(request): 10 | remembered_urls = find_urls([]) 11 | return render(request, "home.html", {"remembered_urls": remembered_urls}) 12 | 13 | 14 | def commafy(s): 15 | r = [] 16 | for i, c in enumerate(reversed(str(s))): 17 | if i and (not (i % 3)): 18 | r.insert(0, ",") 19 | r.insert(0, c) 20 | return "".join(r) 21 | 22 | 23 | @cache_page(60) 24 | def page1(request): 25 | print("CACHE MISS", request.build_absolute_uri()) 26 | t0 = time.time() 27 | result = sum(x for x in xrange(25000000)) 28 | t1 = time.time() 29 | print(t1 - t0) 30 | return render(request, "page.html", dict(result=commafy(result), page="1")) 31 | 32 | 33 | def key_prefixer(request): 34 | # if it's not there, don't cache 35 | return request.GET.get("number") 36 | 37 | 38 | @cache_page(60, key_prefix=key_prefixer) 39 | def page2(request): 40 | if not request.GET.get("number"): 41 | return redirect(request.build_absolute_uri() + "?number=25000000") 42 | print("CACHE MISS", request.build_absolute_uri()) 43 | t0 = time.time() 44 | result = sum(x for x in xrange(25000000)) 45 | t1 = time.time() 46 | print(t1 - t0) 47 | return render(request, "page.html", dict(result=commafy(result), page="2")) 48 | 49 | 50 | def post_processor(response, request): 51 | response.content = response.content.replace( 52 | "", "" 53 | ) 54 | return response 55 | 56 | 57 | @cache_page(60, post_process_response=post_processor) 58 | def page3(request): 59 | print("CACHE MISS", request.build_absolute_uri()) 60 | t0 = time.time() 61 | result = sum(x for x in xrange(25000000)) 62 | t1 = time.time() 63 | print(t1 - t0) 64 | return render(request, "page.html", dict(result=commafy(result), page="3")) 65 | 66 | 67 | def post_processor_always(response, request): 68 | import datetime 69 | 70 | now = datetime.datetime.now() 71 | assert "Right here right now" not in response.content, "already there!" 72 | response.content = response.content.replace( 73 | "", "" % now 74 | ) 75 | return response 76 | 77 | 78 | @cache_page(60, post_process_response_always=post_processor_always) 79 | def page4(request): 80 | print("CACHE MISS", request.build_absolute_uri()) 81 | t0 = time.time() 82 | result = sum(x for x in xrange(25000000)) 83 | t1 = time.time() 84 | print(t1 - t0) 85 | return render(request, "page.html", dict(result=commafy(result), page="4")) 86 | 87 | 88 | @cache_page(60, only_get_keys=["foo", "bar"]) 89 | def page5(request): 90 | print("CACHE MISS", request.build_absolute_uri()) 91 | t0 = time.time() 92 | result = sum(x for x in xrange(25000000)) 93 | t1 = time.time() 94 | print(t1 - t0) 95 | return render(request, "page.html", dict(result=commafy(result), page="5")) 96 | -------------------------------------------------------------------------------- /example/example/settings.py: -------------------------------------------------------------------------------- 1 | # Django settings for example project. 2 | 3 | FANCY_REMEMBER_ALL_URLS = True 4 | FANCY_REMEMBER_STATS_ALL_URLS = True 5 | 6 | DEBUG = True 7 | TEMPLATE_DEBUG = DEBUG 8 | 9 | ADMINS = ( 10 | # ('Your Name', 'your_email@example.com'), 11 | ) 12 | 13 | MANAGERS = ADMINS 14 | 15 | DATABASES = { 16 | "default": { 17 | "ENGINE": "django.db.backends.sqlite3", 18 | "NAME": "database.db", 19 | } 20 | } 21 | 22 | CACHES = { 23 | "default": { 24 | "BACKEND": "django.core.cache.backends.locmem.LocMemCache", 25 | "LOCATION": "unique-snowflake", 26 | } 27 | } 28 | 29 | CACHES = { 30 | "default": { 31 | "BACKEND": "django.core.cache.backends.memcached.MemcachedCache", 32 | "LOCATION": "localhost:11211", 33 | "TIMEOUT": 500, 34 | "KEY_PREFIX": "example", 35 | } 36 | } 37 | 38 | 39 | # Local time zone for this installation. Choices can be found here: 40 | # http://en.wikipedia.org/wiki/List_of_tz_zones_by_name 41 | # although not all choices may be available on all operating systems. 42 | # In a Windows environment this must be set to your system time zone. 43 | TIME_ZONE = "America/Chicago" 44 | 45 | # Language code for this installation. All choices can be found here: 46 | # http://www.i18nguy.com/unicode/language-identifiers.html 47 | LANGUAGE_CODE = "en-us" 48 | 49 | SITE_ID = 1 50 | 51 | # If you set this to False, Django will make some optimizations so as not 52 | # to load the internationalization machinery. 53 | USE_I18N = True 54 | 55 | # If you set this to False, Django will not format dates, numbers and 56 | # calendars according to the current locale. 57 | USE_L10N = True 58 | 59 | # If you set this to False, Django will not use timezone-aware datetimes. 60 | USE_TZ = True 61 | 62 | # Absolute filesystem path to the directory that will hold user-uploaded files. 63 | # Example: "/var/www/example.com/media/" 64 | MEDIA_ROOT = "" 65 | 66 | # URL that handles the media served from MEDIA_ROOT. Make sure to use a 67 | # trailing slash. 68 | # Examples: "http://example.com/media/", "http://media.example.com/" 69 | MEDIA_URL = "" 70 | 71 | # Absolute path to the directory static files should be collected to. 72 | # Don't put anything in this directory yourself; store your static files 73 | # in apps' "static/" subdirectories and in STATICFILES_DIRS. 74 | # Example: "/var/www/example.com/static/" 75 | STATIC_ROOT = "" 76 | 77 | # URL prefix for static files. 78 | # Example: "http://example.com/static/", "http://static.example.com/" 79 | STATIC_URL = "/static/" 80 | 81 | # Additional locations of static files 82 | STATICFILES_DIRS = ( 83 | # Put strings here, like "/home/html/static" or "C:/www/django/static". 84 | # Always use forward slashes, even on Windows. 85 | # Don't forget to use absolute paths, not relative paths. 86 | ) 87 | 88 | # List of finder classes that know how to find static files in 89 | # various locations. 90 | STATICFILES_FINDERS = ( 91 | "django.contrib.staticfiles.finders.FileSystemFinder", 92 | "django.contrib.staticfiles.finders.AppDirectoriesFinder", 93 | # 'django.contrib.staticfiles.finders.DefaultStorageFinder', 94 | ) 95 | 96 | # Make this unique, and don't share it with anybody. 97 | SECRET_KEY = "#yd#+#!6uf2$0!65rx1_rg7zspp*e8wr&e)#mxvo$&!y_(^wc2" 98 | 99 | # List of callables that know how to import templates from various sources. 100 | TEMPLATE_LOADERS = ( 101 | "django.template.loaders.filesystem.Loader", 102 | "django.template.loaders.app_directories.Loader", 103 | # 'django.template.loaders.eggs.Loader', 104 | ) 105 | 106 | MIDDLEWARE_CLASSES = ( 107 | "django.middleware.common.CommonMiddleware", 108 | "django.contrib.sessions.middleware.SessionMiddleware", 109 | "django.middleware.csrf.CsrfViewMiddleware", 110 | "django.contrib.auth.middleware.AuthenticationMiddleware", 111 | "django.contrib.messages.middleware.MessageMiddleware", 112 | # Uncomment the next line for simple clickjacking protection: 113 | # 'django.middleware.clickjacking.XFrameOptionsMiddleware', 114 | ) 115 | 116 | ROOT_URLCONF = "example.urls" 117 | 118 | # Python dotted path to the WSGI application used by Django's runserver. 119 | WSGI_APPLICATION = "example.wsgi.application" 120 | 121 | TEMPLATE_DIRS = ( 122 | # Put strings here, like "/home/html/django_templates" or "C:/www/django/templates". 123 | # Always use forward slashes, even on Windows. 124 | # Don't forget to use absolute paths, not relative paths. 125 | ) 126 | 127 | INSTALLED_APPS = ( 128 | "django.contrib.auth", 129 | "django.contrib.contenttypes", 130 | "django.contrib.sessions", 131 | "django.contrib.sites", 132 | "django.contrib.messages", 133 | "django.contrib.staticfiles", 134 | "example.app", 135 | "fancy_cache", 136 | # Uncomment the next line to enable the admin: 137 | # 'django.contrib.admin', 138 | # Uncomment the next line to enable admin documentation: 139 | # 'django.contrib.admindocs', 140 | ) 141 | 142 | # A sample logging configuration. The only tangible logging 143 | # performed by this configuration is to send an email to 144 | # the site admins on every HTTP 500 error when DEBUG=False. 145 | # See http://docs.djangoproject.com/en/dev/topics/logging for 146 | # more details on how to customize your logging configuration. 147 | LOGGING = { 148 | "version": 1, 149 | "disable_existing_loggers": False, 150 | "filters": { 151 | "require_debug_false": {"()": "django.utils.log.RequireDebugFalse"} 152 | }, 153 | "handlers": { 154 | "mail_admins": { 155 | "level": "ERROR", 156 | "filters": ["require_debug_false"], 157 | "class": "django.utils.log.AdminEmailHandler", 158 | } 159 | }, 160 | "loggers": { 161 | "django.request": { 162 | "handlers": ["mail_admins"], 163 | "level": "ERROR", 164 | "propagate": True, 165 | }, 166 | }, 167 | } 168 | -------------------------------------------------------------------------------- /docs/Makefile: -------------------------------------------------------------------------------- 1 | # Makefile for Sphinx documentation 2 | # 3 | 4 | # You can set these variables from the command line. 5 | SPHINXOPTS = 6 | SPHINXBUILD = sphinx-build 7 | PAPER = 8 | BUILDDIR = _build 9 | 10 | # Internal variables. 11 | PAPEROPT_a4 = -D latex_paper_size=a4 12 | PAPEROPT_letter = -D latex_paper_size=letter 13 | ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . 14 | # the i18n builder cannot share the environment and doctrees with the others 15 | I18NSPHINXOPTS = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . 16 | 17 | .PHONY: help clean html dirhtml singlehtml pickle json htmlhelp qthelp devhelp epub latex latexpdf text man changes linkcheck doctest gettext 18 | 19 | help: 20 | @echo "Please use \`make ' where is one of" 21 | @echo " html to make standalone HTML files" 22 | @echo " dirhtml to make HTML files named index.html in directories" 23 | @echo " singlehtml to make a single large HTML file" 24 | @echo " pickle to make pickle files" 25 | @echo " json to make JSON files" 26 | @echo " htmlhelp to make HTML files and a HTML help project" 27 | @echo " qthelp to make HTML files and a qthelp project" 28 | @echo " devhelp to make HTML files and a Devhelp project" 29 | @echo " epub to make an epub" 30 | @echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter" 31 | @echo " latexpdf to make LaTeX files and run them through pdflatex" 32 | @echo " text to make text files" 33 | @echo " man to make manual pages" 34 | @echo " texinfo to make Texinfo files" 35 | @echo " info to make Texinfo files and run them through makeinfo" 36 | @echo " gettext to make PO message catalogs" 37 | @echo " changes to make an overview of all changed/added/deprecated items" 38 | @echo " linkcheck to check all external links for integrity" 39 | @echo " doctest to run all doctests embedded in the documentation (if enabled)" 40 | 41 | clean: 42 | -rm -rf $(BUILDDIR)/* 43 | 44 | html: 45 | $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html 46 | @echo 47 | @echo "Build finished. The HTML pages are in $(BUILDDIR)/html." 48 | 49 | dirhtml: 50 | $(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml 51 | @echo 52 | @echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml." 53 | 54 | singlehtml: 55 | $(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) $(BUILDDIR)/singlehtml 56 | @echo 57 | @echo "Build finished. The HTML page is in $(BUILDDIR)/singlehtml." 58 | 59 | pickle: 60 | $(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle 61 | @echo 62 | @echo "Build finished; now you can process the pickle files." 63 | 64 | json: 65 | $(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json 66 | @echo 67 | @echo "Build finished; now you can process the JSON files." 68 | 69 | htmlhelp: 70 | $(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp 71 | @echo 72 | @echo "Build finished; now you can run HTML Help Workshop with the" \ 73 | ".hhp project file in $(BUILDDIR)/htmlhelp." 74 | 75 | qthelp: 76 | $(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp 77 | @echo 78 | @echo "Build finished; now you can run "qcollectiongenerator" with the" \ 79 | ".qhcp project file in $(BUILDDIR)/qthelp, like this:" 80 | @echo "# qcollectiongenerator $(BUILDDIR)/qthelp/django-fancy-cache.qhcp" 81 | @echo "To view the help file:" 82 | @echo "# assistant -collectionFile $(BUILDDIR)/qthelp/django-fancy-cache.qhc" 83 | 84 | devhelp: 85 | $(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILDDIR)/devhelp 86 | @echo 87 | @echo "Build finished." 88 | @echo "To view the help file:" 89 | @echo "# mkdir -p $$HOME/.local/share/devhelp/django-fancy-cache" 90 | @echo "# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/django-fancy-cache" 91 | @echo "# devhelp" 92 | 93 | epub: 94 | $(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILDDIR)/epub 95 | @echo 96 | @echo "Build finished. The epub file is in $(BUILDDIR)/epub." 97 | 98 | latex: 99 | $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex 100 | @echo 101 | @echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex." 102 | @echo "Run \`make' in that directory to run these through (pdf)latex" \ 103 | "(use \`make latexpdf' here to do that automatically)." 104 | 105 | latexpdf: 106 | $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex 107 | @echo "Running LaTeX files through pdflatex..." 108 | $(MAKE) -C $(BUILDDIR)/latex all-pdf 109 | @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." 110 | 111 | text: 112 | $(SPHINXBUILD) -b text $(ALLSPHINXOPTS) $(BUILDDIR)/text 113 | @echo 114 | @echo "Build finished. The text files are in $(BUILDDIR)/text." 115 | 116 | man: 117 | $(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man 118 | @echo 119 | @echo "Build finished. The manual pages are in $(BUILDDIR)/man." 120 | 121 | texinfo: 122 | $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo 123 | @echo 124 | @echo "Build finished. The Texinfo files are in $(BUILDDIR)/texinfo." 125 | @echo "Run \`make' in that directory to run these through makeinfo" \ 126 | "(use \`make info' here to do that automatically)." 127 | 128 | info: 129 | $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo 130 | @echo "Running Texinfo files through makeinfo..." 131 | make -C $(BUILDDIR)/texinfo info 132 | @echo "makeinfo finished; the Info files are in $(BUILDDIR)/texinfo." 133 | 134 | gettext: 135 | $(SPHINXBUILD) -b gettext $(I18NSPHINXOPTS) $(BUILDDIR)/locale 136 | @echo 137 | @echo "Build finished. The message catalogs are in $(BUILDDIR)/locale." 138 | 139 | changes: 140 | $(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes 141 | @echo 142 | @echo "The overview file is in $(BUILDDIR)/changes." 143 | 144 | linkcheck: 145 | $(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck 146 | @echo 147 | @echo "Link check complete; look for any errors in the above output " \ 148 | "or in $(BUILDDIR)/linkcheck/output.txt." 149 | 150 | doctest: 151 | $(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest 152 | @echo "Testing of doctests in the sources finished, look at the " \ 153 | "results in $(BUILDDIR)/doctest/output.txt." 154 | -------------------------------------------------------------------------------- /fancy_tests/tests/test_memory.py: -------------------------------------------------------------------------------- 1 | import json 2 | import mock 3 | import time 4 | import unittest 5 | import zlib 6 | 7 | from nose.tools import eq_, ok_ 8 | from django.core.cache import cache, caches 9 | from unittest import mock 10 | 11 | from fancy_cache.constants import REMEMBERED_URLS_KEY 12 | from fancy_cache.memory import find_urls 13 | 14 | 15 | class TestMemory(unittest.TestCase): 16 | def setUp(self): 17 | expiration_time = int(time.time()) + 5 18 | self.urls = { 19 | "/page1.html": ("key1", expiration_time), 20 | "/page2.html": ("key2", expiration_time), 21 | "/page3.html?foo=bar": ("key3", expiration_time), 22 | "/page3.html?foo=else": ("key4", expiration_time), 23 | } 24 | for key, value in self.urls.items(): 25 | cache.set(value[0], key) 26 | cache.set(REMEMBERED_URLS_KEY, self.urls, 5) 27 | 28 | def tearDown(self): 29 | cache.clear() 30 | 31 | def test_find_all_urls(self): 32 | found = list(find_urls([])) 33 | eq_(len(found), 4) 34 | for key, value in self.urls.items(): 35 | pair = (key, value[0], None) 36 | ok_(pair in found) 37 | 38 | def test_find_and_purge_all_urls(self): 39 | found = list(find_urls([], purge=True)) 40 | eq_(len(found), 4) 41 | for key, value in self.urls.items(): 42 | pair = (key, value[0], None) 43 | ok_(pair in found) 44 | found = list(find_urls([])) 45 | eq_(len(found), 0) 46 | 47 | @mock.patch("fancy_cache.memory.COMPRESS_REMEMBERED_URLS", True) 48 | def test_find_and_purge_all_urls_with_zlib_compression(self): 49 | remembered_urls = cache.get(REMEMBERED_URLS_KEY) 50 | remembered_urls = zlib.compress(json.dumps(remembered_urls).encode()) 51 | cache.set(REMEMBERED_URLS_KEY, remembered_urls, 5) 52 | found = list(find_urls([], purge=True)) 53 | eq_(len(found), 4) 54 | for key, value in self.urls.items(): 55 | pair = (key, value[0], None) 56 | ok_(pair in found) 57 | found = list(find_urls([])) 58 | eq_(len(found), 0) 59 | 60 | @mock.patch("fancy_cache.memory.COMPRESS_REMEMBERED_URLS", True) 61 | def test_find_and_purge_all_urls_with_zlib_compression_first_time(self): 62 | """ 63 | When enabling zlib compression, the existing REMEMBERED_URLS 64 | will not be compressed yet. This test ensures that the transition to 65 | compressed REMEMBERED_URLS is seamless. 66 | """ 67 | remembered_urls = cache.get(REMEMBERED_URLS_KEY) 68 | cache.set(REMEMBERED_URLS_KEY, remembered_urls, 5) 69 | found = list(find_urls([], purge=True)) 70 | eq_(len(found), 4) 71 | for key, value in self.urls.items(): 72 | pair = (key, value[0], None) 73 | ok_(pair in found) 74 | found = list(find_urls([])) 75 | eq_(len(found), 0) 76 | 77 | def test_find_one_url(self): 78 | found = list(find_urls(["/page1.html"])) 79 | eq_(len(found), 1) 80 | ok_(("/page1.html", "key1", None) in found) 81 | 82 | def test_purge_one_url(self): 83 | ok_(cache.get("key1")) 84 | ok_("/page1.html" in cache.get(REMEMBERED_URLS_KEY)) 85 | found = list(find_urls(["/page1.html"], purge=True)) 86 | eq_(len(found), 1) 87 | ok_(("/page1.html", "key1", None) in found) 88 | 89 | ok_(not cache.get("key1")) 90 | ok_("/page1.html" not in cache.get(REMEMBERED_URLS_KEY)) 91 | # find all the rest in there 92 | found = list(find_urls([])) 93 | eq_(len(found), 3) 94 | ok_(("/page1.html", "key1", None) not in found) 95 | 96 | def test_purge_one_expired_url(self): 97 | """ 98 | If a URL has expired from the cache it should still 99 | be deleted from remembered URLs when `find_urls` is called 100 | with `purge=True`. 101 | """ 102 | cache.delete("key1") 103 | ok_(not cache.get("key1")) 104 | ok_("/page1.html" in cache.get(REMEMBERED_URLS_KEY)) 105 | found = list(find_urls(["/page1.html"], purge=True)) 106 | eq_(len(found), 0) 107 | ok_(not cache.get("key1")) 108 | ok_("/page1.html" not in cache.get(REMEMBERED_URLS_KEY)) 109 | 110 | # find all the rest in there 111 | found = list(find_urls([])) 112 | eq_(len(found), 3) 113 | ok_(("/page1.html", "key1", None) not in found) 114 | 115 | def test_some_urls(self): 116 | found = list(find_urls(["/page2.html*"])) 117 | eq_(len(found), 1) 118 | ok_(("/page2.html", "key2", None) in found) 119 | 120 | def test_some_urls_double_star(self): 121 | found = list(find_urls(["/page*.html?*"])) 122 | eq_(len(found), 2) 123 | ok_(("/page3.html?foo=bar", "key3", None) in found) 124 | ok_(("/page3.html?foo=else", "key4", None) in found) 125 | 126 | 127 | class TestMemoryWithMemcached(unittest.TestCase): 128 | def setUp(self): 129 | expiration_time = int(time.time()) + 5 130 | self.urls = { 131 | "/page1.html": ("key1", expiration_time), 132 | "/page2.html": ("key2", expiration_time), 133 | "/page3.html?foo=bar": ("key3", expiration_time), 134 | "/page3.html?foo=else": ("key4", expiration_time), 135 | } 136 | self.cache = caches["memcached_backend"] 137 | for key, value in self.urls.items(): 138 | self.cache.set(value[0], key) 139 | self.cache._cache.set(REMEMBERED_URLS_KEY, self.urls, 5) 140 | 141 | def tearDown(self): 142 | cache.clear() 143 | 144 | @mock.patch("fancy_cache.memory.USE_MEMCACHED_CAS", True) 145 | @mock.patch("fancy_cache.memory.cache", caches["memcached_backend"]) 146 | def test_purge_one_url_with_memcached_check_and_set(self): 147 | ok_(self.cache.get("key1")) 148 | ok_("/page1.html" in self.cache._cache.get(REMEMBERED_URLS_KEY)) 149 | found = list(find_urls(["/page1.html"], purge=True)) 150 | eq_(len(found), 1) 151 | ok_(("/page1.html", "key1", None) in found) 152 | 153 | ok_(not cache.get("key1")) 154 | ok_("/page1.html" not in self.cache._cache.get(REMEMBERED_URLS_KEY)) 155 | # find all the rest in there 156 | found = list(find_urls([])) 157 | eq_(len(found), 3) 158 | ok_(("/page1.html", "key1", None) not in found) 159 | -------------------------------------------------------------------------------- /fancy_cache/memory.py: -------------------------------------------------------------------------------- 1 | import json 2 | import logging 3 | import re 4 | import typing 5 | import zlib 6 | 7 | from django.conf import settings 8 | from django.core.cache import cache 9 | 10 | from fancy_cache.constants import LONG_TIME, REMEMBERED_URLS_KEY 11 | from fancy_cache.middleware import USE_MEMCACHED_CAS 12 | from fancy_cache.utils import md5, filter_remembered_urls 13 | 14 | __all__ = ("find_urls",) 15 | 16 | LOGGER = logging.getLogger(__name__) 17 | 18 | COMPRESS_REMEMBERED_URLS = getattr( 19 | settings, "FANCY_COMPRESS_REMEMBERED_URLS", False 20 | ) 21 | 22 | 23 | def _match(url: str, regexes: typing.List[typing.Pattern[str]]): 24 | if not regexes: 25 | return url 26 | for regex in regexes: 27 | if regex.match(url): 28 | return True 29 | return False 30 | 31 | 32 | def _urls_to_regexes( 33 | urls: typing.List[str], 34 | ) -> typing.List[typing.Pattern[str]]: 35 | regexes = [] 36 | for each in urls: 37 | parts = each.split("*") 38 | if len(parts) == 1: 39 | regexes.append(re.compile("^%s$" % re.escape(parts[0]))) 40 | else: 41 | _re = ".*".join(re.escape(x) for x in parts) 42 | regexes.append(re.compile("^%s$" % _re)) 43 | return regexes 44 | 45 | 46 | def find_urls( 47 | urls: typing.List[str] = None, purge: bool = False 48 | ) -> typing.Generator[ 49 | typing.Tuple[str, str, typing.Optional[typing.Dict[str, int]]], None, None 50 | ]: 51 | if USE_MEMCACHED_CAS is True: 52 | remembered_urls = cache._cache.get(REMEMBERED_URLS_KEY, {}) 53 | else: 54 | remembered_urls = cache.get(REMEMBERED_URLS_KEY, {}) 55 | if COMPRESS_REMEMBERED_URLS: 56 | if not isinstance(remembered_urls, dict): 57 | remembered_urls = json.loads( 58 | zlib.decompress(remembered_urls).decode() 59 | ) 60 | keys_to_delete = [] 61 | if urls: 62 | regexes = _urls_to_regexes(urls) 63 | for url in remembered_urls: 64 | if not urls or _match(url, regexes): 65 | cache_key_tuple = remembered_urls[url] 66 | 67 | # TODO: Remove the check for tuple in a future release as it will 68 | # no longer be needed once the new dictionary structure {url: (cache_key, expiration_time)} 69 | # has been implemented. 70 | if isinstance(cache_key_tuple, str): 71 | cache_key_tuple = ( 72 | cache_key_tuple, 73 | 0, 74 | ) 75 | 76 | cache_key = cache_key_tuple[0] 77 | 78 | if not cache.get(cache_key): 79 | if purge: 80 | keys_to_delete.append(url) 81 | continue 82 | if purge: 83 | cache.delete(cache_key) 84 | keys_to_delete.append(url) 85 | misses_cache_key = "%s__misses" % url 86 | misses_cache_key = md5(misses_cache_key) 87 | hits_cache_key = "%s__hits" % url 88 | hits_cache_key = md5(hits_cache_key) 89 | 90 | misses = cache.get(misses_cache_key) 91 | hits = cache.get(hits_cache_key) 92 | if misses is None and hits is None: 93 | stats = None 94 | else: 95 | stats = {"hits": hits or 0, "misses": misses or 0} 96 | yield (url, cache_key, stats) 97 | 98 | if keys_to_delete: 99 | # means something was changed 100 | 101 | if USE_MEMCACHED_CAS is True: 102 | deleted = delete_keys_cas(keys_to_delete) 103 | if deleted is True: 104 | return 105 | # CAS uses `cache._cache.get/set` so we need to set the 106 | # REMEMBERED_URLS dict at that location. 107 | # This is because CAS cannot call `BaseCache.make_key` to generate 108 | # the key when it tries to get a cache entry set by `cache.get/set`. 109 | remembered_urls = cache._cache.get(REMEMBERED_URLS_KEY, {}) 110 | remembered_urls = delete_keys(keys_to_delete, remembered_urls) 111 | cache._cache.set(REMEMBERED_URLS_KEY, remembered_urls, LONG_TIME) 112 | return 113 | 114 | remembered_urls = cache.get(REMEMBERED_URLS_KEY, {}) 115 | if COMPRESS_REMEMBERED_URLS: 116 | if not isinstance(remembered_urls, dict): 117 | remembered_urls = json.loads( 118 | zlib.decompress(remembered_urls).decode() 119 | ) 120 | remembered_urls = delete_keys(keys_to_delete, remembered_urls) 121 | if COMPRESS_REMEMBERED_URLS: 122 | remembered_urls = zlib.compress( 123 | json.dumps(remembered_urls).encode() 124 | ) 125 | cache.set(REMEMBERED_URLS_KEY, remembered_urls, LONG_TIME) 126 | 127 | 128 | def delete_keys_cas(keys_to_delete: typing.List[str]) -> bool: 129 | result = False 130 | tries = 0 131 | while result is False and tries < 100: 132 | remembered_urls, cas_token = cache._cache.gets(REMEMBERED_URLS_KEY) 133 | if remembered_urls is None: 134 | return False 135 | 136 | if COMPRESS_REMEMBERED_URLS: 137 | if not isinstance(remembered_urls, dict): 138 | remembered_urls = json.loads( 139 | zlib.decompress(remembered_urls).decode() 140 | ) 141 | 142 | remembered_urls = delete_keys(keys_to_delete, remembered_urls) 143 | if COMPRESS_REMEMBERED_URLS: 144 | remembered_urls = zlib.compress( 145 | json.dumps(remembered_urls).encode() 146 | ) 147 | result = cache._cache.cas( 148 | REMEMBERED_URLS_KEY, remembered_urls, cas_token, LONG_TIME 149 | ) 150 | tries += 1 151 | if result is False: 152 | LOGGER.error("Fancy cache delete_keys_cas failed after %s tries", tries) 153 | return result 154 | 155 | 156 | def delete_keys( 157 | keys_to_delete: typing.List[str], 158 | remembered_urls: typing.Dict[str, typing.Tuple[str, int]], 159 | ) -> typing.Dict[str, typing.Tuple[str, int]]: 160 | """ 161 | Helper function to delete `keys_to_delete` from the `remembered_urls` dict. 162 | """ 163 | for url in keys_to_delete: 164 | remembered_urls.pop(url) 165 | misses_cache_key = "%s__misses" % url 166 | hits_cache_key = "%s__hits" % url 167 | cache.delete(misses_cache_key) 168 | cache.delete(hits_cache_key) 169 | remembered_urls = filter_remembered_urls(remembered_urls) 170 | return remembered_urls 171 | -------------------------------------------------------------------------------- /docs/conf.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # 3 | # django-fancy-cache documentation build configuration file, created by 4 | # sphinx-quickstart on Sun Feb 10 16:18:37 2013. 5 | # 6 | # This file is execfile()d with the current directory set to its containing dir. 7 | # 8 | # Note that not all possible configuration values are present in this 9 | # autogenerated file. 10 | # 11 | # All configuration values have a default; values that are commented out 12 | # serve to show the default. 13 | 14 | import sys, os 15 | 16 | # If extensions (or modules to document with autodoc) are in another directory, 17 | # add these directories to sys.path here. If the directory is relative to the 18 | # documentation root, use os.path.abspath to make it absolute, like shown here. 19 | # sys.path.insert(0, os.path.abspath('.')) 20 | 21 | # -- General configuration ----------------------------------------------------- 22 | 23 | # If your documentation needs a minimal Sphinx version, state it here. 24 | # needs_sphinx = '1.0' 25 | 26 | # Add any Sphinx extension module names here, as strings. They can be extensions 27 | # coming with Sphinx (named 'sphinx.ext.*') or your custom ones. 28 | extensions = [] 29 | 30 | # Add any paths that contain templates here, relative to this directory. 31 | templates_path = ["_templates"] 32 | 33 | # The suffix of source filenames. 34 | source_suffix = ".rst" 35 | 36 | # The encoding of source files. 37 | # source_encoding = 'utf-8-sig' 38 | 39 | # The master toctree document. 40 | master_doc = "index" 41 | 42 | # General information about the project. 43 | project = "django-fancy-cache" 44 | copyright = "2013, Peter Bengtsson" 45 | 46 | # The version info for the project you're documenting, acts as replacement for 47 | # |version| and |release|, also used in various other places throughout the 48 | # built documents. 49 | # 50 | # The short X.Y version. 51 | version = "0.3" 52 | # The full version, including alpha/beta/rc tags. 53 | release = "0.3" 54 | 55 | # The language for content autogenerated by Sphinx. Refer to documentation 56 | # for a list of supported languages. 57 | # language = None 58 | 59 | # There are two options for replacing |today|: either, you set today to some 60 | # non-false value, then it is used: 61 | # today = '' 62 | # Else, today_fmt is used as the format for a strftime call. 63 | # today_fmt = '%B %d, %Y' 64 | 65 | # List of patterns, relative to source directory, that match files and 66 | # directories to ignore when looking for source files. 67 | exclude_patterns = ["_build"] 68 | 69 | # The reST default role (used for this markup: `text`) to use for all documents. 70 | # default_role = None 71 | 72 | # If true, '()' will be appended to :func: etc. cross-reference text. 73 | # add_function_parentheses = True 74 | 75 | # If true, the current module name will be prepended to all description 76 | # unit titles (such as .. function::). 77 | # add_module_names = True 78 | 79 | # If true, sectionauthor and moduleauthor directives will be shown in the 80 | # output. They are ignored by default. 81 | # show_authors = False 82 | 83 | # The name of the Pygments (syntax highlighting) style to use. 84 | pygments_style = "sphinx" 85 | 86 | # A list of ignored prefixes for module index sorting. 87 | # modindex_common_prefix = [] 88 | 89 | 90 | # -- Options for HTML output --------------------------------------------------- 91 | 92 | # The theme to use for HTML and HTML Help pages. See the documentation for 93 | # a list of builtin themes. 94 | html_theme = "default" 95 | 96 | # Theme options are theme-specific and customize the look and feel of a theme 97 | # further. For a list of options available for each theme, see the 98 | # documentation. 99 | # html_theme_options = {} 100 | 101 | # Add any paths that contain custom themes here, relative to this directory. 102 | # html_theme_path = [] 103 | 104 | # The name for this set of Sphinx documents. If None, it defaults to 105 | # " v documentation". 106 | # html_title = None 107 | 108 | # A shorter title for the navigation bar. Default is the same as html_title. 109 | # html_short_title = None 110 | 111 | # The name of an image file (relative to this directory) to place at the top 112 | # of the sidebar. 113 | # html_logo = None 114 | 115 | # The name of an image file (within the static path) to use as favicon of the 116 | # docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 117 | # pixels large. 118 | # html_favicon = None 119 | 120 | # Add any paths that contain custom static files (such as style sheets) here, 121 | # relative to this directory. They are copied after the builtin static files, 122 | # so a file named "default.css" will overwrite the builtin "default.css". 123 | html_static_path = ["_static"] 124 | 125 | # If not '', a 'Last updated on:' timestamp is inserted at every page bottom, 126 | # using the given strftime format. 127 | # html_last_updated_fmt = '%b %d, %Y' 128 | 129 | # If true, SmartyPants will be used to convert quotes and dashes to 130 | # typographically correct entities. 131 | # html_use_smartypants = True 132 | 133 | # Custom sidebar templates, maps document names to template names. 134 | # html_sidebars = {} 135 | 136 | # Additional templates that should be rendered to pages, maps page names to 137 | # template names. 138 | # html_additional_pages = {} 139 | 140 | # If false, no module index is generated. 141 | # html_domain_indices = True 142 | 143 | # If false, no index is generated. 144 | # html_use_index = True 145 | 146 | # If true, the index is split into individual pages for each letter. 147 | # html_split_index = False 148 | 149 | # If true, links to the reST sources are added to the pages. 150 | # html_show_sourcelink = True 151 | 152 | # If true, "Created using Sphinx" is shown in the HTML footer. Default is True. 153 | # html_show_sphinx = True 154 | 155 | # If true, "(C) Copyright ..." is shown in the HTML footer. Default is True. 156 | # html_show_copyright = True 157 | 158 | # If true, an OpenSearch description file will be output, and all pages will 159 | # contain a tag referring to it. The value of this option must be the 160 | # base URL from which the finished HTML is served. 161 | # html_use_opensearch = '' 162 | 163 | # This is the file name suffix for HTML files (e.g. ".xhtml"). 164 | # html_file_suffix = None 165 | 166 | # Output file base name for HTML help builder. 167 | htmlhelp_basename = "django-fancy-cachedoc" 168 | 169 | 170 | # -- Options for LaTeX output -------------------------------------------------- 171 | 172 | latex_elements = { 173 | # The paper size ('letterpaper' or 'a4paper'). 174 | #'papersize': 'letterpaper', 175 | # The font size ('10pt', '11pt' or '12pt'). 176 | #'pointsize': '10pt', 177 | # Additional stuff for the LaTeX preamble. 178 | #'preamble': '', 179 | } 180 | 181 | # Grouping the document tree into LaTeX files. List of tuples 182 | # (source start file, target name, title, author, documentclass [howto/manual]). 183 | latex_documents = [ 184 | ( 185 | "index", 186 | "django-fancy-cache.tex", 187 | "django-fancy-cache Documentation", 188 | "Peter Bengtsson", 189 | "manual", 190 | ), 191 | ] 192 | 193 | # The name of an image file (relative to this directory) to place at the top of 194 | # the title page. 195 | # latex_logo = None 196 | 197 | # For "manual" documents, if this is true, then toplevel headings are parts, 198 | # not chapters. 199 | # latex_use_parts = False 200 | 201 | # If true, show page references after internal links. 202 | # latex_show_pagerefs = False 203 | 204 | # If true, show URL addresses after external links. 205 | # latex_show_urls = False 206 | 207 | # Documents to append as an appendix to all manuals. 208 | # latex_appendices = [] 209 | 210 | # If false, no module index is generated. 211 | # latex_domain_indices = True 212 | 213 | 214 | # -- Options for manual page output -------------------------------------------- 215 | 216 | # One entry per manual page. List of tuples 217 | # (source start file, name, description, authors, manual section). 218 | man_pages = [ 219 | ( 220 | "index", 221 | "django-fancy-cache", 222 | "django-fancy-cache Documentation", 223 | ["Peter Bengtsson"], 224 | 1, 225 | ) 226 | ] 227 | 228 | # If true, show URL addresses after external links. 229 | # man_show_urls = False 230 | 231 | 232 | # -- Options for Texinfo output ------------------------------------------------ 233 | 234 | # Grouping the document tree into Texinfo files. List of tuples 235 | # (source start file, target name, title, author, 236 | # dir menu entry, description, category) 237 | texinfo_documents = [ 238 | ( 239 | "index", 240 | "django-fancy-cache", 241 | "django-fancy-cache Documentation", 242 | "Peter Bengtsson", 243 | "django-fancy-cache", 244 | "One line description of project.", 245 | "Miscellaneous", 246 | ), 247 | ] 248 | 249 | # Documents to append as an appendix to all manuals. 250 | # texinfo_appendices = [] 251 | 252 | # If false, no module index is generated. 253 | # texinfo_domain_indices = True 254 | 255 | # How to display URL addresses: 'footnote', 'no', or 'inline'. 256 | # texinfo_show_urls = 'footnote' 257 | -------------------------------------------------------------------------------- /README.rst: -------------------------------------------------------------------------------- 1 | django-fancy-cache 2 | ================== 3 | 4 | Copyright Peter Bengtsson, mail@peterbe.com, 2013-2022 5 | 6 | License: BSD 7 | 8 | About django-fancy-cache 9 | ------------------------ 10 | 11 | A Django ``cache_page`` decorator on steroids. 12 | 13 | Unlike the stock ``django.views.decorators.cache.change_page`` this 14 | decorator makes it possible to set a ``key_prefix`` that is a 15 | callable. This callable is passed the request and if it returns ``None`` 16 | the page is not cached. 17 | 18 | Also, you can set another callable called ``post_process_response`` 19 | (which is passed the response and the request) which can do some 20 | additional changes to the response before it's set in cache. 21 | 22 | Lastly, you can set ``post_process_response_always=True`` so that the 23 | ``post_process_response`` callable is always called, even when the 24 | response is coming from the cache. 25 | 26 | 27 | How to use it 28 | ------------- 29 | 30 | In your Django views: 31 | 32 | .. code:: python 33 | 34 | from fancy_cache import cache_page 35 | from django.utils.decorators import method_decorator 36 | from django.views.generic import TemplateView 37 | 38 | @cache_page(60 * 60) 39 | def myview(request): 40 | return render(request, 'page1.html') 41 | 42 | def prefixer(request): 43 | if request.method != 'GET': 44 | return None 45 | if request.GET.get('no-cache'): 46 | return None 47 | return 'myprefix' 48 | 49 | @cache_page(60 * 60, key_prefix=prefixer) 50 | def myotherview(request): 51 | return render(request, 'page2.html') 52 | 53 | def post_processor(response, request): 54 | response.content += '' 55 | return response 56 | 57 | @cache_page( 58 | 60 * 60, 59 | key_prefix=prefixer, 60 | post_process_response=post_processor) 61 | def yetanotherotherview(request): 62 | return render(request, 'page3.html') 63 | 64 | 65 | class MyClassBasedView(TemplateView): 66 | template_name = 'page4.html' 67 | 68 | @method_decorator(cache_page(60*60)) 69 | def get(self, request, *args, **kwargs): 70 | return super().get(request, *args, **kwargs) 71 | 72 | Optional uses 73 | ------------- 74 | 75 | If you want to you can have ``django-fancy-cache`` record every URL it 76 | caches. This can be useful for things like invalidation or curious 77 | statistical inspection. 78 | 79 | You can either switch this on on the decorator itself. Like this: 80 | 81 | .. code:: python 82 | 83 | from fancy_cache import cache_page 84 | 85 | @cache_page(60 * 60, remember_all_urls=True) 86 | def myview(request): 87 | return render(request, 'page1.html') 88 | 89 | Or, more conveniently to apply it to all uses of the ``cache_page`` 90 | decorator you can set the default in your settings with: 91 | 92 | .. code:: python 93 | 94 | FANCY_REMEMBER_ALL_URLS = True 95 | 96 | Now, suppose you have the this option enabled. Now you can do things 97 | like this: 98 | 99 | .. code:: python 100 | 101 | >>> from fancy_cache.memory import find_urls 102 | >>> list(find_urls(['/some/searchpath', '/or/like/*/this.*'])) 103 | >>> # or, to get all: 104 | >>> list(find_urls([])) 105 | 106 | There is also another option to this and that is to purge (aka. 107 | invalidate) the remembered URLs. You simply all the ``purge=True`` 108 | option like this: 109 | 110 | .. code:: python 111 | 112 | >>> from fancy_cache.memory import find_urls 113 | >>> list(find_urls([], purge=True)) 114 | 115 | Note: Since ``find_urls()`` returns a generator, the purging won't 116 | happen unless you exhaust the generator. E.g. looping over it or 117 | turning it into a list. 118 | 119 | **If you are using Memcached**, you must enable check-and-set to 120 | remember all urls by enabling the ``FANCY_USE_MEMCACHED_CHECK_AND_SET`` 121 | flag and enabling ``cas`` in your ``CACHES`` settings: 122 | 123 | .. code:: python 124 | 125 | # in settings.py 126 | 127 | FANCY_USE_MEMCACHED_CHECK_AND_SET = True 128 | 129 | CACHES = { 130 | 'default': { 131 | 'BACKEND': 'django.core.cache.backends.memcached.PyLibMCCache', 132 | 'LOCATION': '127.0.0.1:11211', 133 | # This OPTIONS setting enables Memcached check-and-set which is 134 | # required for remember_all_urls or FANCY_REMEMBER_ALL_URLS. 135 | 'OPTIONS': { 136 | 'behaviors': { 137 | 'cas': True 138 | } 139 | } 140 | } 141 | } 142 | 143 | The second way to inspect all recorded URLs is to use the 144 | ``fancy-cache`` management command. This is only available if you have 145 | added ``fancy_cache`` to your ``INSTALLED_APPS`` setting. Now you can do 146 | this:: 147 | 148 | $ ./manage.py fancy-cache --help 149 | $ ./manage.py fancy-cache 150 | $ ./manage.py fancy-cache /some/searchpath /or/like/*/this.* 151 | $ ./manage.py fancy-cache /some/place/* --purge 152 | $ # or to purge them all! 153 | $ ./manage.py fancy-cache --purge 154 | 155 | Note, it will only print out URLs that if found (and purged, if 156 | applicable). 157 | 158 | The third way to inspect the recorded URLs is to add this to your root 159 | ``urls.py``: 160 | 161 | .. code:: python 162 | 163 | url(r'fancy-cache', include('fancy_cache.urls')), 164 | 165 | Now, if you visit ``http://localhost:8000/fancy-cache`` you get a table 166 | listing every URL that ``django-fancy-cache`` has recorded. 167 | 168 | 169 | Optional uses (for the exceptionally curious) 170 | --------------------------------------------- 171 | 172 | If you have enabled ``FANCY_REMEMBER_ALL_URLS`` you can also enable 173 | ``FANCY_REMEMBER_STATS_ALL_URLS`` in your settings. What this does is 174 | that it attempts to count the number of cache hits and cache misses 175 | you have for each URL. 176 | 177 | This counting of hits and misses is configured to last "a long time". 178 | Possibly longer than you cache your view. So, over time you can expect 179 | to have more than one miss because your view cache expires and it 180 | starts over. 181 | 182 | You can see the stats whenever you use any of the ways described in 183 | the section above. For example like this: 184 | 185 | .. code:: python 186 | 187 | >>> from fancy_cache.memory import find_urls 188 | >>> found = list(find_urls([]))[0] 189 | >>> found[0] 190 | '/some/page.html' 191 | >>> found[2] 192 | {'hits': 1235, 'misses': 12} 193 | 194 | There is obviously a small additional performance cost of using the 195 | ``FANCY_REMEMBER_ALL_URLS`` and/or ``FANCY_REMEMBER_STATS_ALL_URLS`` in 196 | your project so only use it if you don't have any smarter way to 197 | invalidate, for debugging or if you really want make it possible to 198 | purge all cached responses when you run an upgrade of your site or 199 | something. 200 | 201 | Running the test suite 202 | ---------------------- 203 | 204 | The simplest way is to simply run:: 205 | 206 | $ pip install tox 207 | $ tox 208 | 209 | Or to run it without ``tox`` you can simply run:: 210 | 211 | $ export PYTHONPATH=`pwd` 212 | $ export DJANGO_SETTINGS_MODULE=fancy_tests.tests.settings 213 | $ django-admin.py test 214 | 215 | 216 | Changelog 217 | --------- 218 | 219 | 1.3.1 220 | * Fix a bug whereby ``FANCY_COMPRESS_REMEMBERED_URLS`` setting 221 | raises a TypeError upon first implementation. 222 | 223 | 1.3.0 224 | * Enable ``FANCY_COMPRESS_REMEMBERED_URLS`` setting to compress 225 | ``remembered_urls`` dictionary when ``FANCY_REMEMBER_ALL_URLS`` 226 | is True. 227 | * Bugfix: use correct location for ``REMEMBERED_URLS`` 228 | when using Memcached. 229 | * Add support for Python 3.11, Django 4.1 & 4.2 230 | * Drop support for Python < 3.8, Django < 3.2, Django 4.0 231 | 232 | 1.2.1 233 | * Bugfix: conflict between the DummyCache backend when 234 | ``FANCY_USE_MEMCACHED_CHECK_AND_SET`` is ``True`` 235 | 236 | 1.2.0 237 | * Restructure the remembered_urls cache dict to clean up stale entries 238 | * Update FancyCacheMiddleware to match latest Django CacheMiddlware 239 | (Also renames to FancyCacheMiddleware) 240 | * Apply Memcached check-and-set to the delete_keys function 241 | if ``settings.FANCY_USE_MEMCACHED_CHECK_AND_SET = True`` 242 | * Drop support for Python <3.6 243 | * Add support for Python 3.10 and Django 4.0 244 | 245 | 1.1.0 246 | * If you use Memcached you can set 247 | ``settings.FANCY_USE_MEMCACHED_CHECK_AND_SET = True`` so that you 248 | can use ``cache._cache.cas`` which only workd with Memcached 249 | 250 | 1.0.0 251 | * Drop support for Python <3.5 and Django <2.2.0 252 | 253 | 0.11.0 254 | * Fix for ``parse_qs`` correctly between Python 2 and Python 3 255 | 256 | 0.10.0 257 | * Fix for keeping blank strings in query strings. #39 258 | 259 | 0.9.0 260 | * Django 1.10 support 261 | 262 | 0.8.2 263 | * Remove deprecated way to define URL patterns and tests in python 3.5 264 | 265 | 0.8.1 266 | * Ability to specify different cache backends to be used 267 | https://github.com/peterbe/django-fancy-cache/pull/31 268 | 269 | 0.8.0 270 | * Started keeping a Changelog 271 | -------------------------------------------------------------------------------- /docs/gettingfancy.rst: -------------------------------------------------------------------------------- 1 | .. index:: gettingfancy 2 | 3 | .. _gettingfancy-chapter: 4 | 5 | Getting fancy 6 | =============== 7 | 8 | Invalidation based on key prefix 9 | -------------------------------- 10 | 11 | That :ref:`gettingstarted-chapter` stuff is basically what you get 12 | from standard Django. Now let's get a bit more fancy. For starters you 13 | want to write some very specific code to effectively control the 14 | invalidation. 15 | 16 | Suppose you have a view that represents a blog post. The blog post 17 | contains a list of comments. You want the page cached but as soon as a 18 | new comment is added, the page should be cleared from cache. 19 | 20 | Step one, add the prefixing:: 21 | 22 | 23 | def latest_comment_prefix(request, pk): 24 | try: 25 | latest, = ( 26 | Comment.objects.filter(post_id=request.pk) 27 | .order_by('-date')[:1] 28 | ) 29 | return latest.date.strftime('%f') # microsecond 30 | except ValueError: 31 | return 'no-comments' 32 | 33 | @cache_page(3600, key_prefix=latest_comment_prefix) 34 | def blog_post(request, pk): 35 | post = Post.objects.get(pk=pk) 36 | comments = Comment.objects.all().order_by('date') 37 | return render(request, 'post.html', 38 | {'comments': comments, 'post': post}) 39 | 40 | That means that the page will be cached nicely based on the latest 41 | comment. If a new comment is added, it's going to have a different 42 | timestamp and thus the blog post view cache will be refreshed. 43 | 44 | However, this means that if no new comments are added for a while, we 45 | have to do that look-up each and every time. Better cache that too. 46 | Here's an updated version:: 47 | 48 | from django.core.cache import cache 49 | 50 | def latest_comment_prefix(request, pk): 51 | cache_key = 'latest-comment-for-%s' % pk 52 | timestamp = cache.get(cache_key) 53 | if timestamp is None: 54 | try: 55 | latest, = ( 56 | Comment.objects 57 | .filter(post_id=request.pk) 58 | .order_by('-date')[:1] 59 | ) 60 | timestamp = latest.date.strftime('%f') 61 | except ValueError: 62 | timestamp = 'no-comments' 63 | cache.set(cache_key, timestamp, 3600) 64 | return timestamp 65 | 66 | So we're looking up what the latest comment is and caching it for the 67 | same amount of time as we cache the view. That means that the least 68 | effort you need to return fresh view is to do 1 cache look-up which is 69 | very fast. 70 | 71 | But now you're committed to this timestamp value for one hour (aka. 72 | 3,600 seconds) which isn't cool. You want a fresh view immediately 73 | after a new comment is added. Waiting one hour isn't good enough. You 74 | have a choice here. You can either hook in some code in where new 75 | comments are created in another view or you can use signals. Like this:: 76 | 77 | # now in models.py 78 | 79 | from django.db.models.signals import post_save 80 | from django.dispatch import receiver 81 | from django.core.cache import cache 82 | 83 | @receiver(post_save, sender=Comment) 84 | def invalidate_latest_comment_timestamp(sender, instance, **kwargs): 85 | pk = instance.post.pk 86 | cache_key = 'latest-comment-for-%s' % pk 87 | cache.delete(cache_key) 88 | 89 | Now you get the ideal set up you can have. Almost. All it takes to 90 | render the blog post page is 1 quick cache look-up. And as soon as a 91 | new comment is posted, the page is immediately refreshed. 92 | 93 | As soon as you've implemented something like this and you get 94 | confident that your ``key_prefix`` callable is good you can start 95 | increasing the timeout from one hour to one week or something like 96 | that. 97 | 98 | Another realistic use case of this is if your logged-in users get 99 | different content but whatever it is, it can be cached. Here's an 100 | example of that:: 101 | 102 | def dashboard_prefix(request): 103 | # it depends on the user 104 | if not request.user.is_active: 105 | # no cache key applicable 106 | return None 107 | return str(request.user.pk) 108 | 109 | @cache_page(60 * 60 * 24, key_prefix=dashboard_prefix) 110 | def users_dashboard(request): 111 | if not request.user.is_active: 112 | raise FuckOff() 113 | stuff = get_interesting_stuff(request.user) 114 | return render(request, 'dashboard.html', {'stuff': stuff}) 115 | 116 | 117 | Remembering what's cached 118 | ------------------------- 119 | 120 | Writing an advanced ``key_prefix`` callable gives you a lot of 121 | flexibility. You site is probably filled with lots of interesting edge 122 | cases and other pieces of data that affect what it means for a page to 123 | be fresh and when it needs to be invalidated. 124 | 125 | A simpler approach is to let ``django-fancy-cache`` keep track of *all* 126 | its internal cache keys so that you can reverse that based on a URL. 127 | 128 | Again, let's imagine you have a blog post that shows the latest 129 | comments. As soon as a new comment is added you want the blog post to 130 | refresh. The first thing you need to do is make all cached 131 | URLs remembered. Add this to your settings:: 132 | 133 | FANCY_REMEMBER_ALL_URLS = True 134 | 135 | Now, every time ``django-fancy-cache`` wraps and caches a view it 136 | remembers what the URL was and what cache key it led to. Now, let's 137 | add a signal that uses this information to invalidate the cache when a 138 | new comment is added:: 139 | 140 | # in models.py 141 | 142 | from django.db.models.signals import post_save 143 | from django.dispatch import receiver 144 | from django.core.urlresolvers import reverse 145 | from fancy_cache.memory import find_urls 146 | 147 | @receiver(post_save, sender=Comment) 148 | def invalidate_latest_comment_timestamp(sender, instance, **kwargs): 149 | post_pk = instance.post.pk 150 | post_url = reverse('blog:post', args=(post_pk,)) 151 | list(find_urls([post_url], purge=True)) 152 | 153 | Note: Since find_urls() returns a generator, the purging won't happen unless you exhaust the generator. 154 | E.g. looping over it or turning it into a list. 155 | 156 | > :warning: **If you are using Memcached, you must 157 | > enable check-and-set to remember all urls** 158 | > by enabling the `FANCY_USE_MEMCACHED_CHECK_AND_SET` 159 | > flag and enabling `cas` in your `CACHES` settings: 160 | 161 | # in settings.py 162 | 163 | FANCY_USE_MEMCACHED_CHECK_AND_SET = True 164 | 165 | CACHES = { 166 | 'default': { 167 | 'BACKEND': 'django.core.cache.backends.memcached.PyLibMCCache', 168 | 'LOCATION': '127.0.0.1:11211', 169 | # This OPTIONS setting enables Memcached check-and-set which is 170 | # required for remember_all_urls or FANCY_REMEMBER_ALL_URLS. 171 | 'OPTIONS': { 172 | 'behaviors': { 173 | 'cas': True 174 | } 175 | } 176 | } 177 | } 178 | 179 | Voila! As soon as a new comment is added to a post, all cached URLs 180 | with that URL are purged from the cache. 181 | 182 | The page is now aggressively cached meaning you're ready for the next 183 | Hacker News stampeding herd, and as soon as a new comment it added the 184 | page is refreshed automatically so people don't get stale content. 185 | 186 | 187 | Do some last minute fixes to the output 188 | --------------------------------------- 189 | 190 | Suppose that you want to cache the view rendering but only apply 191 | certain cool optimizations to the rendered output just before the rendered 192 | output is put into the cache. Or perhaps you have some piece of code 193 | that you can't easily weave into your view code. 194 | 195 | What you can do, is some "last minute" changes to the response with a 196 | callable function like this:: 197 | 198 | from django.shortcuts import render 199 | from fancy_cache import cache_page 200 | 201 | def css_stats(response, request): 202 | no_styles = response.content.count('') 203 | response.content += ( 204 | '\n