├── .gitignore ├── fastestcache ├── benchmarking │ ├── __init__.py │ ├── migrations │ │ └── __init__.py │ ├── models.py │ ├── tests.py │ ├── admin.py │ ├── urls.py │ └── views.py ├── __init__.py ├── apps.py ├── ujson_serializer.py ├── wsgi.py ├── urls.py └── settings.py ├── requirements.txt ├── manage.py └── README.md /.gitignore: -------------------------------------------------------------------------------- 1 | db.sqlite3 2 | -------------------------------------------------------------------------------- /fastestcache/benchmarking/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /fastestcache/benchmarking/migrations/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /fastestcache/__init__.py: -------------------------------------------------------------------------------- 1 | default_app_config = 'fastestcache.apps.BenchmarkingConfig' 2 | -------------------------------------------------------------------------------- /fastestcache/benchmarking/models.py: -------------------------------------------------------------------------------- 1 | from django.db import models 2 | 3 | # Create your models here. 4 | -------------------------------------------------------------------------------- /fastestcache/benchmarking/tests.py: -------------------------------------------------------------------------------- 1 | from django.test import TestCase 2 | 3 | # Create your tests here. 4 | -------------------------------------------------------------------------------- /fastestcache/benchmarking/admin.py: -------------------------------------------------------------------------------- 1 | from django.contrib import admin 2 | 3 | # Register your models here. 4 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | Django==1.11.1 2 | django-redis==4.8.0 3 | python-decouple==3.0 4 | ascii-graph==1.3.0 5 | hiredis==0.2.0 6 | msgpack-python==0.4.8 7 | ujson==1.35 8 | -------------------------------------------------------------------------------- /fastestcache/benchmarking/urls.py: -------------------------------------------------------------------------------- 1 | from django.conf.urls import url 2 | from django.conf import settings 3 | from .views import run, summary 4 | 5 | 6 | rest = '|'.join(settings.CACHE_NAMES) 7 | 8 | urlpatterns = [ 9 | url(r'(?P(random|{}))'.format(rest), run), 10 | url(r'summary', summary), 11 | ] 12 | -------------------------------------------------------------------------------- /fastestcache/apps.py: -------------------------------------------------------------------------------- 1 | from django_redis import get_redis_connection 2 | 3 | from django.apps import AppConfig 4 | 5 | 6 | class BenchmarkingConfig(AppConfig): 7 | name = 'fastestcache' 8 | 9 | def ready(self): 10 | connection = get_redis_connection('default') 11 | print("All Redis flushed") 12 | connection.flushall() 13 | -------------------------------------------------------------------------------- /fastestcache/ujson_serializer.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import, unicode_literals 2 | 3 | import ujson as json 4 | 5 | from django.utils.encoding import force_bytes, force_text 6 | 7 | 8 | from django_redis.serializers.base import BaseSerializer 9 | 10 | 11 | class UJSONSerializer(BaseSerializer): 12 | def dumps(self, value): 13 | return force_bytes(json.dumps(value)) 14 | 15 | def loads(self, value): 16 | return json.loads(force_text(value)) 17 | -------------------------------------------------------------------------------- /fastestcache/wsgi.py: -------------------------------------------------------------------------------- 1 | """ 2 | WSGI config for fastestcache project. 3 | 4 | It exposes the WSGI callable as a module-level variable named ``application``. 5 | 6 | For more information on this file, see 7 | https://docs.djangoproject.com/en/1.11/howto/deployment/wsgi/ 8 | """ 9 | 10 | import os 11 | 12 | from django.core.wsgi import get_wsgi_application 13 | 14 | os.environ.setdefault("DJANGO_SETTINGS_MODULE", "fastestcache.settings") 15 | 16 | application = get_wsgi_application() 17 | -------------------------------------------------------------------------------- /fastestcache/urls.py: -------------------------------------------------------------------------------- 1 | """fastestcache URL Configuration 2 | 3 | The `urlpatterns` list routes URLs to views. For more information please see: 4 | https://docs.djangoproject.com/en/1.11/topics/http/urls/ 5 | Examples: 6 | Function views 7 | 1. Add an import: from my_app import views 8 | 2. Add a URL to urlpatterns: url(r'^$', views.home, name='home') 9 | Class-based views 10 | 1. Add an import: from other_app.views import Home 11 | 2. Add a URL to urlpatterns: url(r'^$', Home.as_view(), name='home') 12 | Including another URLconf 13 | 1. Import the include() function: from django.conf.urls import url, include 14 | 2. Add a URL to urlpatterns: url(r'^blog/', include('blog.urls')) 15 | """ 16 | from django.conf.urls import url, include 17 | 18 | 19 | urlpatterns = [ 20 | url(r'', include('fastestcache.benchmarking.urls')), 21 | ] 22 | -------------------------------------------------------------------------------- /manage.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | import os 3 | import sys 4 | 5 | if __name__ == "__main__": 6 | os.environ.setdefault("DJANGO_SETTINGS_MODULE", "fastestcache.settings") 7 | try: 8 | from django.core.management import execute_from_command_line 9 | except ImportError: 10 | # The above import may fail for some other reason. Ensure that the 11 | # issue is really that Django is missing to avoid masking other 12 | # exceptions on Python 2. 13 | try: 14 | import django 15 | except ImportError: 16 | raise ImportError( 17 | "Couldn't import Django. Are you sure it's installed and " 18 | "available on your PYTHONPATH environment variable? Did you " 19 | "forget to activate a virtual environment?" 20 | ) 21 | raise 22 | execute_from_command_line(sys.argv) 23 | -------------------------------------------------------------------------------- /fastestcache/benchmarking/views.py: -------------------------------------------------------------------------------- 1 | import time 2 | import random 3 | 4 | from ascii_graph import Pyasciigraph 5 | from django_redis import get_redis_connection 6 | 7 | from django import http 8 | from django.conf import settings 9 | from django.core.cache import caches 10 | 11 | 12 | def run(request, cache_name): 13 | if cache_name == 'random': 14 | cache_name = random.choice(settings.CACHE_NAMES) 15 | 16 | cache = caches[cache_name] 17 | t0 = time.time() 18 | data = cache.get('benchmarking', []) 19 | t1 = time.time() 20 | if random.random() < settings.WRITE_CHANCE: 21 | data.append(t1 - t0) 22 | cache.set('benchmarking', data, 100) 23 | if data: 24 | avg = 1000 * sum(data) / len(data) 25 | else: 26 | avg = 'notyet' 27 | # print(cache_name, '#', len(data), 'avg:', avg, ' size:', len(str(data))) 28 | return http.HttpResponse('{}\n'.format(avg)) 29 | 30 | 31 | def _stats(r): 32 | # returns the median, average and standard deviation of a sequence 33 | tot = sum(r) 34 | avg = tot/len(r) 35 | sdsq = sum([(i-avg)**2 for i in r]) 36 | s = list(r) 37 | s.sort() 38 | return s[len(s)//2], avg, (sdsq/(len(r)-1 or 1))**.5 39 | 40 | 41 | def summary(request): 42 | 43 | P = 15 44 | 45 | def fmt_ms(s): 46 | return ('{:.3f}ms'.format(1000 * s)).rjust(P) 47 | 48 | r = http.HttpResponse() 49 | r.write(''.ljust(P)) 50 | r.write('TIMES'.rjust(P)) 51 | r.write('AVERAGE'.rjust(P)) 52 | r.write('MEDIAN'.rjust(P)) 53 | r.write('STDDEV'.rjust(P)) 54 | r.write('\n') 55 | avgs = [] 56 | medians = [] 57 | for CACHE in settings.CACHE_NAMES: 58 | data = caches[CACHE].get('benchmarking') 59 | if data is None: 60 | r.write('Nothing for {}\n'.format(CACHE)) 61 | else: 62 | # Always chop off the first 10 measurements because it's usually 63 | # way higher than all the others. That way we're only comparing 64 | # configurations once they're all warmed up 65 | data = data[10:] 66 | median, avg, stddev = _stats(data) 67 | avgs.append((CACHE, avg * 1000)) 68 | medians.append((CACHE, median * 1000)) 69 | r.write( 70 | '{}{}{}{}{}\n' 71 | .format( 72 | CACHE.ljust(P), 73 | str(len(data)).rjust(P), 74 | fmt_ms(avg), 75 | fmt_ms(median), 76 | fmt_ms(stddev), 77 | ) 78 | ) 79 | 80 | r.write('\n') 81 | 82 | graph = Pyasciigraph( 83 | float_format='{0:,.3f}' 84 | ) 85 | for line in graph.graph('Best Averages (shorter better)', avgs): 86 | print(line, file=r) 87 | for line in graph.graph('Best Medians (shorter better)', medians): 88 | print(line, file=r) 89 | 90 | print('\n', file=r) 91 | 92 | sizes = [] 93 | for name in settings.CACHE_NAMES: 94 | connection = get_redis_connection(name) 95 | sizes.append((name, connection.strlen(":1:benchmarking"))) 96 | 97 | graph = Pyasciigraph( 98 | human_readable='si', 99 | ) 100 | for line in graph.graph('Size of Data Saved (shorter better)', sizes): 101 | print(line, file=r) 102 | 103 | print('\n', file=r) 104 | return r 105 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | Fastest Redis 2 | ============= 3 | 4 | Blog post 5 | --------- 6 | 7 | Please see [Fastest Redis configuration for Django](https://www.peterbe.com/plog/fastest-redis-optimization-for-django). 8 | 9 | Introduction 10 | ------------ 11 | 12 | An experiment ground for testing which way to use Redis 13 | as a cache backend is the fastest. 14 | 15 | All these tests are variations of configurations 16 | using [django-redis](https://niwinz.github.io/django-redis/latest/). 17 | 18 | 19 | Sample Run 20 | ---------- 21 | 22 | Start the server: 23 | 24 | ./manage.py runserver 25 | 26 | First run it a bunch of times: 27 | 28 | wrk -d20s "http://127.0.0.1:8000/random" 29 | 30 | Then to see which was the fastest: 31 | 32 | curl http://127.0.0.1:8000/summary 33 | 34 | You'll get an output like this: 35 | 36 | TIMES AVERAGE MEDIAN STDDEV 37 | json 1508 2.178ms 1.551ms 1.866ms 38 | lzma 1110 2.016ms 1.075ms 2.102ms 39 | ujson 1835 1.634ms 0.829ms 1.862ms 40 | zlib 1656 1.618ms 0.781ms 1.882ms 41 | hires 1791 1.513ms 0.743ms 1.701ms 42 | default 1763 1.508ms 0.745ms 1.773ms 43 | msgpack 1784 1.543ms 0.735ms 1.768ms 44 | 45 | Best Averages (shorter better) 46 | ############################################################################### 47 | ███████████████████████████████████████████████████████████████ 2.178 json 48 | ██████████████████████████████████████████████████████████ 2.016 lzma 49 | ███████████████████████████████████████████████ 1.634 ujson 50 | ██████████████████████████████████████████████ 1.618 zlib 51 | ███████████████████████████████████████████ 1.513 hires 52 | ███████████████████████████████████████████ 1.508 default 53 | ████████████████████████████████████████████ 1.543 msgpack 54 | Best Medians (shorter better) 55 | ############################################################################### 56 | ███████████████████████████████████████████████████████████████ 1.551 json 57 | ███████████████████████████████████████████ 1.075 lzma 58 | █████████████████████████████████ 0.829 ujson 59 | ███████████████████████████████ 0.781 zlib 60 | ██████████████████████████████ 0.743 hires 61 | ██████████████████████████████ 0.745 default 62 | █████████████████████████████ 0.735 msgpack 63 | 64 | 65 | Size of Data Saved (shorter better) 66 | ############################################################################### 67 | █████████████████████████████████████████████████████████████████ 34K json 68 | █████ 3K lzma 69 | █████████████████████████████████████████████ 24K ujson 70 | █████████ 5K zlib 71 | ██████████████████████████████ 16K hires 72 | ██████████████████████████████ 16K default 73 | ██████████████████████████████ 16K msgpack 74 | -------------------------------------------------------------------------------- /fastestcache/settings.py: -------------------------------------------------------------------------------- 1 | """ 2 | Django settings for fastestcache project. 3 | 4 | Generated by 'django-admin startproject' using Django 1.11. 5 | 6 | For more information on this file, see 7 | https://docs.djangoproject.com/en/1.11/topics/settings/ 8 | 9 | For the full list of settings and their values, see 10 | https://docs.djangoproject.com/en/1.11/ref/settings/ 11 | """ 12 | 13 | import os 14 | 15 | from decouple import config, Csv 16 | 17 | # Build paths inside the project like this: os.path.join(BASE_DIR, ...) 18 | BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) 19 | 20 | 21 | # Quick-start development settings - unsuitable for production 22 | # See https://docs.djangoproject.com/en/1.11/howto/deployment/checklist/ 23 | 24 | # SECURITY WARNING: keep the secret key used in production secret! 25 | SECRET_KEY = config( 26 | 'SECRET_KEY', '*4)4&y3sbfv0btm05+@c1&18w50m%w9o(97#94+h&zfi+2f(4q' 27 | ) 28 | 29 | # SECURITY WARNING: don't run with debug turned on in production! 30 | DEBUG = config('DEBUG', True) 31 | DEBUG_PROPAGATE_EXCEPTIONS = config( 32 | 'DEBUG_PROPAGATE_EXCEPTIONS', 33 | False, 34 | cast=bool, 35 | ) 36 | ALLOWED_HOSTS = config('ALLOWED_HOSTS', '', cast=Csv()) 37 | 38 | 39 | # Application definition 40 | 41 | INSTALLED_APPS = [ 42 | 'fastestcache.apps.BenchmarkingConfig', 43 | 'fastestcache.benchmarking', 44 | ] 45 | 46 | MIDDLEWARE = [ 47 | ] 48 | 49 | ROOT_URLCONF = 'fastestcache.urls' 50 | 51 | TEMPLATES = [ 52 | { 53 | 'BACKEND': 'django.template.backends.django.DjangoTemplates', 54 | 'DIRS': [], 55 | 'APP_DIRS': True, 56 | 'OPTIONS': { 57 | 'context_processors': [ 58 | 'django.template.context_processors.debug', 59 | 'django.template.context_processors.request', 60 | 'django.contrib.auth.context_processors.auth', 61 | 'django.contrib.messages.context_processors.messages', 62 | ], 63 | }, 64 | }, 65 | ] 66 | 67 | WSGI_APPLICATION = 'fastestcache.wsgi.application' 68 | 69 | 70 | # Database 71 | # https://docs.djangoproject.com/en/1.11/ref/settings/#databases 72 | 73 | DATABASES = { 74 | 'default': { 75 | 'ENGINE': 'django.db.backends.sqlite3', 76 | 'NAME': os.path.join(BASE_DIR, 'db.sqlite3'), 77 | } 78 | } 79 | 80 | 81 | # Internationalization 82 | # https://docs.djangoproject.com/en/1.11/topics/i18n/ 83 | 84 | LANGUAGE_CODE = 'en-us' 85 | 86 | TIME_ZONE = 'UTC' 87 | 88 | USE_I18N = False 89 | 90 | USE_L10N = True 91 | 92 | USE_TZ = True 93 | 94 | 95 | # Static files (CSS, JavaScript, Images) 96 | # https://docs.djangoproject.com/en/1.11/howto/static-files/ 97 | 98 | STATIC_URL = '/static/' 99 | 100 | 101 | CACHES = { 102 | "default": { 103 | "BACKEND": "django_redis.cache.RedisCache", 104 | "LOCATION": config('REDIS_LOCATION', 'redis://127.0.0.1:6379') + '/0', 105 | "OPTIONS": { 106 | "CLIENT_CLASS": "django_redis.client.DefaultClient", 107 | } 108 | }, 109 | "json": { 110 | "BACKEND": "django_redis.cache.RedisCache", 111 | "LOCATION": config('REDIS_LOCATION', 'redis://127.0.0.1:6379') + '/1', 112 | "OPTIONS": { 113 | "CLIENT_CLASS": "django_redis.client.DefaultClient", 114 | "SERIALIZER": "django_redis.serializers.json.JSONSerializer", 115 | } 116 | }, 117 | "ujson": { 118 | "BACKEND": "django_redis.cache.RedisCache", 119 | "LOCATION": config('REDIS_LOCATION', 'redis://127.0.0.1:6379') + '/2', 120 | "OPTIONS": { 121 | "CLIENT_CLASS": "django_redis.client.DefaultClient", 122 | "SERIALIZER": "fastestcache.ujson_serializer.UJSONSerializer", 123 | } 124 | }, 125 | "msgpack": { 126 | "BACKEND": "django_redis.cache.RedisCache", 127 | "LOCATION": config('REDIS_LOCATION', 'redis://127.0.0.1:6379') + '/3', 128 | "OPTIONS": { 129 | "CLIENT_CLASS": "django_redis.client.DefaultClient", 130 | "SERIALIZER": "django_redis.serializers.msgpack.MSGPackSerializer", 131 | } 132 | }, 133 | "hires": { 134 | "BACKEND": "django_redis.cache.RedisCache", 135 | "LOCATION": config('REDIS_LOCATION', 'redis://127.0.0.1:6379') + '/4', 136 | "OPTIONS": { 137 | "CLIENT_CLASS": "django_redis.client.DefaultClient", 138 | "PARSER_CLASS": "redis.connection.HiredisParser", 139 | } 140 | }, 141 | "zlib": { 142 | "BACKEND": "django_redis.cache.RedisCache", 143 | "LOCATION": config('REDIS_LOCATION', 'redis://127.0.0.1:6379') + '/5', 144 | "OPTIONS": { 145 | "CLIENT_CLASS": "django_redis.client.DefaultClient", 146 | "COMPRESSOR": "django_redis.compressors.zlib.ZlibCompressor", 147 | } 148 | }, 149 | "lzma": { 150 | "BACKEND": "django_redis.cache.RedisCache", 151 | "LOCATION": config('REDIS_LOCATION', 'redis://127.0.0.1:6379') + '/6', 152 | "OPTIONS": { 153 | "CLIENT_CLASS": "django_redis.client.DefaultClient", 154 | "COMPRESSOR": "django_redis.compressors.lzma.LzmaCompressor" 155 | } 156 | }, 157 | "msgpack_zlib": { 158 | "BACKEND": "django_redis.cache.RedisCache", 159 | "LOCATION": config('REDIS_LOCATION', 'redis://127.0.0.1:6379') + '/7', 160 | "OPTIONS": { 161 | "CLIENT_CLASS": "django_redis.client.DefaultClient", 162 | "COMPRESSOR": "django_redis.compressors.zlib.ZlibCompressor", 163 | "SERIALIZER": "django_redis.serializers.msgpack.MSGPackSerializer", 164 | } 165 | }, 166 | } 167 | 168 | 169 | CACHE_NAMES = config( 170 | 'CACHE_NAMES', 171 | default=','.join(list(CACHES.keys())), 172 | cast=Csv() 173 | ) 174 | 175 | # 1.0 = Always make a write 176 | WRITE_CHANCE = config('WRITE_CHANCE', 1.0, cast=float) 177 | --------------------------------------------------------------------------------