├── .gitignore ├── COPYING ├── README ├── django_dumpslow ├── __init__.py ├── management │ ├── __init__.py │ └── commands │ │ ├── __init__.py │ │ └── dumpslow.py ├── middleware.py ├── models.py └── utils.py ├── django_dumpslow_example ├── __init__.py ├── example │ ├── __init__.py │ ├── models.py │ ├── templates │ │ └── index.html │ ├── urls.py │ └── views.py ├── manage.py ├── settings.py └── urls.py └── setup.py /.gitignore: -------------------------------------------------------------------------------- 1 | *.pyc 2 | slow-requests.log 3 | django_dumpslow.egg-info 4 | -------------------------------------------------------------------------------- /COPYING: -------------------------------------------------------------------------------- 1 | Copyright © 2009 Chris Lamb 2 | All rights reserved. 3 | 4 | Redistribution and use in source and binary forms, with or without 5 | modification, are permitted provided that the following conditions 6 | are met: 7 | 1. Redistributions of source code must retain the above copyright 8 | notice, this list of conditions and the following disclaimer. 9 | 2. Redistributions in binary form must reproduce the above copyright 10 | notice, this list of conditions and the following disclaimer in the 11 | documentation and/or other materials provided with the distribution. 12 | 13 | THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 14 | ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 15 | IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 16 | ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 17 | FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 18 | DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 19 | OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 20 | HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 21 | LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 22 | OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 23 | SUCH DAMAGE. 24 | -------------------------------------------------------------------------------- /README: -------------------------------------------------------------------------------- 1 | From: http://chris-lamb.co.uk/projects/django-dumpslow/ 2 | 3 | django-dumpslow is a reusable Django application that logs requests that take a 4 | long time to execute and provides an tool to summarise the resulting data. 5 | 6 | Modelled on mysqldumpslow, django-dumpslow is intended as a tool to determine 7 | which parts of a project might benefit most from optimisation and to provide 8 | valuable empirical data. 9 | 10 | The tool groups requests that are similar by exploiting Django's ability to 11 | reverse URLs - requests that that use the same view should not counted 12 | distinctly. Views can then be grouped by the total accumulated time spent by 13 | that view or simply by their raw frequency in the logs. 14 | 15 | Additionally, if the log entries contain timestamps then django-dumpslow can 16 | limit queries by a user-defined interval. This can be useful for generating 17 | regular reports of slow pages, perhaps by regularly emailing them to a 18 | development list. 19 | 20 | Data is stored in Redis to allow easy collation of data from multiple front-end 21 | servers (django-dumpslow requires the Redis Python client library). 22 | 23 | An example output is: 24 | 25 | $ ./manage.py dumpslow 26 | View Accumulated time 27 | ===================================================== 28 | example.views.slow 92.88 29 | /unknown (unreversible url) 16.84 30 | 31 | django-dumpslow ships with an example project that demonstrates its 32 | functionality. 33 | 34 | Installation 35 | ------------ 36 | 37 | 1. Get Redis working in django, it's required. You'll need a redis server 38 | running and REDIS_HOST and REDIS_PORT set in settings.py 39 | 40 | 2. Add the following to INSTALLED_APPS in settings.py:: 41 | 42 | 'django_dumpslow' 43 | 44 | 3. Add the following to MIDDLEWARE_CLASSES in settings.py:: 45 | 46 | 'django_dumpslow.middleware.LogLongRequestMiddleware' 47 | 48 | License 49 | ------- 50 | 51 | django-dumpslow is released under the BSD license. 52 | -------------------------------------------------------------------------------- /django_dumpslow/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lamby/django-dumpslow/0c2c46d16510596ee830cc97cb69b53992ed3fe6/django_dumpslow/__init__.py -------------------------------------------------------------------------------- /django_dumpslow/management/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lamby/django-dumpslow/0c2c46d16510596ee830cc97cb69b53992ed3fe6/django_dumpslow/management/__init__.py -------------------------------------------------------------------------------- /django_dumpslow/management/commands/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lamby/django-dumpslow/0c2c46d16510596ee830cc97cb69b53992ed3fe6/django_dumpslow/management/commands/__init__.py -------------------------------------------------------------------------------- /django_dumpslow/management/commands/dumpslow.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # 3 | # django-dumpslow -- Django application to log and summarize slow requests 4 | # 5 | # 6 | # Copyright © 2009-2019 Chris Lamb 7 | # 8 | # Redistribution and use in source and binary forms, with or without 9 | # modification, are permitted provided that the following conditions 10 | # are met: 11 | # 1. Redistributions of source code must retain the above copyright 12 | # notice, this list of conditions and the following disclaimer. 13 | # 2. Redistributions in binary form must reproduce the above copyright 14 | # notice, this list of conditions and the following disclaimer in the 15 | # documentation and/or other materials provided with the distribution. 16 | 17 | import time 18 | import redis 19 | from tabulate import tabulate 20 | 21 | from operator import itemgetter 22 | from optparse import make_option 23 | 24 | from django.conf import settings 25 | from django.core.management.base import BaseCommand, CommandError 26 | 27 | from django_dumpslow.utils import parse_interval 28 | 29 | class Command(BaseCommand): 30 | help = "Parse and summarize the django-dumpslow slow request log" 31 | 32 | def add_arguments(self, parser): 33 | parser.add_argument('-s', 34 | dest='order', 35 | default='at', 36 | help='what to sort by (at, count, average) (default: at)') 37 | parser.add_argument('-i', 38 | dest='after', 39 | default=0, 40 | help='interval to report on (eg. 3d 1w 1y) (default: all)') 41 | parser.add_argument('-r', 42 | action='store_true', 43 | dest='reverse', 44 | default=False, 45 | help='reverse the sort order (largest last instead of first)') 46 | parser.add_argument('-t', 47 | dest='limit', 48 | default=None, 49 | help='just show the top NUM queries') 50 | parser.add_argument('-m', 51 | dest='max_duration', 52 | default=20, 53 | help='ignore entries over SECS seconds (default: 20)') 54 | 55 | def handle(self, **options): 56 | def check_option(name, val): 57 | try: 58 | val = int(val) 59 | if val < 0: 60 | raise ValueError() 61 | return val 62 | except ValueError: 63 | raise CommandError( 64 | 'Invalid value for %s %r' % (name, val) 65 | ) 66 | except TypeError: 67 | pass 68 | 69 | limit = check_option('-t', options['limit']) 70 | max_duration = check_option('-m', options['max_duration']) 71 | 72 | after = options['after'] 73 | if after: 74 | try: 75 | after = int(time.time()) - parse_interval(after) 76 | except ValueError: 77 | raise CommandError('Invalid interval %r' % after) 78 | 79 | order = options['order'] 80 | if order not in ('at', 'count', 'average'): 81 | raise CommandError('Invalid sort order %r' % options['order']) 82 | 83 | if getattr(settings, 'REDIS_URL', None): 84 | client = redis.from_url(settings.REDIS_URL) 85 | else: 86 | client = redis.Redis( 87 | host=settings.REDIS_HOST, 88 | port=settings.REDIS_PORT, 89 | ) 90 | 91 | data = {} 92 | results = client.zrangebyscore( 93 | getattr(settings, 'DUMPSLOW_REDIS_KEY', 'dumpslow'), after, '+inf', 94 | ) 95 | 96 | for line in results: 97 | view, duration = line.split(b'\n', 1) 98 | 99 | duration = float(duration) 100 | 101 | if max_duration and duration >= max_duration: 102 | continue 103 | 104 | try: 105 | data[view]['at'] += duration 106 | data[view]['count'] += 1 107 | except KeyError: 108 | data[view] = {'at': duration, 'count': 1 } 109 | 110 | data[view]['average'] = data[view]['at'] / data[view]['count'] 111 | 112 | items = sorted(data.items(), key=lambda item: item[1][order], reverse=not options['reverse']) 113 | del data 114 | 115 | if limit is not None: 116 | items = items[:limit] 117 | 118 | headers=['View', 'Count', 'Accumulated time', 'Average time'] 119 | print(tabulate([[view, values['count'], values['at'], values['average']] for view, values in items], headers=headers)) 120 | -------------------------------------------------------------------------------- /django_dumpslow/middleware.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # 3 | # django-dumpslow -- Django application to log and summarize slow requests 4 | # 5 | # 6 | # Copyright © 2009-2019 Chris Lamb 7 | # 8 | # Redistribution and use in source and binary forms, with or without 9 | # modification, are permitted provided that the following conditions 10 | # are met: 11 | # 1. Redistributions of source code must retain the above copyright 12 | # notice, this list of conditions and the following disclaimer. 13 | # 2. Redistributions in binary form must reproduce the above copyright 14 | # notice, this list of conditions and the following disclaimer in the 15 | # documentation and/or other materials provided with the distribution. 16 | 17 | import sys 18 | import time 19 | import redis 20 | import threading 21 | 22 | from django.conf import settings 23 | from django.core.mail import mail_admins 24 | from django.utils.deprecation import MiddlewareMixin 25 | 26 | from django_dumpslow.utils import parse_interval 27 | 28 | class LogLongRequestMiddleware(MiddlewareMixin): 29 | def __init__(self, get_response=None): 30 | self.local = threading.local() 31 | super().__init__(get_response=get_response) 32 | 33 | def process_view(self, request, callback, callback_args, callback_kwargs): 34 | view = '%s.' % callback.__module__ 35 | 36 | try: 37 | view += callback.__name__ 38 | except (AttributeError, TypeError): 39 | # Some view functions (eg. class-based views) do not have a 40 | # __name__ attribute; try and get the name of its class 41 | view += callback.__class__.__name__ 42 | 43 | self.local.view = view 44 | self.local.start_time = time.time() 45 | 46 | def process_response(self, request, response): 47 | try: 48 | view = self.local.view 49 | time_taken = time.time() - self.local.start_time 50 | except AttributeError: 51 | # If, for whatever reason, the variables are not available, don't 52 | # do anything else. 53 | return response 54 | 55 | if time_taken < getattr(settings, 'DUMPSLOW_LONG_REQUEST_TIME', 1): 56 | return response 57 | 58 | if getattr(settings, 'REDIS_URL', None): 59 | client = redis.from_url(settings.REDIS_URL) 60 | else: 61 | client = redis.Redis( 62 | host=settings.REDIS_HOST, 63 | port=settings.REDIS_PORT, 64 | ) 65 | 66 | map_key = '%s\n%.3f' % (view, time_taken) 67 | mapping = { map_key: self.local.start_time } 68 | client.zadd( 69 | getattr(settings, 'DUMPSLOW_REDIS_KEY', 'dumpslow'), 70 | mapping, 71 | ) 72 | 73 | # Clean up old values 74 | 75 | delete_after = parse_interval( 76 | getattr(settings, 'DUMPSLOW_DELETE_AFTER', '4w'), 77 | ) 78 | 79 | client.zremrangebyscore( 80 | getattr(settings, 'DUMPSLOW_REDIS_KEY', 'dumpslow'), 81 | 0, 82 | int(time.time()) - delete_after, 83 | ) 84 | 85 | # If it was really slow, email admins. Disabled by default. 86 | email_threshold = getattr(settings, 'DUMPSLOW_EMAIL_REQUEST_TIME', sys.maxsize) 87 | if time_taken > email_threshold: 88 | mail_admins( 89 | "SLOW PAGE: %s" % request.path, 90 | "This page took %2.2f seconds to render, which is over the threshold " 91 | "threshold of %s.\n\n%s" % ( 92 | time_taken, 93 | email_threshold, 94 | str(request), 95 | ), 96 | ) 97 | 98 | return response 99 | -------------------------------------------------------------------------------- /django_dumpslow/models.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lamby/django-dumpslow/0c2c46d16510596ee830cc97cb69b53992ed3fe6/django_dumpslow/models.py -------------------------------------------------------------------------------- /django_dumpslow/utils.py: -------------------------------------------------------------------------------- 1 | import re 2 | import datetime 3 | 4 | def parse_interval(val): 5 | match = re.match(r'^(\d+)([smhdwy])$', val) 6 | if not match: 7 | raise ValueError() 8 | 9 | unit = { 10 | 's': 'seconds', 11 | 'm': 'minutes', 12 | 'h': 'hours', 13 | 'd': 'days', 14 | 'w': 'weeks', 15 | }[match.group(2)] 16 | 17 | td = datetime.timedelta(**{unit: int(match.group(1))}) 18 | 19 | return (td.days * 86400) + td.seconds 20 | -------------------------------------------------------------------------------- /django_dumpslow_example/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lamby/django-dumpslow/0c2c46d16510596ee830cc97cb69b53992ed3fe6/django_dumpslow_example/__init__.py -------------------------------------------------------------------------------- /django_dumpslow_example/example/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lamby/django-dumpslow/0c2c46d16510596ee830cc97cb69b53992ed3fe6/django_dumpslow_example/example/__init__.py -------------------------------------------------------------------------------- /django_dumpslow_example/example/models.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lamby/django-dumpslow/0c2c46d16510596ee830cc97cb69b53992ed3fe6/django_dumpslow_example/example/models.py -------------------------------------------------------------------------------- /django_dumpslow_example/example/templates/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | django-dumpslow demo 4 | 21 | 22 | 23 | 24 |

django-dumpslow demo

25 | 26 |

Artificially slow page

27 | 31 | 32 | 33 | 34 | -------------------------------------------------------------------------------- /django_dumpslow_example/example/urls.py: -------------------------------------------------------------------------------- 1 | from django.conf.urls.defaults import * 2 | 3 | from .views import slow2 4 | 5 | urlpatterns = patterns('example.views', 6 | url(r'^$', 'index', name='index'), 7 | url(r'^slow$', 'slow', name='slow'), 8 | url(r'^slow2$', slow2(), name='slow2'), 9 | ) 10 | -------------------------------------------------------------------------------- /django_dumpslow_example/example/views.py: -------------------------------------------------------------------------------- 1 | import time 2 | 3 | from django.http import HttpResponse 4 | from django.shortcuts import render_to_response 5 | 6 | def index(request): 7 | return render_to_response('index.html') 8 | 9 | def slow(request): 10 | time.sleep(2) 11 | return HttpResponse('This page should have taken >=2s to render.') 12 | 13 | class slow2(object): 14 | def __call__(self, request): 15 | time.sleep(5) 16 | return HttpResponse('This page should have taken >=5s to render.') 17 | -------------------------------------------------------------------------------- /django_dumpslow_example/manage.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | from django.core.management import execute_manager 3 | try: 4 | import settings # Assumed to be in the same directory. 5 | except ImportError: 6 | import sys 7 | sys.stderr.write("Error: Can't find the file 'settings.py' in the directory containing %r. It appears you've customized things.\nYou'll have to run django-admin.py, passing it your settings module.\n(If the file settings.py does indeed exist, it's causing an ImportError somehow.)\n" % __file__) 8 | sys.exit(1) 9 | 10 | if __name__ == "__main__": 11 | execute_manager(settings) 12 | -------------------------------------------------------------------------------- /django_dumpslow_example/settings.py: -------------------------------------------------------------------------------- 1 | # Django settings for example project. 2 | 3 | DEBUG = True 4 | TEMPLATE_DEBUG = DEBUG 5 | 6 | ADMINS = ( 7 | # ('Your Name', 'your_email@domain.com'), 8 | ) 9 | 10 | MANAGERS = ADMINS 11 | 12 | DATABASE_ENGINE = '' # 'postgresql_psycopg2', 'postgresql', 'mysql', 'sqlite3' or 'oracle'. 13 | DATABASE_NAME = '' # Or path to database file if using sqlite3. 14 | DATABASE_USER = '' # Not used with sqlite3. 15 | DATABASE_PASSWORD = '' # Not used with sqlite3. 16 | DATABASE_HOST = '' # Set to empty string for localhost. Not used with sqlite3. 17 | DATABASE_PORT = '' # Set to empty string for default. Not used with sqlite3. 18 | 19 | # Local time zone for this installation. Choices can be found here: 20 | # http://en.wikipedia.org/wiki/List_of_tz_zones_by_name 21 | # although not all choices may be available on all operating systems. 22 | # If running in a Windows environment this must be set to the same as your 23 | # system time zone. 24 | TIME_ZONE = 'America/Chicago' 25 | 26 | # Language code for this installation. All choices can be found here: 27 | # http://www.i18nguy.com/unicode/language-identifiers.html 28 | LANGUAGE_CODE = 'en-us' 29 | 30 | SITE_ID = 1 31 | 32 | # If you set this to False, Django will make some optimizations so as not 33 | # to load the internationalization machinery. 34 | USE_I18N = True 35 | 36 | # Absolute path to the directory that holds media. 37 | # Example: "/home/media/media.lawrence.com/" 38 | MEDIA_ROOT = '' 39 | 40 | # URL that handles the media served from MEDIA_ROOT. Make sure to use a 41 | # trailing slash if there is a path component (optional in other cases). 42 | # Examples: "http://media.lawrence.com", "http://example.com/media/" 43 | MEDIA_URL = '' 44 | 45 | # URL prefix for admin media -- CSS, JavaScript and images. Make sure to use a 46 | # trailing slash. 47 | # Examples: "http://foo.com/media/", "/media/". 48 | ADMIN_MEDIA_PREFIX = '/media/' 49 | 50 | # Make this unique, and don't share it with anybody. 51 | SECRET_KEY = '^hx*32ty@5fe(uas%vw_w&-s$+o%i^(jk%2me)bqqr8ise)5br' 52 | 53 | # List of callables that know how to import templates from various sources. 54 | TEMPLATE_LOADERS = ( 55 | 'django.template.loaders.filesystem.load_template_source', 56 | 'django.template.loaders.app_directories.load_template_source', 57 | # 'django.template.loaders.eggs.load_template_source', 58 | ) 59 | 60 | MIDDLEWARE_CLASSES = ( 61 | 'django_dumpslow.middleware.LogLongRequestMiddleware', 62 | 'django.middleware.common.CommonMiddleware', 63 | 'django.contrib.sessions.middleware.SessionMiddleware', 64 | 'django.contrib.auth.middleware.AuthenticationMiddleware', 65 | ) 66 | 67 | ROOT_URLCONF = 'urls' 68 | 69 | TEMPLATE_DIRS = ( 70 | # Put strings here, like "/home/html/django_templates" or "C:/www/django/templates". 71 | # Always use forward slashes, even on Windows. 72 | # Don't forget to use absolute paths, not relative paths. 73 | ) 74 | 75 | import sys 76 | from os.path import abspath, dirname 77 | sys.path.insert(0, dirname(dirname(abspath(__file__)))) 78 | 79 | INSTALLED_APPS = ( 80 | 'django.contrib.auth', 81 | 'django.contrib.contenttypes', 82 | 'django.contrib.sessions', 83 | 'django.contrib.sites', 84 | 'django_dumpslow', 85 | 'example', 86 | ) 87 | 88 | ###################### 89 | 90 | # How long a request has to take to be considered too long. Default is 1 second. 91 | # DUMPSLOW_LONG_REQUEST_TIME = 0.5 92 | # Email admins if the page takes longer than this to render 93 | # DUMPSLOW_EMAIL_REQUEST_TIME = 10 94 | 95 | REDIS_HOST = 'localhost' 96 | REDIS_PORT = 6379 97 | 98 | # Name of the redis key to store data in. Use this if you share your redis 99 | # instance between Django projects using dumpslow. 100 | # DUMPSLOW_REDIS_KEY = 'dumpslow' 101 | 102 | # Delete dumpslow entries that are older than the specified interval 103 | # DUMPSLOW_DELETE_AFTER = '4w' 104 | -------------------------------------------------------------------------------- /django_dumpslow_example/urls.py: -------------------------------------------------------------------------------- 1 | from django.conf.urls.defaults import * 2 | 3 | urlpatterns = patterns('', 4 | (r'^', include('example.urls')), 5 | ) 6 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # 4 | # django-dumpslow -- Django application to log and summarize slow requests 5 | # 6 | # 7 | # Copyright © 2009-2019 Chris Lamb 8 | # 9 | # Redistribution and use in source and binary forms, with or without 10 | # modification, are permitted provided that the following conditions 11 | # are met: 12 | # 1. Redistributions of source code must retain the above copyright 13 | # notice, this list of conditions and the following disclaimer. 14 | # 2. Redistributions in binary form must reproduce the above copyright 15 | # notice, this list of conditions and the following disclaimer in the 16 | # documentation and/or other materials provided with the distribution. 17 | 18 | setup_args = dict( 19 | name='django-dumpslow', 20 | version='3.1.0', 21 | license='BSD-2-Clause', 22 | url='https://chris-lamb.co.uk/projects/django-dumpslow', 23 | packages=( 24 | 'django_dumpslow', 25 | 'django_dumpslow.management', 26 | 'django_dumpslow.management.commands', 27 | ), 28 | author='Chris Lamb', 29 | author_email='chris@chris-lamb.co.uk', 30 | description="Django application to log and summarize slow requests", 31 | 32 | install_requires=( 33 | 'Django>=1.11', 34 | 'redis>=3.0', 35 | 'tabulate', 36 | ), 37 | ) 38 | 39 | try: 40 | from setuptools import setup 41 | except ImportError: 42 | from distutils.core import setup 43 | 44 | setup(**setup_args) 45 | --------------------------------------------------------------------------------