├── src └── timelog │ ├── __init__.py │ ├── management │ ├── __init__.py │ └── commands │ │ ├── __init__.py │ │ └── analyze_timelog.py │ ├── middleware.py │ └── lib.py ├── .gitignore ├── AUTHORS ├── CHANGES.txt ├── setup.py ├── LICENSE └── README.textile /src/timelog/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /src/timelog/management/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /src/timelog/management/commands/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .DS_Store 2 | *~ 3 | *.pyc 4 | build/* 5 | dist/* 6 | src/*.egg-info/* 7 | -------------------------------------------------------------------------------- /AUTHORS: -------------------------------------------------------------------------------- 1 | Gareth Rushgrove 2 | Ross Poulton 3 | Mikhail Korobov 4 | -------------------------------------------------------------------------------- /CHANGES.txt: -------------------------------------------------------------------------------- 1 | Changes 2 | ======= 3 | 4 | 0.4 (2011-06-23) 5 | ----------------------- 6 | 7 | * Don't swallow an exception if it happens earlier in the middleware stack 8 | 9 | 0.3 (2011-06-22) 10 | ----------------------- 11 | 12 | * Fix dependency regression in previous release 13 | 14 | 0.2 (2011-06-22) 15 | ----------------------- 16 | 17 | * Support Python versions odler than 2.6 18 | * Add ability to set URIs to be excluded from the report 19 | 20 | 0.1 (2011-06-09) 21 | ----------------------- 22 | 23 | * Initial working release 24 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | from setuptools import setup, find_packages 3 | 4 | setup( 5 | name = "django-timelog", 6 | version = "0.4", 7 | author = "Gareth Rushgrove", 8 | author_email = "gareth@morethanseven.net", 9 | url = "http://github.com/garethr/django-timelog/", 10 | 11 | packages = find_packages('src'), 12 | package_dir = {'':'src'}, 13 | license = "MIT License", 14 | keywords = "django", 15 | description = "Performance logging middlware and analysis tools for Django", 16 | install_requires=[ 17 | 'texttable', 18 | 'progressbar', 19 | ], 20 | classifiers = [ 21 | "Intended Audience :: Developers", 22 | "License :: OSI Approved :: MIT License", 23 | 'Operating System :: OS Independent', 24 | 'Programming Language :: Python', 25 | ] 26 | ) 27 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2011 Gareth Rushgrove 2 | 3 | Permission is hereby granted, free of charge, to any person 4 | obtaining a copy of this software and associated documentation 5 | files (the "Software"), to deal in the Software without 6 | restriction, including without limitation the rights to use, 7 | copy, modify, merge, publish, distribute, sublicense, and/or sell 8 | copies of the Software, and to permit persons to whom the 9 | Software is furnished to do so, subject to the following 10 | conditions: 11 | 12 | The above copyright notice and this permission notice shall be 13 | included in all copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 16 | EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES 17 | OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 18 | NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT 19 | HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, 20 | WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 21 | FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 22 | OTHER DEALINGS IN THE SOFTWARE. 23 | -------------------------------------------------------------------------------- /src/timelog/management/commands/analyze_timelog.py: -------------------------------------------------------------------------------- 1 | from optparse import make_option 2 | 3 | from django.core.management.base import BaseCommand 4 | from django.conf import settings 5 | 6 | from timelog.lib import generate_table_from, analyze_log_file, PATTERN 7 | 8 | 9 | class Command(BaseCommand): 10 | 11 | option_list = BaseCommand.option_list + ( 12 | make_option('--file', 13 | dest='file', 14 | default=settings.TIMELOG_LOG, 15 | help='Specify file to use'), 16 | make_option('--noreverse', 17 | dest='reverse', 18 | action='store_false', 19 | default=True, 20 | help='Show paths instead of views'), 21 | ) 22 | 23 | def handle(self, *args, **options): 24 | # --date-from YY-MM-DD 25 | # specify a date filter start 26 | # default to first date in file 27 | # --date-to YY-MM-DD 28 | # specify a date filter end 29 | # default to now 30 | 31 | LOGFILE = options.get('file') 32 | 33 | try: 34 | data = analyze_log_file(LOGFILE, PATTERN, reverse_paths=options.get('reverse')) 35 | except IOError: 36 | print "File not found" 37 | exit(2) 38 | 39 | print generate_table_from(data) 40 | -------------------------------------------------------------------------------- /src/timelog/middleware.py: -------------------------------------------------------------------------------- 1 | import time 2 | import logging 3 | from django.db import connection 4 | from django.utils.encoding import smart_str 5 | 6 | logger = logging.getLogger(__name__) 7 | 8 | 9 | class TimeLogMiddleware(object): 10 | 11 | def process_request(self, request): 12 | request._start = time.time() 13 | 14 | def process_response(self, request, response): 15 | # if an exception is occured in a middleware listed 16 | # before TimeLogMiddleware then request won't have '_start' attribute 17 | # and the original traceback will be lost (original exception will be 18 | # replaced with AttributeError) 19 | 20 | sqltime = 0.0 21 | 22 | for q in connection.queries: 23 | sqltime += float(getattr(q, 'time', 0.0)) 24 | 25 | if hasattr(request, '_start'): 26 | d = { 27 | 'method': request.method, 28 | 'time': time.time() - request._start, 29 | 'code': response.status_code, 30 | 'url': smart_str(request.path_info), 31 | 'sql': len(connection.queries), 32 | 'sqltime': sqltime, 33 | } 34 | msg = '%(method)s "%(url)s" (%(code)s) %(time).2f (%(sql)dq, %(sqltime).4f)' % d 35 | logger.info(msg) 36 | return response 37 | -------------------------------------------------------------------------------- /README.textile: -------------------------------------------------------------------------------- 1 | A simple django middleware that logs request times using the Django 1.3 logging support, and a management command to analyze the resulting data. Once installed and configured you can run a command line these: 2 | 3 | 
python manage.py analyze_timelog
 4 | python manage.py analyze_timelog --noreverse
5 | 6 | And generate useful tabular data like this: 7 | 8 | 
 9 | +--------------------------+--------+--------+-------+---------+---------+-------+-----------------+
10 | | view                     | method | status | count | minimum | maximum | mean  | stdev           |
11 | +--------------------------+--------+--------+-------+---------+---------+-------+-----------------+
12 | | boxes.viewsBoxDetailView | GET    | 200    | 9430  | 0.14    | 0.28    | 0.21  | 0.0700037118541 |
13 | +--------------------------+--------+--------+-------+---------+---------+-------+-----------------+
14 | | boxes.viewsBoxListView   | GET    | 200    | 66010 | 0.17    | 0.28    | 0.232 | 0.0455415351076 |
15 | +--------------------------+--------+--------+-------+---------+---------+-------+-----------------+
16 | | django.views.staticserve | GET    | 200    | 61295 | 0.00    | 0.02    | 0.007 | 0.0060574669888 |
17 | +--------------------------+--------+--------+-------+---------+---------+-------+-----------------+
18 |
19 | 20 | This project was heavily influenced by the Rails "Request log analyzer":https://github.com/wvanbergen/request-log-analyzer. 21 | 22 | h2. Installation 23 | 24 | 
pip install django-timelog
25 | 26 | Once installed you need to do a little configuration to get things working. First add the middleware to your MIDDLEWARE_CLASSES in your settings file. 27 | 28 | 
MIDDLEWARE_CLASSES = (
29 |   'timelog.middleware.TimeLogMiddleware',
30 | 31 | Next add timelog to your INSTALLED_APPS list. This is purely for the management command discovery. 32 | 33 | 
INSTALLED_APPS = (
34 |   'timelog',
35 | 36 | Then configure the logger you want to use. This really depends on what you want to do, the django 1.3 logging setup is pretty powerful. Here’s how I’ve got logging setup as an example: 37 | 38 | 
TIMELOG_LOG = '/path/to/logs/timelog.log'
39 | 
40 | LOGGING = {
41 |   'version': 1,
42 |   'formatters': {
43 |     'plain': {
44 |       'format': '%(asctime)s %(message)s'},
45 |     },
46 |   'handlers': {
47 |     'timelog': {
48 |       'level': 'DEBUG',
49 |       'class': 'logging.handlers.RotatingFileHandler',
50 |       'filename': TIMELOG_LOG,
51 |       'maxBytes': 1024 * 1024 * 5,  # 5 MB
52 |       'backupCount': 5,
53 |       'formatter': 'plain',
54 |     },
55 |   },
56 |   'loggers': {
57 |     'timelog.middleware': {
58 |       'handlers': ['timelog'],
59 |       'level': 'DEBUG',
60 |       'propogate': False,
61 |      }
62 |   }
63 | }
64 | 65 | Lastly, if you have particular URIs you wish to ignore you can define them using basic regular expressions in the TIMELOG_IGNORE_URIS list in settings.py: 66 | 67 | 
TIMELOG_IGNORE_URIS = (
68 |     '^/admin/',         # Ignores all URIs beginning with '/admin/'
69 |     '^/other_page/$',   # Ignores the URI '/other_page/' only, but not '/other_page/more/'.
70 |     '.jpg$',            # Ignores all URIs ending in .jpg
71 | )
72 | -------------------------------------------------------------------------------- /src/timelog/lib.py: -------------------------------------------------------------------------------- 1 | import fileinput 2 | from re import compile 3 | from django.conf import settings 4 | 5 | from texttable import Texttable 6 | from progressbar import ProgressBar, Percentage, Bar 7 | 8 | from django.core.urlresolvers import resolve, Resolver404 9 | 10 | PATTERN = r"""^([0-9]{4}-[0-9]{2}-[0-9]{2} [0-9:]{8},[0-9]{3}) (GET|POST|PUT|DELETE|HEAD) "(.*)" \((.*)\) (.*?) \((\d+)q, (.*?)\)""" 11 | 12 | CACHED_VIEWS = {} 13 | 14 | IGNORE_PATHS = getattr(settings, 'TIMELOG_IGNORE_URIS', ()) 15 | 16 | def count_lines_in(filename): 17 | "Count lines in a file" 18 | f = open(filename) 19 | lines = 0 20 | buf_size = 1024 * 1024 21 | read_f = f.read # loop optimization 22 | 23 | buf = read_f(buf_size) 24 | while buf: 25 | lines += buf.count('\n') 26 | buf = read_f(buf_size) 27 | 28 | return lines 29 | 30 | def view_name_from(path): 31 | "Resolve a path to the full python module name of the related view function" 32 | try: 33 | return CACHED_VIEWS[path] 34 | 35 | except KeyError: 36 | view = resolve(path) 37 | module = path 38 | name = '' 39 | if hasattr(view.func, '__module__'): 40 | module = resolve(path).func.__module__ 41 | if hasattr(view.func, '__name__'): 42 | name = resolve(path).func.__name__ 43 | 44 | view = "%s.%s" % (module, name) 45 | CACHED_VIEWS[path] = view 46 | return view 47 | 48 | def generate_table_from(data): 49 | "Output a nicely formatted ascii table" 50 | table = Texttable(max_width=120) 51 | table.add_row(["view", "method", "status", "count", "minimum", "maximum", "mean", "stdev", "queries", "querytime"]) 52 | table.set_cols_align(["l", "l", "l", "r", "r", "r", "r", "r", "r", "r"]) 53 | 54 | for item in sorted(data): 55 | mean = round(sum(data[item]['times'])/data[item]['count'], 3) 56 | 57 | mean_sql = round(sum(data[item]['sql'])/data[item]['count'], 3) 58 | mean_sqltime = round(sum(data[item]['sqltime'])/data[item]['count'], 3) 59 | 60 | sdsq = sum([(i - mean) ** 2 for i in data[item]['times']]) 61 | try: 62 | stdev = '%.2f' % ((sdsq / (len(data[item]['times']) - 1)) ** .5) 63 | except ZeroDivisionError: 64 | stdev = '0.00' 65 | 66 | minimum = "%.2f" % min(data[item]['times']) 67 | maximum = "%.2f" % max(data[item]['times']) 68 | table.add_row([data[item]['view'], data[item]['method'], data[item]['status'], data[item]['count'], minimum, maximum, '%.3f' % mean, stdev, mean_sql, mean_sqltime]) 69 | 70 | return table.draw() 71 | 72 | def analyze_log_file(logfile, pattern, reverse_paths=True, progress=True): 73 | "Given a log file and regex group and extract the performance data" 74 | if progress: 75 | lines = count_lines_in(logfile) 76 | pbar = ProgressBar(widgets=[Percentage(), Bar()], maxval=lines+1).start() 77 | counter = 0 78 | 79 | data = {} 80 | 81 | compiled_pattern = compile(pattern) 82 | for line in fileinput.input([logfile]): 83 | 84 | if progress: 85 | counter = counter + 1 86 | 87 | parsed = compiled_pattern.findall(line)[0] 88 | date = parsed[0] 89 | method = parsed[1] 90 | path = parsed[2] 91 | status = parsed[3] 92 | time = parsed[4] 93 | sql = parsed[5] 94 | sqltime = parsed[6] 95 | 96 | try: 97 | ignore = False 98 | for ignored_path in IGNORE_PATHS: 99 | compiled_path = compile(ignored_path) 100 | if compiled_path.match(path): 101 | ignore = True 102 | if not ignore: 103 | if reverse_paths: 104 | view = view_name_from(path) 105 | else: 106 | view = path 107 | key = "%s-%s-%s" % (view, status, method) 108 | try: 109 | data[key]['count'] = data[key]['count'] + 1 110 | data[key]['times'].append(float(time)) 111 | data[key]['sql'].append(int(sql)) 112 | data[key]['sqltime'].append(float(sqltime)) 113 | except KeyError: 114 | data[key] = { 115 | 'count': 1, 116 | 'status': status, 117 | 'view': view, 118 | 'method': method, 119 | 'times': [float(time)], 120 | 'sql': [int(sql)], 121 | 'sqltime': [float(sqltime)], 122 | } 123 | except Resolver404: 124 | pass 125 | 126 | if progress: 127 | pbar.update(counter) 128 | 129 | if progress: 130 | pbar.finish() 131 | 132 | return data 133 | --------------------------------------------------------------------------------