├── qsstats ├── models.py ├── compat.py ├── exceptions.py ├── utils.py ├── tests.py └── __init__.py ├── test_settings ├── mysql_tz.py ├── sqlite_tz.py ├── postgres_tz.py ├── __init__.py ├── sqlite.py ├── mysql.py └── postgres.py ├── MANIFEST.in ├── .hgtags ├── AUTHORS.rst ├── .travis.yml ├── .hgignore ├── .gitignore ├── setup.py ├── LICENSE ├── tox.ini └── README.rst /qsstats/models.py: -------------------------------------------------------------------------------- 1 | # Hello, testrunner! 2 | -------------------------------------------------------------------------------- /test_settings/mysql_tz.py: -------------------------------------------------------------------------------- 1 | from .mysql import * 2 | USE_TZ = True -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include *.txt 2 | include *.rst 3 | include LICENSE 4 | -------------------------------------------------------------------------------- /test_settings/sqlite_tz.py: -------------------------------------------------------------------------------- 1 | from .sqlite import * 2 | USE_TZ = True -------------------------------------------------------------------------------- /test_settings/postgres_tz.py: -------------------------------------------------------------------------------- 1 | from .postgres import * 2 | USE_TZ = True -------------------------------------------------------------------------------- /test_settings/__init__.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | from __future__ import absolute_import -------------------------------------------------------------------------------- /.hgtags: -------------------------------------------------------------------------------- 1 | 2e207697bf89c48d3782f2cdef087d6bd9e30f12 0.7 2 | 594cb073eb2bf969102d2dcf93220a9584b0b4d7 0.7.1 3 | 7746dfab97663286f2d64ee26689ca18a824d2fb 0.7.2 4 | -------------------------------------------------------------------------------- /AUTHORS.rst: -------------------------------------------------------------------------------- 1 | Authors & Contributors 2 | ====================== 3 | 4 | * Matt Croydon; 5 | * Mikhail Korobov; 6 | * Pawel Tomasiewicz; 7 | * Steve Jones; 8 | * Petr Dlouhý; 9 | * @ivirabyan. 10 | -------------------------------------------------------------------------------- /qsstats/compat.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | from __future__ import absolute_import 3 | 4 | import datetime 5 | try: 6 | from django.utils.timezone import now 7 | except ImportError: 8 | now = datetime.datetime.now 9 | -------------------------------------------------------------------------------- /test_settings/sqlite.py: -------------------------------------------------------------------------------- 1 | INSTALLED_APPS = ( 2 | 'qsstats', 3 | 'django.contrib.auth', 4 | 'django.contrib.contenttypes' 5 | ) 6 | 7 | DATABASES = { 8 | 'default': { 9 | 'ENGINE': 'django.db.backends.sqlite3', 10 | 'NAME': 'test' 11 | } 12 | } 13 | 14 | SECRET_KEY = 'foo' -------------------------------------------------------------------------------- /qsstats/exceptions.py: -------------------------------------------------------------------------------- 1 | class QuerySetStatsError(Exception): 2 | pass 3 | 4 | class InvalidInterval(QuerySetStatsError): 5 | pass 6 | 7 | class InvalidOperator(QuerySetStatsError): 8 | pass 9 | 10 | class DateFieldMissing(QuerySetStatsError): 11 | pass 12 | 13 | class QuerySetMissing(QuerySetStatsError): 14 | pass 15 | -------------------------------------------------------------------------------- /test_settings/mysql.py: -------------------------------------------------------------------------------- 1 | INSTALLED_APPS = ( 2 | 'qsstats', 3 | 'django.contrib.auth', 4 | 'django.contrib.contenttypes' 5 | ) 6 | 7 | DATABASES = { 8 | 'default': { 9 | 'ENGINE': 'django.db.backends.mysql', 10 | 'NAME': 'qsstats_test', 11 | 'USER': 'root', 12 | 'PASSWORD': '', 13 | } 14 | } 15 | 16 | SECRET_KEY = 'foo' 17 | -------------------------------------------------------------------------------- /test_settings/postgres.py: -------------------------------------------------------------------------------- 1 | INSTALLED_APPS = ( 2 | 'qsstats', 3 | 'django.contrib.auth', 4 | 'django.contrib.contenttypes' 5 | ) 6 | 7 | DATABASES = { 8 | 'default': { 9 | 'ENGINE': 'django.db.backends.postgresql_psycopg2', 10 | 'NAME': 'qsstats_test', 11 | 'USER': 'postgres', 12 | 'PASSWORD': '', 13 | } 14 | } 15 | 16 | SECRET_KEY = 'foo' 17 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | dist: xenial 2 | sudo: false 3 | language: python 4 | services: 5 | - mysql 6 | - postgresql 7 | python: 8 | - "2.7" 9 | - "3.5" 10 | - "3.6" 11 | - "3.7" 12 | install: pip install tox-travis 13 | script: tox 14 | before_script: 15 | - mysql -e 'create database qsstats_test;' 16 | - psql -c 'create database qsstats_test;' -U postgres 17 | - mysql_tzinfo_to_sql /usr/share/zoneinfo | mysql -u root mysql 18 | -------------------------------------------------------------------------------- /.hgignore: -------------------------------------------------------------------------------- 1 | syntax: glob 2 | 3 | #IDE files 4 | .settings/* 5 | .project 6 | .pydevproject 7 | .cache/* 8 | 9 | #temp files 10 | *.pyc 11 | *.pyo 12 | *.orig 13 | *.swp 14 | *~ 15 | .tox 16 | 17 | #misc files 18 | pip-log.txt 19 | 20 | #os files 21 | .DS_Store 22 | Thumbs.db 23 | 24 | #doc files 25 | docs/_build/doctrees/ 26 | 27 | #setup files 28 | build/ 29 | dist/ 30 | .build/ 31 | MANIFEST 32 | django_qsstats_magic.egg-info 33 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | env/ 12 | build/ 13 | develop-eggs/ 14 | dist/ 15 | downloads/ 16 | eggs/ 17 | .eggs/ 18 | lib/ 19 | lib64/ 20 | parts/ 21 | sdist/ 22 | var/ 23 | *.egg-info/ 24 | .installed.cfg 25 | *.egg 26 | 27 | # PyInstaller 28 | # Usually these files are written by a python script from a template 29 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 30 | *.manifest 31 | *.spec 32 | 33 | # Installer logs 34 | pip-log.txt 35 | pip-delete-this-directory.txt 36 | 37 | # Unit test / coverage reports 38 | htmlcov/ 39 | .tox/ 40 | .coverage 41 | .coverage.* 42 | .cache 43 | nosetests.xml 44 | coverage.xml 45 | *,cover 46 | 47 | # Translations 48 | #*.mo 49 | #*.pot 50 | 51 | # Django stuff: 52 | *.log 53 | 54 | # Sphinx documentation 55 | docs/_build/ 56 | 57 | # PyBuilder 58 | target/ 59 | 60 | # PyCharm 61 | .idea/ 62 | 63 | *.db 64 | *.sqlite3 65 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | from distutils.core import setup 3 | 4 | for cmd in ('egg_info', 'develop'): 5 | import sys 6 | if cmd in sys.argv: 7 | from setuptools import setup 8 | 9 | setup( 10 | name='django-qsstats-magic', 11 | version='1.1.0', 12 | description='A django microframework that eases the generation of aggregate data for querysets.', 13 | long_description = open('README.rst').read(), 14 | author='Matt Croydon, Mikhail Korobov', 15 | author_email='mcroydon@gmail.com, kmike84@gmail.com', 16 | url='https://github.com/PetrDlouhy/django-qsstats-magic', 17 | packages=['qsstats'], 18 | requires=['dateutil(>=1.4.1, < 2.0)', 'six'], 19 | classifiers=[ 20 | 'Development Status :: 4 - Beta', 21 | 'Environment :: Web Environment', 22 | 'Framework :: Django', 23 | 'Intended Audience :: Developers', 24 | 'License :: OSI Approved :: MIT License', 25 | 'Programming Language :: Python', 26 | 'Programming Language :: Python :: 2', 27 | 'Programming Language :: Python :: 2.7', 28 | 'Programming Language :: Python :: 3.4', 29 | 'Programming Language :: Python :: 3.5', 30 | 'Programming Language :: Python :: 3.6', 31 | 'Topic :: Software Development :: Libraries :: Python Modules', 32 | ], 33 | ) 34 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2010, Matt Croydon, Mikhail Korobov 2 | All rights reserved. 3 | 4 | Redistribution and use in source and binary forms, with or without 5 | modification, are permitted provided that the following conditions are met: 6 | * Redistributions of source code must retain the above copyright 7 | notice, this list of conditions and the following disclaimer. 8 | * Redistributions in binary form must reproduce the above copyright 9 | notice, this list of conditions and the following disclaimer in the 10 | documentation and/or other materials provided with the distribution. 11 | * Neither the name of the tastypie nor the 12 | names of its contributors may be used to endorse or promote products 13 | derived from this software without specific prior written permission. 14 | 15 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND 16 | ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 17 | WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 18 | DISCLAIMED. IN NO EVENT SHALL MATT CROYDON BE LIABLE FOR ANY 19 | DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 20 | (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 21 | LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND 22 | ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 23 | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 24 | SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 25 | -------------------------------------------------------------------------------- /tox.ini: -------------------------------------------------------------------------------- 1 | [tox] 2 | envlist = 3 | py{27,34,35,36,37}-dj{111}-{sqlite,postgres,mysql,postgres_tz,mysql_tz,sqlite_tz,mysql_tz_nopytz,postgres_tz_nopytz,sqlite_tz_nopytz}, 4 | py{35,36,37}-dj{20,21,22}-{sqlite,postgres,mysql,postgres_tz,mysql_tz,sqlite_tz,mysql_tz_nopytz,postgres_tz_nopytz,sqlite_tz_nopytz}, 5 | py{36,37}-dj{dev}-{sqlite,postgres,mysql,postgres_tz,mysql_tz,sqlite_tz,mysql_tz_nopytz,postgres_tz_nopytz,sqlite_tz_nopytz} 6 | 7 | [testenv] 8 | setenv = 9 | PYTHONPATH = {toxinidir} 10 | basepython = 11 | py27: python2.7 12 | py36: python3.6 13 | py35: python3.5 14 | py36: python3.6 15 | py37: python3.7 16 | deps= 17 | python-dateutil 18 | dj111: Django>=1.11,<2.0 19 | dj20: Django>=2.0,<2.1 20 | dj21: Django>=2.1,<2.2 21 | dj22: Django>=2.2,<3.0 22 | djdev: https://github.com/django/django/archive/master.tar.gz 23 | {postgres,postgres_tz,postgres_tz_nopytz}: psycopg2 24 | {postgres_tz,mysql_tz,sqlite_tz}: pytz 25 | {mysql,mysql_tz,mysql_tz_nopytz}: mysqlclient 26 | commands= 27 | sqlite: django-admin.py test qsstats --settings=test_settings.sqlite [] 28 | {sqlite_tz,sqlite_tz_nopytz}: django-admin.py test qsstats --settings=test_settings.sqlite_tz [] 29 | postgres: django-admin.py test qsstats --settings=test_settings.postgres [] 30 | {postgres_tz,postgres_tz_nopytz}: django-admin.py test qsstats --settings=test_settings.postgres_tz [] 31 | mysql: django-admin.py test qsstats --settings=test_settings.mysql [] 32 | {mysql_tz,mysql_tz_nopytz}: django-admin.py test qsstats --settings=test_settings.mysql_tz [] 33 | -------------------------------------------------------------------------------- /qsstats/utils.py: -------------------------------------------------------------------------------- 1 | import datetime 2 | import re 3 | from dateutil.relativedelta import relativedelta, MO 4 | from qsstats.exceptions import InvalidInterval 5 | from qsstats import compat 6 | 7 | def _remove_time(dt): 8 | tzinfo = getattr(dt, 'tzinfo', compat.now().tzinfo) 9 | return datetime.datetime(dt.year, dt.month, dt.day, tzinfo=tzinfo) 10 | 11 | def _to_datetime(dt): 12 | if isinstance(dt, datetime.datetime): 13 | return dt 14 | return _remove_time(dt) 15 | 16 | def _parse_interval(interval): 17 | num = 1 18 | match = re.match(r'(\d+)([A-Za-z]+)', interval) 19 | 20 | if match: 21 | num = int(match.group(1)) 22 | interval = match.group(2) 23 | return num, interval 24 | 25 | def get_bounds(dt, interval): 26 | ''' Returns interval bounds the datetime is in. ''' 27 | 28 | day = _to_datetime(_remove_time(dt)) 29 | dt = _to_datetime(dt) 30 | 31 | if interval == 'minute': 32 | begin = datetime.datetime(dt.year, dt.month, dt.day, dt.hour, dt.minute, tzinfo=dt.tzinfo) 33 | end = begin + relativedelta(minutes=1) 34 | elif interval == 'hour': 35 | begin = datetime.datetime(dt.year, dt.month, dt.day, dt.hour, tzinfo=dt.tzinfo) 36 | end = begin + relativedelta(hours=1) 37 | elif interval == 'day': 38 | begin = day 39 | end = day + relativedelta(days=1) 40 | elif interval == 'week': 41 | begin = day - relativedelta(weekday=MO(-1)) 42 | end = begin + datetime.timedelta(days=7) 43 | elif interval == 'month': 44 | begin = datetime.datetime(dt.year, dt.month, 1, tzinfo=dt.tzinfo) 45 | end = begin + relativedelta(months=1) 46 | elif interval == 'year': 47 | begin = datetime.datetime(dt.year, 1, 1, tzinfo=dt.tzinfo) 48 | end = datetime.datetime(dt.year+1, 1, 1, tzinfo=dt.tzinfo) 49 | else: 50 | raise InvalidInterval('Inverval not supported.') 51 | end = end - relativedelta(microseconds=1) 52 | return begin, end 53 | -------------------------------------------------------------------------------- /qsstats/tests.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | import datetime 3 | 4 | from django.test import TestCase 5 | from django.contrib.auth.models import User 6 | from qsstats import QuerySetStats, InvalidInterval, DateFieldMissing, QuerySetMissing 7 | from qsstats import compat 8 | from .utils import _remove_time 9 | 10 | class QuerySetStatsTestCase(TestCase): 11 | def test_basic_today(self): 12 | # We'll be making sure that this user is found 13 | u1 = User.objects.create_user('u1', 'u1@example.com') 14 | # And that this user is not 15 | u2 = User.objects.create_user('u2', 'u2@example.com') 16 | u2.is_active = False 17 | u2.save() 18 | 19 | # Create a QuerySet and QuerySetStats 20 | qs = User.objects.filter(is_active=True) 21 | qss = QuerySetStats(qs, 'date_joined') 22 | 23 | # We should only see a single user 24 | self.assertEqual(qss.this_day(), 1) 25 | 26 | def assertTimeSeriesWorks(self, today): 27 | seven_days_ago = today - datetime.timedelta(days=7) 28 | for j in range(1,8): 29 | for i in range(0,j): 30 | u = User.objects.create_user('p-%s-%s' % (j, i), 'p%s-%s@example.com' % (j, i)) 31 | u.date_joined = today - datetime.timedelta(days=i) 32 | u.save() 33 | qs = User.objects.all() 34 | qss = QuerySetStats(qs, 'date_joined') 35 | time_series = qss.time_series(seven_days_ago, today) 36 | self.assertEqual([t[1] for t in time_series], [0, 1, 2, 3, 4, 5, 6, 7]) 37 | 38 | def test_time_series(self): 39 | _now = compat.now() 40 | today = _remove_time(_now) 41 | self.assertTimeSeriesWorks(today) 42 | 43 | def test_time_series_naive(self): 44 | self.assertTimeSeriesWorks(datetime.date.today()) 45 | 46 | def test_time_series_weeks(self): 47 | day = datetime.date(year=2013, month=4, day=5) 48 | 49 | u = User.objects.create_user('user', 'user@example.com') 50 | u.date_joined = day 51 | u.save() 52 | 53 | qs = User.objects.all() 54 | qss = QuerySetStats(qs, 'date_joined') 55 | qss.time_series(day - datetime.timedelta(days=30), day, interval='weeks') 56 | 57 | def test_until(self): 58 | now = compat.now() 59 | today = _remove_time(now) 60 | yesterday = today - datetime.timedelta(days=1) 61 | 62 | u = User.objects.create_user('u', 'u@example.com') 63 | u.date_joined = today 64 | u.save() 65 | 66 | qs = User.objects.all() 67 | qss = QuerySetStats(qs, 'date_joined') 68 | 69 | self.assertEqual(qss.until(now), 1) 70 | self.assertEqual(qss.until(today), 1) 71 | self.assertEqual(qss.until(yesterday), 0) 72 | self.assertEqual(qss.until_now(), 1) 73 | 74 | def test_after(self): 75 | now = compat.now() 76 | today = _remove_time(now) 77 | tomorrow = today + datetime.timedelta(days=1) 78 | 79 | u = User.objects.create_user('u', 'u@example.com') 80 | u.date_joined = today 81 | u.save() 82 | 83 | qs = User.objects.all() 84 | qss = QuerySetStats(qs, 'date_joined') 85 | 86 | self.assertEqual(qss.after(today), 1) 87 | self.assertEqual(qss.after(now), 0) 88 | u.date_joined=tomorrow 89 | u.save() 90 | self.assertEqual(qss.after(now), 1) 91 | 92 | # MC_TODO: aggregate_field tests 93 | 94 | def test_query_set_missing(self): 95 | qss = QuerySetStats(date_field='foo') 96 | for method in ['this_day', 'this_month', 'this_year']: 97 | self.assertRaises(QuerySetMissing, getattr(qss, method)) 98 | 99 | def test_date_field_missing(self): 100 | qss = QuerySetStats(User.objects.all()) 101 | for method in ['this_day', 'this_month', 'this_year']: 102 | self.assertRaises(DateFieldMissing, getattr(qss, method)) 103 | 104 | def test_invalid_interval(self): 105 | qss = QuerySetStats(User.objects.all(), 'date_joined') 106 | def _invalid(): 107 | qss.time_series(qss.today, qss.today, interval='monkeys') 108 | self.assertRaises(InvalidInterval, _invalid) 109 | -------------------------------------------------------------------------------- /qsstats/__init__.py: -------------------------------------------------------------------------------- 1 | __author__ = 'Matt Croydon, Mikhail Korobov, Pawel Tomasiewicz, Petr Dlouhy' 2 | __version__ = (0, 7, 0) 3 | 4 | import warnings 5 | from functools import partial 6 | from dateutil.relativedelta import relativedelta 7 | from dateutil.parser import parse 8 | 9 | from django.db.models import Count 10 | from django.db import DatabaseError, transaction 11 | from django.db.models.functions import Trunc 12 | from django.conf import settings 13 | 14 | from qsstats.utils import get_bounds, _to_datetime, _parse_interval, _remove_time 15 | from qsstats import compat 16 | from qsstats.exceptions import * 17 | from datetime import date, datetime 18 | from six import string_types 19 | 20 | class QuerySetStats(object): 21 | """ 22 | Generates statistics about a queryset using Django aggregates. QuerySetStats 23 | is able to handle snapshots of data (for example this day, week, month, or 24 | year) or generate time series data suitable for graphing. 25 | """ 26 | def __init__(self, qs=None, date_field=None, aggregate=None, today=None): 27 | self.qs = qs 28 | self.date_field = date_field 29 | self.aggregate = aggregate or Count('id', distinct=True) 30 | self.today = today or self.update_today() 31 | 32 | # Aggregates for a specific period of time 33 | 34 | def for_interval(self, interval, dt, date_field=None, aggregate=None): 35 | start, end = get_bounds(dt, interval) 36 | date_field = date_field or self.date_field 37 | kwargs = {'%s__range' % date_field : (start, end)} 38 | return self._aggregate(date_field, aggregate, kwargs) 39 | 40 | def this_interval(self, interval, date_field=None, aggregate=None): 41 | method = getattr(self, 'for_%s' % interval) 42 | return method(self.today, date_field, aggregate) 43 | 44 | # support for this_* and for_* methods 45 | def __getattr__(self, name): 46 | if name.startswith('for_'): 47 | return partial(self.for_interval, name[4:]) 48 | if name.startswith('this_'): 49 | return partial(self.this_interval, name[5:]) 50 | raise AttributeError 51 | 52 | def time_series(self, start, end=None, interval='days', 53 | date_field=None, aggregate=None, engine=None): 54 | ''' Aggregate over time intervals ''' 55 | 56 | end = end or self.today 57 | args = [start, end, interval, date_field, aggregate] 58 | sid = transaction.savepoint() 59 | try: 60 | return self._fast_time_series(*args) 61 | except (ValueError): 62 | transaction.savepoint_rollback(sid) 63 | warnings.warn("Your database doesn't support timezones. Switching to slower QSStats queries.") 64 | return self._slow_time_series(*args) 65 | 66 | def _slow_time_series(self, start, end, interval='days', 67 | date_field=None, aggregate=None): 68 | ''' Aggregate over time intervals using 1 sql query for one interval ''' 69 | 70 | num, interval = _parse_interval(interval) 71 | 72 | if interval not in ['minutes', 'hours', 73 | 'days', 'weeks', 74 | 'months', 'years'] or num != 1: 75 | raise InvalidInterval('Interval is currently not supported.') 76 | 77 | method = getattr(self, 'for_%s' % interval[:-1]) 78 | 79 | stat_list = [] 80 | dt, end = _to_datetime(start), _to_datetime(end) 81 | while dt <= end: 82 | value = method(dt, date_field, aggregate) 83 | stat_list.append((dt, value,)) 84 | dt = dt + relativedelta(**{interval : 1}) 85 | return stat_list 86 | 87 | def _fast_time_series(self, start, end, interval='days', 88 | date_field=None, aggregate=None): 89 | ''' Aggregate over time intervals using just 1 sql query ''' 90 | 91 | date_field = date_field or self.date_field 92 | aggregate = aggregate or self.aggregate 93 | 94 | num, interval = _parse_interval(interval) 95 | 96 | interval_s = interval.rstrip('s') 97 | start, _ = get_bounds(start, interval_s) 98 | _, end = get_bounds(end, interval_s) 99 | 100 | kwargs = {'%s__range' % date_field : (start, end)} 101 | 102 | # TODO: maybe we could use the tzinfo for the user's location 103 | aggregate_data = self.qs.\ 104 | filter(**kwargs).\ 105 | annotate(d=Trunc(date_field, interval_s, tzinfo=start.tzinfo)).\ 106 | order_by().values('d').\ 107 | annotate(agg=aggregate) 108 | 109 | today = _remove_time(compat.now()) 110 | def to_dt(d): 111 | if isinstance(d, string_types): 112 | return parse(d, yearfirst=True, default=today) 113 | if type(d).__name__ == "date": 114 | d = datetime(year=d.year, month=d.month, day=d.day, tzinfo=start.tzinfo) 115 | return d 116 | return d 117 | 118 | data = dict((to_dt(item['d']), item['agg']) for item in aggregate_data) 119 | 120 | stat_list = [] 121 | dt = start 122 | while dt < end: 123 | idx = 0 124 | value = 0 125 | for i in range(num): 126 | value = value + data.get(dt, 0) 127 | if i == 0: 128 | stat_list.append((dt, value,)) 129 | idx = len(stat_list) - 1 130 | elif i == num - 1: 131 | stat_list[idx] = (dt, value,) 132 | dt = dt + relativedelta(**{interval : 1}) 133 | 134 | return stat_list 135 | 136 | # Aggregate totals using a date or datetime as a pivot 137 | 138 | def until(self, dt, date_field=None, aggregate=None): 139 | return self.pivot(dt, 'lte', date_field, aggregate) 140 | 141 | def until_now(self, date_field=None, aggregate=None): 142 | return self.pivot(compat.now(), 'lte', date_field, aggregate) 143 | 144 | def after(self, dt, date_field=None, aggregate=None): 145 | return self.pivot(dt, 'gte', date_field, aggregate) 146 | 147 | def after_now(self, date_field=None, aggregate=None): 148 | return self.pivot(compat.now(), 'gte', date_field, aggregate) 149 | 150 | def pivot(self, dt, operator=None, date_field=None, aggregate=None): 151 | operator = operator or self.operator 152 | if operator not in ['lt', 'lte', 'gt', 'gte']: 153 | raise InvalidOperator("Please provide a valid operator.") 154 | 155 | kwargs = {'%s__%s' % (date_field or self.date_field, operator) : dt} 156 | return self._aggregate(date_field, aggregate, kwargs) 157 | 158 | # Utility functions 159 | def update_today(self): 160 | _now = compat.now() 161 | self.today = _remove_time(_now) 162 | return self.today 163 | 164 | def _aggregate(self, date_field=None, aggregate=None, filter=None): 165 | date_field = date_field or self.date_field 166 | aggregate = aggregate or self.aggregate 167 | 168 | if not date_field: 169 | raise DateFieldMissing("Please provide a date_field.") 170 | 171 | if self.qs is None: 172 | raise QuerySetMissing("Please provide a queryset.") 173 | 174 | agg = self.qs.filter(**filter).aggregate(agg=aggregate) 175 | return agg['agg'] 176 | -------------------------------------------------------------------------------- /README.rst: -------------------------------------------------------------------------------- 1 | ==================================================== 2 | django-qsstats-magic: QuerySet statistics for Django 3 | ==================================================== 4 | 5 | The goal of django-qsstats is to be a microframework to make 6 | repetitive tasks such as generating aggregate statistics of querysets 7 | over time easier. It's probably overkill for the task at hand, but yay 8 | microframeworks! 9 | 10 | django-qsstats-magic is a refactoring of django-qsstats app with slightly 11 | changed API, simplified internals and faster time_series implementation. 12 | 13 | 14 | Requirements 15 | ============ 16 | 17 | * `python-dateutil `_ > 1.4, < 2.0 18 | * `django `_ 1.8+ 19 | 20 | Database 21 | -------- 22 | 23 | If timezone support is enabled in Django, the database must have also timezone support installed. 24 | For MySQL it might be needed to run: 25 | 26 | :: 27 | - mysql_tzinfo_to_sql /usr/share/zoneinfo | mysql -u root mysql 28 | 29 | 30 | License 31 | ======= 32 | 33 | Liensed under a BSD-style license. 34 | 35 | Examples 36 | ======== 37 | 38 | How many users signed up today? this month? this year? 39 | ------------------------------------------------------ 40 | 41 | :: 42 | 43 | from django.contrib.auth.models import User 44 | import qsstats 45 | 46 | qs = User.objects.all() 47 | qss = qsstats.QuerySetStats(qs, 'date_joined') 48 | 49 | print '%s new accounts today.' % qss.this_day() 50 | print '%s new accounts this week.' % qss.this_week() 51 | print '%s new accounts this month.' % qss.this_month() 52 | print '%s new accounts this year.' % qss.this_year() 53 | print '%s new accounts until now.' % qss.until_now() 54 | 55 | This might print something like:: 56 | 57 | 5 new accounts today. 58 | 11 new accounts this week. 59 | 27 new accounts this month. 60 | 377 new accounts this year. 61 | 409 new accounts until now. 62 | 63 | Aggregating time-series data suitable for graphing 64 | -------------------------------------------------- 65 | 66 | :: 67 | 68 | from django.contrib.auth.models import User 69 | import datetime, qsstats 70 | 71 | qs = User.objects.all() 72 | qss = qsstats.QuerySetStats(qs, 'date_joined') 73 | 74 | today = datetime.date.today() 75 | seven_days_ago = today - datetime.timedelta(days=7) 76 | 77 | time_series = qss.time_series(seven_days_ago, today) 78 | print 'New users in the last 7 days: %s' % [t[1] for t in time_series] 79 | 80 | This might print something like:: 81 | 82 | New users in the last 7 days: [3, 10, 7, 4, 12, 9, 11] 83 | 84 | 85 | Please see qsstats/tests.py for similar usage examples. 86 | 87 | API 88 | === 89 | 90 | The ``QuerySetStats`` object 91 | ---------------------------- 92 | 93 | In order to provide maximum flexibility, the ``QuerySetStats`` object 94 | can be instantiated with as little or as much information as you like. 95 | All keword arguments are optional but ``DateFieldMissing`` and 96 | ``QuerySetMissing`` will be raised if you try to use ``QuerySetStats`` 97 | without providing enough information. 98 | 99 | ``qs`` 100 | The queryset to operate on. 101 | 102 | Default: ``None`` 103 | 104 | ``date_field`` 105 | The date field within the queryset to use. 106 | 107 | Default: ``None`` 108 | 109 | ``aggregate`` 110 | The django aggregation instance. Can be set also set when 111 | instantiating or calling one of the methods. 112 | 113 | Default: ``Count('id')`` 114 | 115 | ``operator`` 116 | The default operator to use for the ``pivot`` function. Can be also set 117 | when calling ``pivot``. 118 | 119 | Default: ``'lte'`` 120 | 121 | ``today`` 122 | The date that will be considered as today date. If ``today`` param is None 123 | QuerySetStats' today will be datetime.date.today(). 124 | 125 | Default: ``None`` 126 | 127 | 128 | All of the documented methods take a standard set of keyword arguments 129 | that override any information already stored within the ``QuerySetStats`` 130 | object. These keyword arguments are ``date_field`` and ``aggregate``. 131 | 132 | Once you have a ``QuerySetStats`` object instantiated, you can receive a 133 | single aggregate result by using the following methods: 134 | 135 | * ``for_minute`` 136 | * ``for_hour`` 137 | * ``for_day`` 138 | * ``for_week`` 139 | * ``for_month`` 140 | * ``for_year`` 141 | 142 | Positional arguments: ``dt``, a ``datetime.datetime`` or ``datetime.date`` 143 | object to filter the queryset to this interval (minute, hour, day, week, 144 | month or year). 145 | 146 | * ``this_minute`` 147 | * ``this_hour`` 148 | * ``this_day`` 149 | * ``this_week`` 150 | * ``this_month`` 151 | * ``this_year`` 152 | 153 | Wrappers around ``for_`` that uses ``dateutil.relativedelta`` to 154 | provide aggregate information for this current interval. 155 | 156 | ``QuerySetStats`` also provides a method for returning aggregated 157 | time-series data which may be extremely using in plotting data: 158 | 159 | ``time_series`` 160 | Positional arguments: ``start`` and ``end``, each a 161 | ``datetime.date`` or ``datetime.datetime`` object used in marking 162 | the start and stop of the time series data. 163 | 164 | Keyword arguments: In addition to the standard ``date_field`` and 165 | ``aggregate`` keyword argument, ``time_series`` takes an optional 166 | ``interval`` keyword argument used to mark which interval to use while 167 | calculating aggregate data between ``start`` and ``end``. This argument 168 | defaults to ``'days'`` and can accept ``'years'``, ``'months'``, 169 | ``'weeks'``, ``'days'``, ``'hours'`` or ``'minutes'``. 170 | It will raise ``InvalidInterval`` otherwise. 171 | 172 | This methods returns a list of tuples. The first item in each 173 | tuple is a ``datetime.datetime`` object for the current inverval. The 174 | second item is the result of the aggregate operation. For 175 | example:: 176 | 177 | [(datetime.datetime(2010, 3, 28, 0, 0), 12), (datetime.datetime(2010, 3, 29, 0, 0), 0), ...] 178 | 179 | Formatting of date information is left as an exercise to the user and may 180 | vary depending on interval used. 181 | 182 | ``until`` 183 | Provide aggregate information until a given date or time, filtering the 184 | queryset using ``lte``. 185 | 186 | Positional arguments: ``dt`` a ``datetime.date`` or ``datetime.datetime`` 187 | object to be used for filtering the queryset since. 188 | 189 | Keyword arguments: ``date_field``, ``aggregate``. 190 | 191 | ``until_now`` 192 | Aggregate information until now. 193 | 194 | Positional arguments: ``dt`` a ``datetime.date`` or ``datetime.datetime`` 195 | object to be used for filtering the queryset since (using ``lte``). 196 | 197 | Keyword arguments: ``date_field``, ``aggregate``. 198 | 199 | ``after`` 200 | Aggregate information after a given date or time, filtering the queryset 201 | using ``gte``. 202 | 203 | Positional arguments: ``dt`` a ``datetime.date`` or ``datetime.datetime`` 204 | object to be used for filtering the queryset since. 205 | 206 | Keyword arguments: ``date_field``, ``aggregate``. 207 | 208 | ``after_now`` 209 | Aggregate information after now. 210 | 211 | Positional arguments: ``dt`` a ``datetime.date`` or ``datetime.datetime`` 212 | object to be used for filtering the queryset since (using ``gte``). 213 | 214 | Keyword arguments: ``date_field``, ``aggregate``. 215 | 216 | ``pivot`` 217 | Used by ``since``, ``after``, and ``until_now`` but potentially useful if 218 | you would like to specify your own operator instead of the defaults. 219 | 220 | Positional arguments: ``dt`` a ``datetime.date`` or ``datetime.datetime`` 221 | object to be used for filtering the queryset since (using ``lte``). 222 | 223 | Keyword arguments: ``operator``, ``date_field``, ``aggregate``. 224 | 225 | Raises ``InvalidOperator`` if the operator provided is not one of ``'lt'``, 226 | ``'lte'``, ``gt`` or ``gte``. 227 | 228 | Testing 229 | ======= 230 | 231 | If you'd like to test ``django-qsstats-magic`` against your local configuration, add 232 | ``qsstats`` to your ``INSTALLED_APPS`` and run ``./manage.py test qsstats``. 233 | The test suite assumes that ``django.contrib.auth`` is installed. 234 | 235 | For testing against different python, DB and django versions install tox 236 | (pip install tox) and run 'tox' from the source checkout:: 237 | 238 | $ tox 239 | 240 | Db user 'qsstats_test' with password 'qsstats_test' and a DB 'qsstats_test' 241 | should exist. 242 | 243 | Difference from django-qsstats 244 | ============================== 245 | 246 | 1. Faster time_series method using 1 sql query (currently works for MySQL and 247 | PostgreSQL, with a fallback to the old method for other DB backends). 248 | 2. Single ``aggregate`` parameter instead of ``aggregate_field`` and 249 | ``aggregate_class``. Default value is always ``Count('id')`` and can't be 250 | specified in settings.py. ``QUERYSETSTATS_DEFAULT_OPERATOR`` option is also 251 | unsupported now. 252 | 3. Support for minute and hour aggregates. 253 | 4. ``start_date`` and ``end_date`` arguments are renamed to ``start`` and 254 | ``end`` because of 3. 255 | 5. Internals are changed. 256 | 257 | I don't know if original author (Matt Croydon) would like my changes so 258 | I renamed a project for now. If the changes will be merged then 259 | django-qsstats-magic will become obsolete. 260 | --------------------------------------------------------------------------------