├── demo ├── __init__.py ├── demo_models │ ├── __init__.py │ ├── views.py │ ├── admin.py │ ├── models.py │ └── tests.py ├── manage.py ├── urls.py └── settings.py ├── MANIFEST.in ├── .gitignore ├── setup.py ├── queryset_transform └── __init__.py ├── LICENSE └── README.txt /demo/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /demo/demo_models/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include LICENSE 2 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | ve 2 | demo/data.db 3 | 4 | -------------------------------------------------------------------------------- /demo/demo_models/views.py: -------------------------------------------------------------------------------- 1 | # Create your views here. 2 | -------------------------------------------------------------------------------- /demo/demo_models/admin.py: -------------------------------------------------------------------------------- 1 | from models import Tag, Item 2 | from django.contrib import admin 3 | 4 | admin.site.register(Tag) 5 | admin.site.register(Item) 6 | -------------------------------------------------------------------------------- /demo/demo_models/models.py: -------------------------------------------------------------------------------- 1 | from django.db import models 2 | from queryset_transform import TransformManager 3 | 4 | class Tag(models.Model): 5 | name = models.CharField(max_length = 255) 6 | 7 | def __unicode__(self): 8 | return self.name 9 | 10 | class Item(models.Model): 11 | name = models.CharField(max_length = 255) 12 | tags = models.ManyToManyField(Tag) 13 | 14 | objects = TransformManager() 15 | 16 | def __unicode__(self): 17 | return self.name 18 | -------------------------------------------------------------------------------- /demo/demo_models/tests.py: -------------------------------------------------------------------------------- 1 | """ 2 | This file demonstrates two different styles of tests (one doctest and one 3 | unittest). These will both pass when you run "manage.py test". 4 | 5 | Replace these with more appropriate tests for your application. 6 | """ 7 | 8 | from django.test import TestCase 9 | 10 | class SimpleTest(TestCase): 11 | def test_basic_addition(self): 12 | """ 13 | Tests that 1 + 1 always equals 2. 14 | """ 15 | self.failUnlessEqual(1 + 1, 2) 16 | 17 | __test__ = {"doctest": """ 18 | Another way to test that 1 + 1 is equal to 2. 19 | 20 | >>> 1 + 1 == 2 21 | True 22 | """} 23 | 24 | -------------------------------------------------------------------------------- /demo/manage.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | import sys 3 | sys.path.append('../') 4 | 5 | from django.core.management import execute_manager 6 | try: 7 | import settings # Assumed to be in the same directory. 8 | except ImportError: 9 | import sys 10 | sys.stderr.write("Error: Can't find the file 'settings.py' in the directory containing %r. It appears you've customized things.\nYou'll have to run django-admin.py, passing it your settings module.\n(If the file settings.py does indeed exist, it's causing an ImportError somehow.)\n" % __file__) 11 | sys.exit(1) 12 | 13 | if __name__ == "__main__": 14 | execute_manager(settings) 15 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | from distutils.core import setup 2 | import os 3 | 4 | setup( 5 | name = 'django-queryset-transform', 6 | packages = ['queryset_transform'], 7 | version='0.0.2', 8 | description='Experimental .transform(fn) method for Django QuerySets, for ' 9 | 'clever lazily evaluated optimisations.', 10 | long_description=open('README.txt').read(), 11 | author='Simon Willison', 12 | author_email='simon@simonwillison.net', 13 | url='http://github.com/simonw/django-queryset-transform', 14 | license='BSD', 15 | classifiers=[ 16 | 'Development Status :: 4 - Beta', 17 | 'Environment :: Web Environment', 18 | 'Framework :: Django', 19 | 'Intended Audience :: Developers', 20 | 'License :: OSI Approved :: BSD License', 21 | 'Operating System :: OS Independent', 22 | 'Programming Language :: Python', 23 | 'Topic :: Software Development :: Libraries :: Python Modules', 24 | ], 25 | ) 26 | -------------------------------------------------------------------------------- /queryset_transform/__init__.py: -------------------------------------------------------------------------------- 1 | from django.db import models 2 | 3 | class TransformQuerySet(models.query.QuerySet): 4 | def __init__(self, *args, **kwargs): 5 | super(TransformQuerySet, self).__init__(*args, **kwargs) 6 | self._transform_fns = [] 7 | 8 | def _clone(self, klass=None, setup=False, **kw): 9 | c = super(TransformQuerySet, self)._clone(klass, setup, **kw) 10 | c._transform_fns = self._transform_fns[:] 11 | return c 12 | 13 | def transform(self, fn): 14 | c = self._clone() 15 | c._transform_fns.append(fn) 16 | return c 17 | 18 | def iterator(self): 19 | result_iter = super(TransformQuerySet, self).iterator() 20 | if self._transform_fns: 21 | results = list(result_iter) 22 | for fn in self._transform_fns: 23 | fn(results) 24 | return iter(results) 25 | return result_iter 26 | 27 | class TransformManager(models.Manager): 28 | 29 | def get_query_set(self): 30 | return TransformQuerySet(self.model) 31 | -------------------------------------------------------------------------------- /demo/urls.py: -------------------------------------------------------------------------------- 1 | from django.conf.urls.defaults import * 2 | from django.contrib import admin 3 | from django.http import HttpResponse 4 | from django.db import connection 5 | 6 | from demo_models.models import Item, Tag 7 | 8 | from pprint import pformat 9 | 10 | admin.autodiscover() 11 | 12 | def example(request): 13 | def lookup_tags(item_qs): 14 | item_pks = [item.pk for item in item_qs] 15 | m2mfield = Item._meta.get_field_by_name('tags')[0] 16 | tags_for_item = Tag.objects.filter( 17 | item__in = item_pks 18 | ).extra(select = { 19 | 'item_id': '%s.%s' % ( 20 | m2mfield.m2m_db_table(), m2mfield.m2m_column_name() 21 | ) 22 | }) 23 | tag_dict = {} 24 | for tag in tags_for_item: 25 | tag_dict.setdefault(tag.item_id, []).append(tag) 26 | for item in item_qs: 27 | item.fetched_tags = tag_dict.get(item.pk, []) 28 | 29 | qs = Item.objects.all().transform(lookup_tags) 30 | 31 | s = [] 32 | 33 | for item in qs: 34 | s.append('%s: %s' % (item, [t.name for t in item.fetched_tags])) 35 | 36 | return HttpResponse( 37 | '
'.join(s) + '
%s' % pformat(connection.queries)
38 |     )
39 | 
40 | urlpatterns = patterns('',
41 |     (r'^$', example),
42 |     (r'^admin/', include(admin.site.urls)),
43 | )
44 | 


--------------------------------------------------------------------------------
/LICENSE:
--------------------------------------------------------------------------------
 1 | Copyright (c) 2010, Simon Willison.
 2 | All rights reserved.
 3 | 
 4 | Redistribution and use in source and binary forms, with or without modification,
 5 | are permitted provided that the following conditions are met:
 6 | 
 7 |     1. Redistributions of source code must retain the above copyright notice,
 8 |        this list of conditions and the following disclaimer.
 9 | 
10 |     2. Redistributions in binary form must reproduce the above copyright
11 |        notice, this list of conditions and the following disclaimer in the
12 |        documentation and/or other materials provided with the distribution.
13 | 
14 |     3. Neither the name of Django nor the names of its contributors may be used
15 |        to endorse or promote products derived from this software without
16 |        specific prior written permission.
17 | 
18 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
19 | ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
20 | WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
21 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
22 | ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
23 | (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
24 | LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
25 | ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
26 | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
27 | SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
28 | 


--------------------------------------------------------------------------------
/README.txt:
--------------------------------------------------------------------------------
 1 | django_queryset_transform
 2 | =========================
 3 | 
 4 | Allows you to register a transforming map function with a Django QuerySet 
 5 | that will be executed only when the QuerySet itself has been evaluated.
 6 | 
 7 | This allows you to build optimisations like "fetch all tags for these 10 rows"
 8 | while still benefiting from Django's lazy QuerySet evaluation.
 9 | 
10 | For example:
11 |     
12 |     def lookup_tags(item_qs):
13 |         item_pks = [item.pk for item in item_qs]
14 |         m2mfield = Item._meta.get_field_by_name('tags')[0]
15 |         tags_for_item = Tag.objects.filter(
16 |             item__in = item_pks
17 |         ).extra(select = {
18 |             'item_id': '%s.%s' % (
19 |                 m2mfield.m2m_db_table(), m2mfield.m2m_column_name()
20 |             )
21 |         })
22 |         tag_dict = {}
23 |         for tag in tags_for_item:
24 |             tag_dict.setdefault(tag.item_id, []).append(tag)
25 |         for item in item_qs:
26 |             item.fetched_tags = tag_dict.get(item.pk, [])
27 |     
28 |     qs = Item.objects.filter(name__contains = 'e').transform(lookup_tags)
29 |     
30 |     for item in qs:
31 |         print item, item.fetched_tags
32 | 
33 | Prints:
34 | 
35 |     Winter comes to Ogglesbrook [, , , ]
36 |     Summer now [, ]
37 | 
38 | But only executes two SQL queries - one to fetch the items, and one to fetch ALL of the tags for those items.
39 | 
40 | Since the transformer function can transform an evaluated QuerySet, it 
41 | doesn't need to make extra database calls at all - it should work for things 
42 | like looking up additional data from a cache.multi_get() as well.
43 | 
44 | Originally inspired by http://github.com/lilspikey/django-batch-select/
45 | 


--------------------------------------------------------------------------------
/demo/settings.py:
--------------------------------------------------------------------------------
 1 | # Django settings for demo project.
 2 | 
 3 | DEBUG = True
 4 | TEMPLATE_DEBUG = DEBUG
 5 | 
 6 | ADMINS = (
 7 |     # ('Your Name', 'your_email@domain.com'),
 8 | )
 9 | 
10 | MANAGERS = ADMINS
11 | 
12 | DATABASE_ENGINE = 'sqlite3'           # 'postgresql_psycopg2', 'postgresql', 'mysql', 'sqlite3' or 'oracle'.
13 | DATABASE_NAME = 'data.db'             # Or path to database file if using sqlite3.
14 | DATABASE_USER = ''             # Not used with sqlite3.
15 | DATABASE_PASSWORD = ''         # Not used with sqlite3.
16 | DATABASE_HOST = ''             # Set to empty string for localhost. Not used with sqlite3.
17 | DATABASE_PORT = ''             # Set to empty string for default. Not used with sqlite3.
18 | 
19 | # Local time zone for this installation. Choices can be found here:
20 | # http://en.wikipedia.org/wiki/List_of_tz_zones_by_name
21 | # although not all choices may be available on all operating systems.
22 | # If running in a Windows environment this must be set to the same as your
23 | # system time zone.
24 | TIME_ZONE = 'America/Chicago'
25 | 
26 | # Language code for this installation. All choices can be found here:
27 | # http://www.i18nguy.com/unicode/language-identifiers.html
28 | LANGUAGE_CODE = 'en-us'
29 | 
30 | SITE_ID = 1
31 | 
32 | # If you set this to False, Django will make some optimizations so as not
33 | # to load the internationalization machinery.
34 | USE_I18N = True
35 | 
36 | # Absolute path to the directory that holds media.
37 | # Example: "/home/media/media.lawrence.com/"
38 | MEDIA_ROOT = ''
39 | 
40 | # URL that handles the media served from MEDIA_ROOT. Make sure to use a
41 | # trailing slash if there is a path component (optional in other cases).
42 | # Examples: "http://media.lawrence.com", "http://example.com/media/"
43 | MEDIA_URL = ''
44 | 
45 | # URL prefix for admin media -- CSS, JavaScript and images. Make sure to use a
46 | # trailing slash.
47 | # Examples: "http://foo.com/media/", "/media/".
48 | ADMIN_MEDIA_PREFIX = '/media/'
49 | 
50 | # Make this unique, and don't share it with anybody.
51 | SECRET_KEY = 'hea-c%x=u^&2bypgp81_+tfmxkt3l-ni-3$(yml%d=!@9&+u2x'
52 | 
53 | # List of callables that know how to import templates from various sources.
54 | TEMPLATE_LOADERS = (
55 |     'django.template.loaders.filesystem.load_template_source',
56 |     'django.template.loaders.app_directories.load_template_source',
57 | #     'django.template.loaders.eggs.load_template_source',
58 | )
59 | 
60 | MIDDLEWARE_CLASSES = (
61 |     'django.middleware.common.CommonMiddleware',
62 |     'django.contrib.sessions.middleware.SessionMiddleware',
63 |     'django.contrib.auth.middleware.AuthenticationMiddleware',
64 | )
65 | 
66 | ROOT_URLCONF = 'demo.urls'
67 | 
68 | TEMPLATE_DIRS = (
69 |     # Put strings here, like "/home/html/django_templates" or "C:/www/django/templates".
70 |     # Always use forward slashes, even on Windows.
71 |     # Don't forget to use absolute paths, not relative paths.
72 | )
73 | 
74 | INSTALLED_APPS = (
75 |     'django.contrib.auth',
76 |     'django.contrib.contenttypes',
77 |     'django.contrib.sessions',
78 |     'django.contrib.sites',
79 |     'django.contrib.admin',
80 |     'demo_models',
81 | )
82 | 


--------------------------------------------------------------------------------