├── .gitignore ├── LICENSE ├── MANIFEST.in ├── README.txt ├── demo ├── __init__.py ├── demo_models │ ├── __init__.py │ ├── admin.py │ ├── models.py │ ├── tests.py │ └── views.py ├── manage.py ├── settings.py └── urls.py ├── queryset_transform └── __init__.py └── setup.py /.gitignore: -------------------------------------------------------------------------------- 1 | ve 2 | demo/data.db 3 | 4 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2010, Simon Willison. 2 | All rights reserved. 3 | 4 | Redistribution and use in source and binary forms, with or without modification, 5 | are permitted provided that the following conditions are met: 6 | 7 | 1. Redistributions of source code must retain the above copyright notice, 8 | this list of conditions and the following disclaimer. 9 | 10 | 2. Redistributions in binary form must reproduce the above copyright 11 | notice, this list of conditions and the following disclaimer in the 12 | documentation and/or other materials provided with the distribution. 13 | 14 | 3. Neither the name of Django nor the names of its contributors may be used 15 | to endorse or promote products derived from this software without 16 | specific prior written permission. 17 | 18 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND 19 | ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 20 | WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 21 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR 22 | ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 23 | (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 24 | LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON 25 | ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 26 | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 27 | SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include LICENSE 2 | -------------------------------------------------------------------------------- /README.txt: -------------------------------------------------------------------------------- 1 | django_queryset_transform 2 | ========================= 3 | 4 | Allows you to register a transforming map function with a Django QuerySet 5 | that will be executed only when the QuerySet itself has been evaluated. 6 | 7 | This allows you to build optimisations like "fetch all tags for these 10 rows" 8 | while still benefiting from Django's lazy QuerySet evaluation. 9 | 10 | For example: 11 | 12 | def lookup_tags(item_qs): 13 | item_pks = [item.pk for item in item_qs] 14 | m2mfield = Item._meta.get_field_by_name('tags')[0] 15 | tags_for_item = Tag.objects.filter( 16 | item__in = item_pks 17 | ).extra(select = { 18 | 'item_id': '%s.%s' % ( 19 | m2mfield.m2m_db_table(), m2mfield.m2m_column_name() 20 | ) 21 | }) 22 | tag_dict = {} 23 | for tag in tags_for_item: 24 | tag_dict.setdefault(tag.item_id, []).append(tag) 25 | for item in item_qs: 26 | item.fetched_tags = tag_dict.get(item.pk, []) 27 | 28 | qs = Item.objects.filter(name__contains = 'e').transform(lookup_tags) 29 | 30 | for item in qs: 31 | print item, item.fetched_tags 32 | 33 | Prints: 34 | 35 | Winter comes to Ogglesbrook [, , , ] 36 | Summer now [, ] 37 | 38 | But only executes two SQL queries - one to fetch the items, and one to fetch ALL of the tags for those items. 39 | 40 | Since the transformer function can transform an evaluated QuerySet, it 41 | doesn't need to make extra database calls at all - it should work for things 42 | like looking up additional data from a cache.multi_get() as well. 43 | 44 | Originally inspired by http://github.com/lilspikey/django-batch-select/ 45 | -------------------------------------------------------------------------------- /demo/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/simonw/django-queryset-transform/deee2c605ae589132ee1657aea7c1a33c9b7e325/demo/__init__.py -------------------------------------------------------------------------------- /demo/demo_models/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/simonw/django-queryset-transform/deee2c605ae589132ee1657aea7c1a33c9b7e325/demo/demo_models/__init__.py -------------------------------------------------------------------------------- /demo/demo_models/admin.py: -------------------------------------------------------------------------------- 1 | from models import Tag, Item 2 | from django.contrib import admin 3 | 4 | admin.site.register(Tag) 5 | admin.site.register(Item) 6 | -------------------------------------------------------------------------------- /demo/demo_models/models.py: -------------------------------------------------------------------------------- 1 | from django.db import models 2 | from queryset_transform import TransformManager 3 | 4 | class Tag(models.Model): 5 | name = models.CharField(max_length = 255) 6 | 7 | def __unicode__(self): 8 | return self.name 9 | 10 | class Item(models.Model): 11 | name = models.CharField(max_length = 255) 12 | tags = models.ManyToManyField(Tag) 13 | 14 | objects = TransformManager() 15 | 16 | def __unicode__(self): 17 | return self.name 18 | -------------------------------------------------------------------------------- /demo/demo_models/tests.py: -------------------------------------------------------------------------------- 1 | """ 2 | This file demonstrates two different styles of tests (one doctest and one 3 | unittest). These will both pass when you run "manage.py test". 4 | 5 | Replace these with more appropriate tests for your application. 6 | """ 7 | 8 | from django.test import TestCase 9 | 10 | class SimpleTest(TestCase): 11 | def test_basic_addition(self): 12 | """ 13 | Tests that 1 + 1 always equals 2. 14 | """ 15 | self.failUnlessEqual(1 + 1, 2) 16 | 17 | __test__ = {"doctest": """ 18 | Another way to test that 1 + 1 is equal to 2. 19 | 20 | >>> 1 + 1 == 2 21 | True 22 | """} 23 | 24 | -------------------------------------------------------------------------------- /demo/demo_models/views.py: -------------------------------------------------------------------------------- 1 | # Create your views here. 2 | -------------------------------------------------------------------------------- /demo/manage.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | import sys 3 | sys.path.append('../') 4 | 5 | from django.core.management import execute_manager 6 | try: 7 | import settings # Assumed to be in the same directory. 8 | except ImportError: 9 | import sys 10 | sys.stderr.write("Error: Can't find the file 'settings.py' in the directory containing %r. It appears you've customized things.\nYou'll have to run django-admin.py, passing it your settings module.\n(If the file settings.py does indeed exist, it's causing an ImportError somehow.)\n" % __file__) 11 | sys.exit(1) 12 | 13 | if __name__ == "__main__": 14 | execute_manager(settings) 15 | -------------------------------------------------------------------------------- /demo/settings.py: -------------------------------------------------------------------------------- 1 | # Django settings for demo project. 2 | 3 | DEBUG = True 4 | TEMPLATE_DEBUG = DEBUG 5 | 6 | ADMINS = ( 7 | # ('Your Name', 'your_email@domain.com'), 8 | ) 9 | 10 | MANAGERS = ADMINS 11 | 12 | DATABASE_ENGINE = 'sqlite3' # 'postgresql_psycopg2', 'postgresql', 'mysql', 'sqlite3' or 'oracle'. 13 | DATABASE_NAME = 'data.db' # Or path to database file if using sqlite3. 14 | DATABASE_USER = '' # Not used with sqlite3. 15 | DATABASE_PASSWORD = '' # Not used with sqlite3. 16 | DATABASE_HOST = '' # Set to empty string for localhost. Not used with sqlite3. 17 | DATABASE_PORT = '' # Set to empty string for default. Not used with sqlite3. 18 | 19 | # Local time zone for this installation. Choices can be found here: 20 | # http://en.wikipedia.org/wiki/List_of_tz_zones_by_name 21 | # although not all choices may be available on all operating systems. 22 | # If running in a Windows environment this must be set to the same as your 23 | # system time zone. 24 | TIME_ZONE = 'America/Chicago' 25 | 26 | # Language code for this installation. All choices can be found here: 27 | # http://www.i18nguy.com/unicode/language-identifiers.html 28 | LANGUAGE_CODE = 'en-us' 29 | 30 | SITE_ID = 1 31 | 32 | # If you set this to False, Django will make some optimizations so as not 33 | # to load the internationalization machinery. 34 | USE_I18N = True 35 | 36 | # Absolute path to the directory that holds media. 37 | # Example: "/home/media/media.lawrence.com/" 38 | MEDIA_ROOT = '' 39 | 40 | # URL that handles the media served from MEDIA_ROOT. Make sure to use a 41 | # trailing slash if there is a path component (optional in other cases). 42 | # Examples: "http://media.lawrence.com", "http://example.com/media/" 43 | MEDIA_URL = '' 44 | 45 | # URL prefix for admin media -- CSS, JavaScript and images. Make sure to use a 46 | # trailing slash. 47 | # Examples: "http://foo.com/media/", "/media/". 48 | ADMIN_MEDIA_PREFIX = '/media/' 49 | 50 | # Make this unique, and don't share it with anybody. 51 | SECRET_KEY = 'hea-c%x=u^&2bypgp81_+tfmxkt3l-ni-3$(yml%d=!@9&+u2x' 52 | 53 | # List of callables that know how to import templates from various sources. 54 | TEMPLATE_LOADERS = ( 55 | 'django.template.loaders.filesystem.load_template_source', 56 | 'django.template.loaders.app_directories.load_template_source', 57 | # 'django.template.loaders.eggs.load_template_source', 58 | ) 59 | 60 | MIDDLEWARE_CLASSES = ( 61 | 'django.middleware.common.CommonMiddleware', 62 | 'django.contrib.sessions.middleware.SessionMiddleware', 63 | 'django.contrib.auth.middleware.AuthenticationMiddleware', 64 | ) 65 | 66 | ROOT_URLCONF = 'demo.urls' 67 | 68 | TEMPLATE_DIRS = ( 69 | # Put strings here, like "/home/html/django_templates" or "C:/www/django/templates". 70 | # Always use forward slashes, even on Windows. 71 | # Don't forget to use absolute paths, not relative paths. 72 | ) 73 | 74 | INSTALLED_APPS = ( 75 | 'django.contrib.auth', 76 | 'django.contrib.contenttypes', 77 | 'django.contrib.sessions', 78 | 'django.contrib.sites', 79 | 'django.contrib.admin', 80 | 'demo_models', 81 | ) 82 | -------------------------------------------------------------------------------- /demo/urls.py: -------------------------------------------------------------------------------- 1 | from django.conf.urls.defaults import * 2 | from django.contrib import admin 3 | from django.http import HttpResponse 4 | from django.db import connection 5 | 6 | from demo_models.models import Item, Tag 7 | 8 | from pprint import pformat 9 | 10 | admin.autodiscover() 11 | 12 | def example(request): 13 | def lookup_tags(item_qs): 14 | item_pks = [item.pk for item in item_qs] 15 | m2mfield = Item._meta.get_field_by_name('tags')[0] 16 | tags_for_item = Tag.objects.filter( 17 | item__in = item_pks 18 | ).extra(select = { 19 | 'item_id': '%s.%s' % ( 20 | m2mfield.m2m_db_table(), m2mfield.m2m_column_name() 21 | ) 22 | }) 23 | tag_dict = {} 24 | for tag in tags_for_item: 25 | tag_dict.setdefault(tag.item_id, []).append(tag) 26 | for item in item_qs: 27 | item.fetched_tags = tag_dict.get(item.pk, []) 28 | 29 | qs = Item.objects.all().transform(lookup_tags) 30 | 31 | s = [] 32 | 33 | for item in qs: 34 | s.append('%s: %s' % (item, [t.name for t in item.fetched_tags])) 35 | 36 | return HttpResponse( 37 | '
'.join(s) + '
%s' % pformat(connection.queries)
38 |     )
39 | 
40 | urlpatterns = patterns('',
41 |     (r'^$', example),
42 |     (r'^admin/', include(admin.site.urls)),
43 | )
44 | 


--------------------------------------------------------------------------------
/queryset_transform/__init__.py:
--------------------------------------------------------------------------------
 1 | from django.db import models
 2 | 
 3 | class TransformQuerySet(models.query.QuerySet):
 4 |     def __init__(self, *args, **kwargs):
 5 |         super(TransformQuerySet, self).__init__(*args, **kwargs)
 6 |         self._transform_fns = []
 7 | 
 8 |     def _clone(self, klass=None, setup=False, **kw):
 9 |         c = super(TransformQuerySet, self)._clone(klass, setup, **kw)
10 |         c._transform_fns = self._transform_fns[:]
11 |         return c
12 | 
13 |     def transform(self, fn):
14 |         c = self._clone()
15 |         c._transform_fns.append(fn)
16 |         return c
17 | 
18 |     def iterator(self):
19 |         result_iter = super(TransformQuerySet, self).iterator()
20 |         if self._transform_fns:
21 |             results = list(result_iter)
22 |             for fn in self._transform_fns:
23 |                 fn(results)
24 |             return iter(results)
25 |         return result_iter
26 | 
27 | class TransformManager(models.Manager):
28 | 
29 |     def get_query_set(self):
30 |         return TransformQuerySet(self.model)
31 | 


--------------------------------------------------------------------------------
/setup.py:
--------------------------------------------------------------------------------
 1 | from distutils.core import setup
 2 | import os
 3 | 
 4 | setup(
 5 |     name = 'django-queryset-transform',
 6 |     packages = ['queryset_transform'],
 7 |     version='0.0.2',
 8 |     description='Experimental .transform(fn) method for Django QuerySets, for '
 9 |                 'clever lazily evaluated optimisations.',
10 |     long_description=open('README.txt').read(),
11 |     author='Simon Willison',
12 |     author_email='simon@simonwillison.net',
13 |     url='http://github.com/simonw/django-queryset-transform',
14 |     license='BSD',
15 |     classifiers=[
16 |         'Development Status :: 4 - Beta',
17 |         'Environment :: Web Environment',
18 |         'Framework :: Django',
19 |         'Intended Audience :: Developers',
20 |         'License :: OSI Approved :: BSD License',
21 |         'Operating System :: OS Independent',
22 |         'Programming Language :: Python',
23 |         'Topic :: Software Development :: Libraries :: Python Modules',
24 |     ],
25 | )
26 | 


--------------------------------------------------------------------------------