├── djangotoolbox ├── models.py ├── __init__.py ├── db │ ├── __init__.py │ ├── utils.py │ ├── creation.py │ ├── basecompiler.py │ └── base.py ├── sites │ ├── __init__.py │ └── dynamicsite.py ├── widgets.py ├── errorviews.py ├── admin.py ├── http.py ├── middleware.py ├── utils.py ├── test.py ├── fields.py └── tests.py ├── MANIFEST.in ├── .gitignore ├── README.rst ├── setup.py ├── LICENSE └── CHANGELOG.rst /djangotoolbox/models.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /djangotoolbox/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /djangotoolbox/db/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /djangotoolbox/sites/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include LICENSE 2 | include README.rst 3 | include CHANGELOG.rst 4 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | build 2 | dist 3 | *.egg-info 4 | .project 5 | .pydevproject 6 | .settings 7 | *~ 8 | *.orig 9 | *.pyc 10 | *.pyo 11 | *.swp 12 | *.tmp 13 | desktop.ini 14 | nbproject 15 | .DS_Store 16 | -------------------------------------------------------------------------------- /djangotoolbox/widgets.py: -------------------------------------------------------------------------------- 1 | from django.forms import widgets 2 | from django.template.defaultfilters import filesizeformat 3 | from django.utils.safestring import mark_safe 4 | 5 | 6 | class BlobWidget(widgets.FileInput): 7 | 8 | def render(self, name, value, attrs=None): 9 | try: 10 | blob_size = len(value) 11 | except: 12 | blob_size = 0 13 | 14 | blob_size = filesizeformat(blob_size) 15 | original = super(BlobWidget, self).render(name, value, attrs=None) 16 | return mark_safe("%s

Current size: %s

" % (original, blob_size)) 17 | -------------------------------------------------------------------------------- /djangotoolbox/errorviews.py: -------------------------------------------------------------------------------- 1 | from django import http 2 | from django.template import RequestContext, loader 3 | 4 | 5 | def server_error(request, template_name='500.html'): 6 | """ 7 | 500 error handler. 8 | 9 | Templates: `500.html` 10 | Context: 11 | request_path 12 | The path of the requested URL (e.g., '/app/pages/bad_page/') 13 | """ 14 | 15 | # You need to create a 500.html template. 16 | t = loader.get_template(template_name) 17 | 18 | return http.HttpResponseServerError( 19 | t.render(RequestContext(request, {'request_path': request.path}))) 20 | -------------------------------------------------------------------------------- /djangotoolbox/admin.py: -------------------------------------------------------------------------------- 1 | from django import forms 2 | from django.contrib import admin 3 | from django.contrib.admin.sites import NotRegistered 4 | from django.contrib.auth.admin import UserAdmin 5 | from django.contrib.auth.models import User, Group 6 | 7 | 8 | class UserForm(forms.ModelForm): 9 | 10 | class Meta: 11 | model = User 12 | fields = ('username', 'email', 'first_name', 'last_name', 'is_active', 13 | 'is_staff', 'is_superuser') 14 | 15 | 16 | class CustomUserAdmin(UserAdmin): 17 | fieldsets = None 18 | form = UserForm 19 | search_fields = ('=username',) 20 | list_filter = ('is_staff', 'is_superuser', 'is_active') 21 | 22 | try: 23 | admin.site.unregister(User) 24 | admin.site.register(User, CustomUserAdmin) 25 | except NotRegistered: 26 | pass 27 | 28 | try: 29 | admin.site.unregister(Group) 30 | except NotRegistered: 31 | pass 32 | -------------------------------------------------------------------------------- /djangotoolbox/http.py: -------------------------------------------------------------------------------- 1 | from django.conf import settings 2 | from django.core.serializers.json import DjangoJSONEncoder 3 | from django.http import HttpResponse 4 | from django.utils import simplejson 5 | from django.utils.encoding import force_unicode 6 | from django.utils.functional import Promise 7 | 8 | 9 | class LazyEncoder(DjangoJSONEncoder): 10 | 11 | def default(self, obj): 12 | if isinstance(obj, Promise): 13 | return force_unicode(obj) 14 | return super(LazyEncoder, self).default(obj) 15 | 16 | 17 | class JSONResponse(HttpResponse): 18 | 19 | def __init__(self, pyobj, **kwargs): 20 | super(JSONResponse, self).__init__( 21 | simplejson.dumps(pyobj, cls=LazyEncoder), 22 | content_type='application/json; charset=%s' % 23 | settings.DEFAULT_CHARSET, 24 | **kwargs) 25 | 26 | 27 | class TextResponse(HttpResponse): 28 | 29 | def __init__(self, string='', **kwargs): 30 | super(TextResponse, self).__init__( 31 | string, 32 | content_type='text/plain; charset=%s' % settings.DEFAULT_CHARSET, 33 | **kwargs) 34 | -------------------------------------------------------------------------------- /README.rst: -------------------------------------------------------------------------------- 1 | Djangotoolbox provides a common API for running Django on 2 | non-relational/NoSQL databases (currently via Django-nonrel_). 3 | 4 | In ``djangotoolbox.db`` you can find base classes for writing 5 | non-relational DB backends. Read 6 | `Writing a non-relational Django backend`_ 7 | for more information. 8 | 9 | In ``djangotoolbox.fields`` you can find several common field 10 | types for non-relational DB backends (``ListField``, ``SetField``, 11 | ``DictField``, ``RawField``, ``BlobField``). 12 | 13 | The ``djangotoolbox.admin`` module provides admin overrides for 14 | making ``django.contrib.auth`` work correctly in the admin UI. 15 | Simply add ``'djangotoolbox'`` to ``INSTALLED_APPS`` **after** 16 | ``django.contrib.admin``. This will disable features that 17 | require JOINs. If you still need permission handling you should 18 | use the `nonrel permission backend`_. 19 | 20 | .. _Django-nonrel: http://django-nonrel.org/ 21 | .. _Writing a non-relational Django backend: http://www.allbuttonspressed.com/blog/django/2010/04/Writing-a-non-relational-Django-backend 22 | .. _nonrel permission backend: https://github.com/django-nonrel/django-permission-backend-nonrel 23 | -------------------------------------------------------------------------------- /djangotoolbox/db/utils.py: -------------------------------------------------------------------------------- 1 | from django.db.backends.util import format_number 2 | 3 | 4 | def decimal_to_string(value, max_digits=16, decimal_places=0): 5 | """ 6 | Converts decimal to a unicode string for storage / lookup by nonrel 7 | databases that don't support decimals natively. 8 | 9 | This is an extension to `django.db.backends.util.format_number` 10 | that preserves order -- if one decimal is less than another, their 11 | string representations should compare the same (as strings). 12 | 13 | TODO: Can't this be done using string.format()? 14 | Not in Python 2.5, str.format is backported to 2.6 only. 15 | """ 16 | 17 | # Handle sign separately. 18 | if value.is_signed(): 19 | sign = u'-' 20 | value = abs(value) 21 | else: 22 | sign = u'' 23 | 24 | # Let Django quantize and cast to a string. 25 | value = format_number(value, max_digits, decimal_places) 26 | 27 | # Pad with zeroes to a constant width. 28 | n = value.find('.') 29 | if n < 0: 30 | n = len(value) 31 | if n < max_digits - decimal_places: 32 | value = u'0' * (max_digits - decimal_places - n) + value 33 | return sign + value 34 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | from setuptools import setup, find_packages 2 | 3 | 4 | DESCRIPTION = "Djangotoolbox for Django-nonrel" 5 | LONG_DESCRIPTION = None 6 | try: 7 | LONG_DESCRIPTION = open('README.rst').read() 8 | except: 9 | pass 10 | 11 | setup(name='djangotoolbox', 12 | version='1.8.0', 13 | description=DESCRIPTION, 14 | long_description=LONG_DESCRIPTION, 15 | author='Waldemar Kornewald', 16 | author_email='wkornewald@gmail.com', 17 | url='https://github.com/django-nonrel/djangotoolbox', 18 | packages=find_packages(), 19 | license='3-clause BSD', 20 | zip_safe=False, 21 | classifiers=[ 22 | 'Development Status :: 5 - Production/Stable', 23 | 'Environment :: Web Environment', 24 | 'Framework :: Django', 25 | 'Intended Audience :: Developers', 26 | 'License :: OSI Approved :: BSD License', 27 | 'Operating System :: OS Independent', 28 | 'Programming Language :: Python', 29 | 'Programming Language :: Python :: 2.5', 30 | 'Programming Language :: Python :: 2.6', 31 | 'Programming Language :: Python :: 2.7', 32 | 'Programming Language :: Python :: 3.4', 33 | 'Topic :: Database', 34 | 'Topic :: Software Development :: Libraries :: Python Modules', 35 | ], 36 | ) 37 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) Waldemar Kornewald, Thomas Wanschik, and all contributors. 2 | All rights reserved. 3 | 4 | Redistribution and use in source and binary forms, with or without modification, 5 | are permitted provided that the following conditions are met: 6 | 7 | 1. Redistributions of source code must retain the above copyright notice, 8 | this list of conditions and the following disclaimer. 9 | 10 | 2. Redistributions in binary form must reproduce the above copyright 11 | notice, this list of conditions and the following disclaimer in the 12 | documentation and/or other materials provided with the distribution. 13 | 14 | 3. Neither the name of All Buttons Pressed nor 15 | the names of its contributors may be used to endorse or promote products 16 | derived from this software without specific prior written permission. 17 | 18 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND 19 | ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 20 | WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 21 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR 22 | ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 23 | (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 24 | LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON 25 | ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 26 | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 27 | SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 | -------------------------------------------------------------------------------- /djangotoolbox/sites/dynamicsite.py: -------------------------------------------------------------------------------- 1 | from django.conf import settings 2 | from django.core.cache import cache 3 | from django.contrib.sites.models import Site 4 | from djangotoolbox.utils import make_tls_property 5 | 6 | 7 | _default_site_id = getattr(settings, 'SITE_ID', None) 8 | SITE_ID = settings.__class__.SITE_ID = make_tls_property() 9 | 10 | 11 | class DynamicSiteIDMiddleware(object): 12 | """Sets settings.SITE_ID based on request's domain.""" 13 | 14 | def process_request(self, request): 15 | # Ignore port if it's 80 or 443 16 | if ':' in request.get_host(): 17 | domain, port = request.get_host().split(':') 18 | if int(port) not in (80, 443): 19 | domain = request.get_host() 20 | else: 21 | domain = request.get_host().split(':')[0] 22 | 23 | # Domains are case insensitive 24 | domain = domain.lower() 25 | 26 | # We cache the SITE_ID 27 | cache_key = 'Site:domain:%s' % domain 28 | site = cache.get(cache_key) 29 | if site: 30 | SITE_ID.value = site 31 | else: 32 | try: 33 | site = Site.objects.get(domain=domain) 34 | except Site.DoesNotExist: 35 | site = None 36 | 37 | if not site: 38 | # Fall back to with/without 'www.' 39 | if domain.startswith('www.'): 40 | fallback_domain = domain[4:] 41 | else: 42 | fallback_domain = 'www.' + domain 43 | 44 | try: 45 | site = Site.objects.get(domain=fallback_domain) 46 | except Site.DoesNotExist: 47 | site = None 48 | 49 | # Add site if it doesn't exist 50 | if not site and getattr(settings, 'CREATE_SITES_AUTOMATICALLY', 51 | True): 52 | site = Site(domain=domain, name=domain) 53 | site.save() 54 | 55 | # Set SITE_ID for this thread/request 56 | if site: 57 | SITE_ID.value = site.pk 58 | else: 59 | SITE_ID.value = _default_site_id 60 | 61 | cache.set(cache_key, SITE_ID.value, 5 * 60) 62 | -------------------------------------------------------------------------------- /CHANGELOG.rst: -------------------------------------------------------------------------------- 1 | Changelog 2 | ========= 3 | 4 | Version 1.8.0 (Jul 19, 2015) 5 | ------------- 6 | 7 | * Added support for Django 1.8 in the db package (Thanks @kavdev) 8 | 9 | Version 1.7.0 (Jul 12, 2015) 10 | ------------- 11 | 12 | * Add support for Django 1.7 13 | 14 | Version 1.6.2 (Dec 20, 2013) 15 | ------------- 16 | 17 | * Fixed broken query in User admin 18 | 19 | Version 1.6.1 (Nov 29, 2013) 20 | ------------- 21 | 22 | * Fixed packaging issues 23 | 24 | Version 1.6.0 25 | ------------- 26 | 27 | Note: This is release includes support for Django versions 1.4, 1.5 and 1.6. 28 | You no longer need to use the separate branches for each Django version. 29 | 30 | * Added support for Django 1.6 31 | 32 | Version 1.5.0 33 | ------------- 34 | 35 | * Added support for Django 1.5 36 | 37 | Version 1.4.0 38 | ------------- 39 | 40 | * Added support for Django 1.4 41 | 42 | Version 0.9.2 43 | ------------- 44 | 45 | * Moved all code required for value conversion for / deconversion from 46 | database to ``DatabaseOperations`` 47 | * Moved nonrel fields' values handling to toolbox, allowing backends to 48 | just select an appropriate storage type (including string and binary) 49 | * Moved decimal-to-string implementation preserving comparisons to 50 | toolbox (previously in the GAE backend) 51 | * Let backends use the QuerySet's ``standard_ordering`` when no field 52 | ordering is defined 53 | * Fixed conversion of values for ``EmbeddedModelField`` subfields 54 | * Fixed preparation of lookup arguments for ``List/Set/DictField`` 55 | * Fixed value comparisons in in-memory filtering (only used by GAE) 56 | * Fixed ``update`` for ``EmbeddedModelField`` nested in a nonrel field 57 | 58 | Version 0.9.1 59 | ------------- 60 | 61 | * Added lazy model lookups to EmbeddedModelField 62 | * Simplified CapturingTestSuiteRunner by using Django's integrated unittest2 package 63 | * Several new unit tests 64 | 65 | Version 0.8.1 66 | ------------- 67 | 68 | * Added default implementation for ``check_aggregate_support()``. Contributed by Jonas Haag 69 | * Added ``ListField``/etc. support for fields that require ``SubfieldBase`` 70 | 71 | Version 0.8 72 | ----------- 73 | 74 | This release unifies the field types of all existing nonrel backends. 75 | 76 | * Merged with ``ListField`` from MongoDB backend. Contributed by Jonas Haag 77 | * Added ``SetField``, ``DictField``, and ``RawField``. Contributed by Jonas Haag 78 | * Fixed support for proxy models. Contributed by Vladimir Mihailenco 79 | * Several cleanups and minor bug fixes 80 | -------------------------------------------------------------------------------- /djangotoolbox/middleware.py: -------------------------------------------------------------------------------- 1 | from django.conf import settings 2 | from django.http import HttpResponseRedirect 3 | from django.utils.cache import patch_cache_control 4 | 5 | 6 | LOGIN_REQUIRED_PREFIXES = getattr(settings, 'LOGIN_REQUIRED_PREFIXES', ()) 7 | NO_LOGIN_REQUIRED_PREFIXES = getattr(settings, 8 | 'NO_LOGIN_REQUIRED_PREFIXES', ()) 9 | 10 | ALLOWED_DOMAINS = getattr(settings, 'ALLOWED_DOMAINS', None) 11 | NON_REDIRECTED_PATHS = getattr(settings, 'NON_REDIRECTED_PATHS', ()) 12 | NON_REDIRECTED_BASE_PATHS = tuple(path.rstrip('/') + '/' 13 | for path in NON_REDIRECTED_PATHS) 14 | 15 | 16 | class LoginRequiredMiddleware(object): 17 | """ 18 | Redirects to login page if request path begins with a 19 | LOGIN_REQURED_PREFIXES prefix. You can also specify 20 | NO_LOGIN_REQUIRED_PREFIXES which take precedence. 21 | """ 22 | 23 | def process_request(self, request): 24 | for prefix in NO_LOGIN_REQUIRED_PREFIXES: 25 | if request.path.startswith(prefix): 26 | return None 27 | for prefix in LOGIN_REQUIRED_PREFIXES: 28 | if request.path.startswith(prefix) and \ 29 | not request.user.is_authenticated(): 30 | from django.contrib.auth.views import redirect_to_login 31 | return redirect_to_login(request.get_full_path()) 32 | return None 33 | 34 | 35 | class RedirectMiddleware(object): 36 | """ 37 | A static redirect middleware. Mostly useful for hosting providers 38 | that automatically setup an alternative domain for your website. 39 | You might not want anyone to access the site via those possibly 40 | well-known URLs. 41 | """ 42 | 43 | def process_request(self, request): 44 | host = request.get_host().split(':')[0] 45 | # Turn off redirects when in debug mode, running unit tests, or 46 | # when handling an App Engine cron job. 47 | if (settings.DEBUG or host == 'testserver' or 48 | not ALLOWED_DOMAINS or 49 | request.META.get('HTTP_X_APPENGINE_CRON') == 'true' or 50 | request.path.startswith('/_ah/') or 51 | request.path in NON_REDIRECTED_PATHS or 52 | request.path.startswith(NON_REDIRECTED_BASE_PATHS)): 53 | return 54 | if host not in settings.ALLOWED_DOMAINS: 55 | return HttpResponseRedirect( 56 | 'http://' + settings.ALLOWED_DOMAINS[0] + request.path) 57 | 58 | 59 | class NoHistoryCacheMiddleware(object): 60 | """ 61 | If user is authenticated we disable browser caching of pages in 62 | history. 63 | """ 64 | 65 | def process_response(self, request, response): 66 | if 'Expires' not in response and \ 67 | 'Cache-Control' not in response and \ 68 | hasattr(request, 'session') and \ 69 | request.user.is_authenticated(): 70 | patch_cache_control(response, 71 | no_store=True, no_cache=True, must_revalidate=True, max_age=0) 72 | return response 73 | -------------------------------------------------------------------------------- /djangotoolbox/utils.py: -------------------------------------------------------------------------------- 1 | def make_tls_property(default=None): 2 | """ 3 | Creates a class-wide instance property with a thread-specific 4 | value. 5 | """ 6 | 7 | class TLSProperty(object): 8 | 9 | def __init__(self): 10 | from threading import local 11 | self.local = local() 12 | 13 | def __get__(self, instance, cls): 14 | if not instance: 15 | return self 16 | return self.value 17 | 18 | def __set__(self, instance, value): 19 | self.value = value 20 | 21 | def _get_value(self): 22 | return getattr(self.local, 'value', default) 23 | 24 | def _set_value(self, value): 25 | self.local.value = value 26 | value = property(_get_value, _set_value) 27 | 28 | return TLSProperty() 29 | 30 | 31 | def getattr_by_path(obj, attr, *default): 32 | """ 33 | Like getattr(), but can go down a hierarchy like "attr.subattr". 34 | """ 35 | value = obj 36 | for part in attr.split('.'): 37 | if not hasattr(value, part) and len(default): 38 | return default[0] 39 | value = getattr(value, part) 40 | if callable(value): 41 | value = value() 42 | return value 43 | 44 | 45 | def subdict(data, *attrs): 46 | """Returns a subset of the keys of a dictionary.""" 47 | result = {} 48 | result.update([(key, data[key]) for key in attrs]) 49 | return result 50 | 51 | 52 | def equal_lists(left, right): 53 | """ 54 | Compares two lists and returs True if they contain the same 55 | elements, but doesn't require that they have the same order. 56 | """ 57 | right = list(right) 58 | if len(left) != len(right): 59 | return False 60 | for item in left: 61 | if item in right: 62 | del right[right.index(item)] 63 | else: 64 | return False 65 | return True 66 | 67 | 68 | def object_list_to_table(headings, dict_list): 69 | """ 70 | Converts objects to table-style list of rows with heading: 71 | 72 | Example: 73 | x.a = 1 74 | x.b = 2 75 | x.c = 3 76 | y.a = 11 77 | y.b = 12 78 | y.c = 13 79 | object_list_to_table(('a', 'b', 'c'), [x, y]) 80 | results in the following (dict keys reordered for better readability): 81 | [ 82 | ('a', 'b', 'c'), 83 | (1, 2, 3), 84 | (11, 12, 13), 85 | ] 86 | """ 87 | return [headings] + [tuple([getattr_by_path(row, heading, None) 88 | for heading in headings]) 89 | for row in dict_list] 90 | 91 | 92 | def dict_list_to_table(headings, dict_list): 93 | """ 94 | Converts dict to table-style list of rows with heading: 95 | 96 | Example: 97 | dict_list_to_table(('a', 'b', 'c'), 98 | [{'a': 1, 'b': 2, 'c': 3}, {'a': 11, 'b': 12, 'c': 13}]) 99 | results in the following (dict keys reordered for better readability): 100 | [ 101 | ('a', 'b', 'c'), 102 | (1, 2, 3), 103 | (11, 12, 13), 104 | ] 105 | """ 106 | return [headings] + [tuple([row[heading] for heading in headings]) 107 | for row in dict_list] 108 | -------------------------------------------------------------------------------- /djangotoolbox/test.py: -------------------------------------------------------------------------------- 1 | from django.test import TestCase 2 | from django.utils.unittest import TextTestResult, TextTestRunner 3 | 4 | try: 5 | from django.test.runner import DiscoverRunner as TestRunner 6 | except ImportError: 7 | from django.test.simple import DjangoTestSuiteRunner as TestRunner 8 | 9 | from .utils import object_list_to_table 10 | 11 | import re 12 | 13 | 14 | class ModelTestCase(TestCase): 15 | """ 16 | A test case for models that provides an easy way to validate the DB 17 | contents against a given list of row-values. 18 | 19 | You have to specify the model to validate using the 'model' 20 | attribute: 21 | 22 | class MyTestCase(ModelTestCase): 23 | model = MyModel 24 | """ 25 | 26 | def validate_state(self, columns, *state_table): 27 | """ 28 | Validates that the DB contains exactly the values given in the 29 | state table. The list of columns is given in the columns tuple. 30 | 31 | Example: 32 | self.validate_state( 33 | ('a', 'b', 'c'), 34 | (1, 2, 3), 35 | (11, 12, 13), 36 | ) 37 | validates that the table contains exactly two rows and that 38 | their 'a', 'b', and 'c' attributes are 1, 2, 3 for one row and 39 | 11, 12, 13 for the other row. The order of the rows doesn't 40 | matter. 41 | """ 42 | current_state = object_list_to_table( 43 | columns, self.model.all())[1:] 44 | if not equal_lists(current_state, state_table): 45 | print("DB state not valid:") 46 | print("Current state:") 47 | print(columns) 48 | for state in current_state: 49 | print(state) 50 | print("Should be:") 51 | for state in state_table: 52 | print(state) 53 | self.fail("DB state not valid.") 54 | 55 | 56 | class CapturingTestSuiteRunner(TestRunner): 57 | """ 58 | Captures stdout/stderr during test and shows them next to 59 | tracebacks. 60 | """ 61 | 62 | def run_suite(self, suite, **kwargs): 63 | return TextTestRunner(verbosity=self.verbosity, 64 | failfast=self.failfast, 65 | buffer=True).run(suite) 66 | 67 | _EXPECTED_ERRORS = [ 68 | r"This query is not supported by the database\.", 69 | r"Multi-table inheritance is not supported by non-relational DBs\.", 70 | r"TextField is not indexed, by default, so you can't filter on it\.", 71 | r"First ordering property must be the same as inequality filter property", 72 | r"This database doesn't support filtering on non-primary key ForeignKey fields\.", 73 | r"Only AND filters are supported\.", 74 | r"MultiQuery does not support keys_only\.", 75 | r"You can't query against more than 30 __in filter value combinations\.", 76 | r"Only strings and positive integers may be used as keys on GAE\.", 77 | r"This database does not support aggregates\.", 78 | r"Subqueries are not supported $yet$\.", 79 | r"Cursors are not supported\.", 80 | r"This database backend only supports count queries on the primary key\.", 81 | r"AutoField $default primary key$ values must be strings representing an ObjectId on MongoDB", 82 | ] 83 | 84 | 85 | class NonrelTestResult(TextTestResult): 86 | def __init__(self, *args, **kwargs): 87 | super(NonrelTestResult, self).__init__(*args, **kwargs) 88 | self._compiled_exception_matchers = [re.compile(expr) for expr in _EXPECTED_ERRORS] 89 | 90 | def __match_exception(self, exc): 91 | for exc_match in self._compiled_exception_matchers: 92 | if exc_match.search(str(exc)): 93 | return True 94 | return False 95 | 96 | def addError(self, test, err): 97 | exc = err[1] 98 | if self.__match_exception(exc): 99 | super(NonrelTestResult, self).addExpectedFailure(test, err) 100 | else: 101 | super(NonrelTestResult, self).addError(test, err) 102 | 103 | 104 | class NonrelTestSuiteRunner(TestRunner): 105 | def run_suite(self, suite, **kwargs): 106 | return TextTestRunner( 107 | verbosity=self.verbosity, 108 | failfast=self.failfast, 109 | resultclass=NonrelTestResult, 110 | buffer=False 111 | ).run(suite) 112 | -------------------------------------------------------------------------------- /djangotoolbox/db/creation.py: -------------------------------------------------------------------------------- 1 | import django 2 | 3 | if django.VERSION < (1, 8): 4 | from django.db.backends.creation import BaseDatabaseCreation 5 | else: 6 | from django.db.backends.base.creation import BaseDatabaseCreation 7 | 8 | 9 | class NonrelDatabaseCreation(BaseDatabaseCreation): 10 | 11 | # "Types" used by database conversion methods to decide how to 12 | # convert data for or from the database. Type is understood here 13 | # a bit differently than in vanilla Django -- it should be read 14 | # as an identifier of an encoding / decoding procedure rather than 15 | # just a database column type. 16 | data_types = { 17 | 18 | # NoSQL databases often have specific concepts of entity keys. 19 | # For example, GAE has the db.Key class, MongoDB likes to use 20 | # ObjectIds, Redis uses strings, while Cassandra supports 21 | # different types (including binary data). 22 | 'AutoField': 'key', 23 | 'RelatedAutoField': 'key', 24 | 'ForeignKey': 'key', 25 | 'OneToOneField': 'key', 26 | 'ManyToManyField': 'key', 27 | 28 | # Standard field types, more or less suitable for a database 29 | # (or its client / driver) being able to directly store or 30 | # process Python objects. 31 | 'BigIntegerField': 'long', 32 | 'BooleanField': 'bool', 33 | 'CharField': 'string', 34 | 'CommaSeparatedIntegerField': 'string', 35 | 'DateField': 'date', 36 | 'DateTimeField': 'datetime', 37 | 'DecimalField': 'decimal', 38 | 'EmailField': 'string', 39 | 'FileField': 'string', 40 | 'FilePathField': 'string', 41 | 'FloatField': 'float', 42 | 'ImageField': 'string', 43 | 'IntegerField': 'integer', 44 | 'IPAddressField': 'string', 45 | 'NullBooleanField': 'bool', 46 | 'PositiveIntegerField': 'integer', 47 | 'PositiveSmallIntegerField': 'integer', 48 | 'SlugField': 'string', 49 | 'SmallIntegerField': 'integer', 50 | 'TextField': 'string', 51 | 'TimeField': 'time', 52 | 'URLField': 'string', 53 | 54 | # You may use "list" for SetField, or even DictField and 55 | # EmbeddedModelField (if your database supports nested lists). 56 | # All following fields also support "string" and "bytes" as 57 | # their storage types -- which work by serializing using pickle 58 | # protocol 0 or 2 respectively. 59 | # Please note that if you can't support the "natural" storage 60 | # type then the order of field values will be undetermined, and 61 | # lookups or filters may not work as specified (e.g. the same 62 | # set or dict may be represented by different lists, with 63 | # elements in different order, so the same two instances may 64 | # compare one way or the other). 65 | 'AbstractIterableField': 'list', 66 | 'ListField': 'list', 67 | 'SetField': 'set', 68 | 'DictField': 'dict', 69 | 'EmbeddedModelField': 'dict', 70 | 71 | # RawFields ("raw" db_type) are used when type is not known 72 | # (untyped collections) or for values that do not come from 73 | # a field at all (model info serialization), only do generic 74 | # processing for them (if any). On the other hand, anything 75 | # using the "bytes" db_type should be converted to a database 76 | # blob type or stored as binary data. 77 | 'RawField': 'raw', 78 | 'BlobField': 'bytes', 79 | } 80 | 81 | def db_type(self, field): 82 | """ 83 | Allows back-ends to override db_type determined by the field. 84 | 85 | This has to be called instead of the Field.db_type, because we 86 | may need to override a db_type a custom field returns directly, 87 | and need more freedom in handling types of primary keys and 88 | related fields. 89 | 90 | :param field: A field we want to know the storage type of 91 | 92 | TODO: Field.db_type (as of 1.3.1) is used mostly for generating 93 | SQL statements (through a couple of methods in 94 | DatabaseCreation and DatabaseOperations.field_cast_sql) 95 | or within back-end implementations -- nonrel is not 96 | dependend on any of these; but there are two cases that 97 | might need to be fixed, namely: 98 | -- management/createcachetable (calls field.db_type), 99 | -- and contrib/gis (defines its own geo_db_type method). 100 | """ 101 | return field.db_type(connection=self.connection) 102 | 103 | def sql_create_model(self, model, style, known_models=set()): 104 | """ 105 | Most NoSQL databases are mostly schema-less, no data 106 | definitions are needed. 107 | """ 108 | return [], {} 109 | 110 | def sql_indexes_for_model(self, model, style): 111 | """ 112 | Creates all indexes needed for local (not inherited) fields of 113 | a model. 114 | """ 115 | return [] 116 | -------------------------------------------------------------------------------- /djangotoolbox/fields.py: -------------------------------------------------------------------------------- 1 | # All fields except for BlobField written by Jonas Haag 2 | 3 | from django.core.exceptions import ValidationError 4 | from django.utils.importlib import import_module 5 | from django.db import models 6 | from django.db.models.fields.subclassing import Creator 7 | from django.db.utils import IntegrityError 8 | from django.db.models.fields.related import add_lazy_relation 9 | 10 | 11 | __all__ = ('RawField', 'ListField', 'SetField', 'DictField', 12 | 'EmbeddedModelField', 'BlobField') 13 | 14 | 15 | EMPTY_ITER = () 16 | 17 | 18 | class _FakeModel(object): 19 | """ 20 | An object of this class can pass itself off as a model instance 21 | when used as an arguments to Field.pre_save method (item_fields 22 | of iterable fields are not actually fields of any model). 23 | """ 24 | 25 | def __init__(self, field, value): 26 | setattr(self, field.attname, value) 27 | 28 | 29 | class RawField(models.Field): 30 | """ 31 | Generic field to store anything your database backend allows you 32 | to. No validation or conversions are done for this field. 33 | """ 34 | 35 | def get_internal_type(self): 36 | """ 37 | Returns this field's kind. Nonrel fields are meant to extend 38 | the set of standard fields, so fields subclassing them should 39 | get the same internal type, rather than their own class name. 40 | """ 41 | return 'RawField' 42 | 43 | 44 | class AbstractIterableField(models.Field): 45 | """ 46 | Abstract field for fields for storing iterable data type like 47 | ``list``, ``set`` and ``dict``. 48 | 49 | You can pass an instance of a field as the first argument. 50 | If you do, the iterable items will be piped through the passed 51 | field's validation and conversion routines, converting the items 52 | to the appropriate data type. 53 | """ 54 | 55 | def __init__(self, item_field=None, *args, **kwargs): 56 | default = kwargs.get( 57 | 'default', None if kwargs.get('null') else EMPTY_ITER) 58 | 59 | # Ensure a new object is created every time the default is 60 | # accessed. 61 | if default is not None and not callable(default): 62 | kwargs['default'] = lambda: self._type(default) 63 | 64 | super(AbstractIterableField, self).__init__(*args, **kwargs) 65 | 66 | # Either use the provided item_field or a RawField. 67 | if item_field is None: 68 | item_field = RawField() 69 | elif callable(item_field): 70 | item_field = item_field() 71 | self.item_field = item_field 72 | 73 | # We'll be pretending that item_field is a field of a model 74 | # with just one "value" field. 75 | assert not hasattr(self.item_field, 'attname') 76 | self.item_field.set_attributes_from_name('value') 77 | 78 | def contribute_to_class(self, cls, name): 79 | self.item_field.model = cls 80 | self.item_field.name = name 81 | super(AbstractIterableField, self).contribute_to_class(cls, name) 82 | 83 | # If items' field uses SubfieldBase we also need to. 84 | item_metaclass = getattr(self.item_field, '__metaclass__', None) 85 | if item_metaclass and issubclass(item_metaclass, models.SubfieldBase): 86 | setattr(cls, self.name, Creator(self)) 87 | 88 | if isinstance(self.item_field, models.ForeignKey) and isinstance(self.item_field.rel.to, basestring): 89 | """ 90 | If rel.to is a string because the actual class is not yet defined, look up the 91 | actual class later. Refer to django.models.fields.related.RelatedField.contribute_to_class. 92 | """ 93 | def _resolve_lookup(_, resolved_model, __): 94 | self.item_field.rel.to = resolved_model 95 | self.item_field.do_related_class(self, cls) 96 | 97 | add_lazy_relation(cls, self, self.item_field.rel.to, _resolve_lookup) 98 | 99 | def _map(self, function, iterable, *args, **kwargs): 100 | """ 101 | Applies the function to items of the iterable and returns 102 | an iterable of the proper type for the field. 103 | 104 | Overriden by DictField to only apply the function to values. 105 | """ 106 | return self._type(function(element, *args, **kwargs) 107 | for element in iterable) 108 | 109 | def to_python(self, value): 110 | """ 111 | Passes value items through item_field's to_python. 112 | """ 113 | if value is None: 114 | return None 115 | return self._map(self.item_field.to_python, value) 116 | 117 | def pre_save(self, model_instance, add): 118 | """ 119 | Gets our value from the model_instance and passes its items 120 | through item_field's pre_save (using a fake model instance). 121 | """ 122 | value = getattr(model_instance, self.attname) 123 | if value is None: 124 | return None 125 | return self._map( 126 | lambda item: self.item_field.pre_save( 127 | _FakeModel(self.item_field, item), add), 128 | value) 129 | 130 | def get_db_prep_save(self, value, connection): 131 | """ 132 | Applies get_db_prep_save of item_field on value items. 133 | """ 134 | if value is None: 135 | return None 136 | return self._map(self.item_field.get_db_prep_save, value, 137 | connection=connection) 138 | 139 | def get_db_prep_lookup(self, lookup_type, value, connection, 140 | prepared=False): 141 | """ 142 | Passes the value through get_db_prep_lookup of item_field. 143 | """ 144 | 145 | # TODO/XXX: Remove as_lookup_value() once we have a cleaner 146 | # solution for dot-notation queries. 147 | # See: https://groups.google.com/group/django-non-relational/browse_thread/thread/6056f8384c9caf04/89eeb9fb22ad16f3). 148 | if hasattr(value, 'as_lookup_value'): 149 | value = value.as_lookup_value(self, lookup_type, connection) 150 | 151 | return self.item_field.get_db_prep_lookup( 152 | lookup_type, value, connection=connection, prepared=prepared) 153 | 154 | def validate(self, values, model_instance): 155 | try: 156 | iter(values) 157 | except TypeError: 158 | raise ValidationError("Value of type %r is not iterable." % 159 | type(values)) 160 | 161 | def formfield(self, **kwargs): 162 | raise NotImplementedError("No form field implemented for %r." % 163 | type(self)) 164 | 165 | 166 | class ListField(AbstractIterableField): 167 | """ 168 | Field representing a Python ``list``. 169 | 170 | If the optional keyword argument `ordering` is given, it must be a 171 | callable that is passed to :meth:`list.sort` as `key` argument. If 172 | `ordering` is given, the items in the list will be sorted before 173 | sending them to the database. 174 | """ 175 | _type = list 176 | 177 | def __init__(self, *args, **kwargs): 178 | self.ordering = kwargs.pop('ordering', None) 179 | if self.ordering is not None and not callable(self.ordering): 180 | raise TypeError("'ordering' has to be a callable or None, " 181 | "not of type %r." % type(self.ordering)) 182 | super(ListField, self).__init__(*args, **kwargs) 183 | 184 | def get_internal_type(self): 185 | return 'ListField' 186 | 187 | def pre_save(self, model_instance, add): 188 | value = getattr(model_instance, self.attname) 189 | if value is None: 190 | return None 191 | if value and self.ordering: 192 | value.sort(key=self.ordering) 193 | return super(ListField, self).pre_save(model_instance, add) 194 | 195 | 196 | class SetField(AbstractIterableField): 197 | """ 198 | Field representing a Python ``set``. 199 | """ 200 | _type = set 201 | 202 | def get_internal_type(self): 203 | return 'SetField' 204 | 205 | def value_to_string(self, obj): 206 | """ 207 | Custom method for serialization, as JSON doesn't support 208 | serializing sets. 209 | """ 210 | return list(self._get_val_from_obj(obj)) 211 | 212 | 213 | class DictField(AbstractIterableField): 214 | """ 215 | Field representing a Python ``dict``. 216 | 217 | Type conversions described in :class:`AbstractIterableField` only 218 | affect values of the dictionary, not keys. Depending on the 219 | back-end, keys that aren't strings might not be allowed. 220 | """ 221 | _type = dict 222 | 223 | def get_internal_type(self): 224 | return 'DictField' 225 | 226 | def _map(self, function, iterable, *args, **kwargs): 227 | return self._type((key, function(value, *args, **kwargs)) 228 | for key, value in iterable.iteritems()) 229 | 230 | def validate(self, values, model_instance): 231 | if not isinstance(values, dict): 232 | raise ValidationError("Value is of type %r. Should be a dict." % 233 | type(values)) 234 | 235 | 236 | class EmbeddedModelField(models.Field): 237 | """ 238 | Field that allows you to embed a model instance. 239 | 240 | :param embedded_model: (optional) The model class of instances we 241 | will be embedding; may also be passed as a 242 | string, similar to relation fields 243 | 244 | TODO: Make sure to delegate all signals and other field methods to 245 | the embedded instance (not just pre_save, get_db_prep_* and 246 | to_python). 247 | """ 248 | __metaclass__ = models.SubfieldBase 249 | 250 | def __init__(self, embedded_model=None, *args, **kwargs): 251 | self.embedded_model = embedded_model 252 | kwargs.setdefault('default', None) 253 | super(EmbeddedModelField, self).__init__(*args, **kwargs) 254 | 255 | def get_internal_type(self): 256 | return 'EmbeddedModelField' 257 | 258 | 259 | def _set_model(self, model): 260 | """ 261 | Resolves embedded model class once the field knows the model it 262 | belongs to. 263 | 264 | If the model argument passed to __init__ was a string, we need 265 | to make sure to resolve that string to the corresponding model 266 | class, similar to relation fields. 267 | However, we need to know our own model to generate a valid key 268 | for the embedded model class lookup and EmbeddedModelFields are 269 | not contributed_to_class if used in iterable fields. Thus we 270 | rely on the collection field telling us its model (by setting 271 | our "model" attribute in its contribute_to_class method). 272 | """ 273 | self._model = model 274 | if model is not None and isinstance(self.embedded_model, basestring): 275 | 276 | def _resolve_lookup(self_, resolved_model, model): 277 | self.embedded_model = resolved_model 278 | 279 | add_lazy_relation(model, self, self.embedded_model, _resolve_lookup) 280 | 281 | model = property(lambda self: self._model, _set_model) 282 | 283 | 284 | def stored_model(self, column_values): 285 | """ 286 | Returns the fixed embedded_model this field was initialized 287 | with (typed embedding) or tries to determine the model from 288 | _module / _model keys stored together with column_values 289 | (untyped embedding). 290 | 291 | We give precedence to the field's definition model, as silently 292 | using a differing serialized one could hide some data integrity 293 | problems. 294 | 295 | Note that a single untyped EmbeddedModelField may process 296 | instances of different models (especially when used as a type 297 | of a collection field). 298 | """ 299 | module = column_values.pop('_module', None) 300 | model = column_values.pop('_model', None) 301 | if self.embedded_model is not None: 302 | return self.embedded_model 303 | elif module is not None: 304 | return getattr(import_module(module), model) 305 | else: 306 | raise IntegrityError("Untyped EmbeddedModelField trying to load " 307 | "data without serialized model class info.") 308 | 309 | def to_python(self, value): 310 | """ 311 | Passes embedded model fields' values through embedded fields 312 | to_python methods and reinstiatates the embedded instance. 313 | 314 | We expect to receive a field.attname => value dict together 315 | with a model class from back-end database deconversion (which 316 | needs to know fields of the model beforehand). 317 | """ 318 | 319 | # Either the model class has already been determined during 320 | # deconverting values from the database or we've got a dict 321 | # from a deserializer that may contain model class info. 322 | if isinstance(value, tuple): 323 | embedded_model, attribute_values = value 324 | elif isinstance(value, dict): 325 | embedded_model = self.stored_model(value) 326 | attribute_values = value 327 | else: 328 | return value 329 | 330 | # Pass values through respective fields' to_python, leaving 331 | # fields for which no value is specified uninitialized. 332 | attribute_values = dict( 333 | (field.attname, field.to_python(attribute_values[field.attname])) 334 | for field in embedded_model._meta.fields 335 | if field.attname in attribute_values) 336 | 337 | # Create the model instance. 338 | instance = embedded_model(**attribute_values) 339 | instance._state.adding = False 340 | return instance 341 | 342 | def get_db_prep_save(self, embedded_instance, connection): 343 | """ 344 | Applies pre_save and get_db_prep_save of embedded instance 345 | fields and passes a field => value mapping down to database 346 | type conversions. 347 | 348 | The embedded instance will be saved as a column => value dict 349 | in the end (possibly augmented with info about instance's model 350 | for untyped embedding), but because we need to apply database 351 | type conversions on embedded instance fields' values and for 352 | these we need to know fields those values come from, we need to 353 | entrust the database layer with creating the dict. 354 | """ 355 | if embedded_instance is None: 356 | return None 357 | 358 | # The field's value should be an instance of the model given in 359 | # its declaration or at least of some model. 360 | embedded_model = self.embedded_model or models.Model 361 | if not isinstance(embedded_instance, embedded_model): 362 | raise TypeError("Expected instance of type %r, not %r." % 363 | (embedded_model, type(embedded_instance))) 364 | 365 | # Apply pre_save and get_db_prep_save of embedded instance 366 | # fields, create the field => value mapping to be passed to 367 | # storage preprocessing. 368 | field_values = {} 369 | add = embedded_instance._state.adding 370 | for field in embedded_instance._meta.fields: 371 | value = field.get_db_prep_save( 372 | field.pre_save(embedded_instance, add), connection=connection) 373 | 374 | # Exclude unset primary keys (e.g. {'id': None}). 375 | if field.primary_key and value is None: 376 | continue 377 | 378 | field_values[field] = value 379 | 380 | # Let untyped fields store model info alongside values. 381 | # We use fake RawFields for additional values to avoid passing 382 | # embedded_instance to database conversions and to give 383 | # back-ends a chance to apply generic conversions. 384 | if self.embedded_model is None: 385 | module_field = RawField() 386 | module_field.set_attributes_from_name('_module') 387 | model_field = RawField() 388 | model_field.set_attributes_from_name('_model') 389 | field_values.update( 390 | ((module_field, embedded_instance.__class__.__module__), 391 | (model_field, embedded_instance.__class__.__name__))) 392 | 393 | # This instance will exist in the database soon. 394 | # TODO.XXX: Ensure that this doesn't cause race conditions. 395 | embedded_instance._state.adding = False 396 | 397 | return field_values 398 | 399 | # TODO/XXX: Remove this once we have a cleaner solution. 400 | def get_db_prep_lookup(self, lookup_type, value, connection, 401 | prepared=False): 402 | if hasattr(value, 'as_lookup_value'): 403 | value = value.as_lookup_value(self, lookup_type, connection) 404 | return value 405 | 406 | 407 | class BlobField(models.Field): 408 | """ 409 | A field for storing blobs of binary data. 410 | 411 | The value might either be a string (or something that can be 412 | converted to a string), or a file-like object. 413 | 414 | In the latter case, the object has to provide a ``read`` method 415 | from which the blob is read. 416 | """ 417 | 418 | def get_internal_type(self): 419 | return 'BlobField' 420 | 421 | def formfield(self, **kwargs): 422 | """ 423 | A file widget is provided, but use model FileField or 424 | ImageField for storing specific files most of the time. 425 | """ 426 | from .widgets import BlobWidget 427 | from django.forms import FileField 428 | defaults = {'form_class': FileField, 'widget': BlobWidget} 429 | defaults.update(kwargs) 430 | return super(BlobField, self).formfield(**defaults) 431 | 432 | def get_db_prep_save(self, value, connection): 433 | if hasattr(value, 'read'): 434 | return value.read() 435 | else: 436 | return str(value) 437 | 438 | def get_db_prep_lookup(self, lookup_type, value, connection, 439 | prepared=False): 440 | raise TypeError("BlobFields do not support lookups.") 441 | 442 | def value_to_string(self, obj): 443 | return str(self._get_val_from_obj(obj)) 444 | -------------------------------------------------------------------------------- /djangotoolbox/db/basecompiler.py: -------------------------------------------------------------------------------- 1 | import datetime 2 | 3 | import django 4 | from django.conf import settings 5 | from django.db.models.fields import NOT_PROVIDED 6 | from django.db.models.query import QuerySet 7 | from django.db.models.sql.compiler import SQLCompiler 8 | from django.db.models.sql.constants import MULTI, SINGLE 9 | from django.db.models.sql.where import AND, OR 10 | from django.db.utils import DatabaseError, IntegrityError 11 | from django.utils.tree import Node 12 | from django.db import connections 13 | 14 | try: 15 | from django.db.models.sql.where import SubqueryConstraint 16 | except ImportError: 17 | SubqueryConstraint = None 18 | 19 | try: 20 | from django.db.models.sql.datastructures import EmptyResultSet 21 | except ImportError: 22 | class EmptyResultSet(Exception): 23 | pass 24 | 25 | 26 | if django.VERSION >= (1, 5): 27 | from django.db.models.constants import LOOKUP_SEP 28 | else: 29 | from django.db.models.sql.constants import LOOKUP_SEP 30 | 31 | if django.VERSION >= (1, 6): 32 | def get_selected_fields(query): 33 | if query.select: 34 | return [info.field for info in (query.select + 35 | query.related_select_cols)] 36 | else: 37 | return query.model._meta.fields 38 | else: 39 | def get_selected_fields(query): 40 | if query.select_fields: 41 | return (query.select_fields + query.related_select_fields) 42 | else: 43 | return query.model._meta.fields 44 | 45 | EMULATED_OPS = { 46 | 'exact': lambda x, y: y in x if isinstance(x, (list, tuple)) else x == y, 47 | 'iexact': lambda x, y: x.lower() == y.lower(), 48 | 'startswith': lambda x, y: x.startswith(y[0]), 49 | 'istartswith': lambda x, y: x.lower().startswith(y[0].lower()), 50 | 'isnull': lambda x, y: x is None if y else x is not None, 51 | 'in': lambda x, y: x in y, 52 | 'lt': lambda x, y: x < y, 53 | 'lte': lambda x, y: x <= y, 54 | 'gt': lambda x, y: x > y, 55 | 'gte': lambda x, y: x >= y, 56 | } 57 | 58 | 59 | class NonrelQuery(object): 60 | """ 61 | Base class for nonrel queries. 62 | 63 | Compilers build a nonrel query when they want to fetch some data. 64 | They work by first allowing sql.compiler.SQLCompiler to partly build 65 | a sql.Query, constructing a NonrelQuery query on top of it, and then 66 | iterating over its results. 67 | 68 | This class provides in-memory filtering and ordering and a 69 | framework for converting SQL constraint tree built by Django to a 70 | "representation" more suitable for most NoSQL databases. 71 | 72 | TODO: Replace with FetchCompiler, there are too many query concepts 73 | around, and it isn't a good abstraction for NoSQL databases. 74 | 75 | TODO: Nonrel currently uses constraint's tree built by Django for 76 | its SQL back-ends to handle filtering. However, Django 77 | intermingles translating its lookup / filtering abstraction 78 | to a logical formula with some preprocessing for joins and 79 | this results in hacks in nonrel. It would be a better to pull 80 | out SQL-specific parts from the constraints preprocessing. 81 | """ 82 | 83 | # ---------------------------------------------- 84 | # Public API 85 | # ---------------------------------------------- 86 | 87 | def __init__(self, compiler, fields): 88 | self.compiler = compiler 89 | self.connection = compiler.connection 90 | self.ops = compiler.connection.ops 91 | self.query = compiler.query # sql.Query 92 | self.fields = fields 93 | self._negated = False 94 | 95 | def fetch(self, low_mark=0, high_mark=None): 96 | """ 97 | Returns an iterator over some part of query results. 98 | """ 99 | raise NotImplementedError 100 | 101 | def count(self, limit=None): 102 | """ 103 | Returns the number of objects that would be returned, if 104 | this query was executed, up to `limit`. 105 | """ 106 | raise NotImplementedError 107 | 108 | def delete(self): 109 | """ 110 | Called by NonrelDeleteCompiler after it builds a delete query. 111 | """ 112 | raise NotImplementedError 113 | 114 | def order_by(self, ordering): 115 | """ 116 | Reorders query results or execution order. Called by 117 | NonrelCompilers during query building. 118 | 119 | :param ordering: A list with (field, ascending) tuples or a 120 | boolean -- use natural ordering, if any, when 121 | the argument is True and its reverse otherwise 122 | """ 123 | raise NotImplementedError 124 | 125 | def add_filter(self, field, lookup_type, negated, value): 126 | """ 127 | Adds a single constraint to the query. Called by add_filters for 128 | each constraint leaf in the WHERE tree built by Django. 129 | 130 | :param field: Lookup field (instance of Field); field.column 131 | should be used for database keys 132 | :param lookup_type: Lookup name (e.g. "startswith") 133 | :param negated: Is the leaf negated 134 | :param value: Lookup argument, such as a value to compare with; 135 | already prepared for the database 136 | """ 137 | raise NotImplementedError 138 | 139 | def add_filters(self, filters): 140 | """ 141 | Converts a constraint tree (sql.where.WhereNode) created by 142 | Django's SQL query machinery to nonrel style filters, calling 143 | add_filter for each constraint. 144 | 145 | This assumes the database doesn't support alternatives of 146 | constraints, you should override this method if it does. 147 | 148 | TODO: Simulate both conjunctions and alternatives in general 149 | let GAE override conjunctions not to split them into 150 | multiple queries. 151 | """ 152 | if filters.negated: 153 | self._negated = not self._negated 154 | 155 | if not self._negated and filters.connector != AND: 156 | raise DatabaseError("Only AND filters are supported.") 157 | 158 | # Remove unneeded children from the tree. 159 | children = self._get_children(filters.children) 160 | 161 | if self._negated and filters.connector != OR and len(children) > 1: 162 | raise DatabaseError("When negating a whole filter subgroup " 163 | "(e.g. a Q object) the subgroup filters must " 164 | "be connected via OR, so the non-relational " 165 | "backend can convert them like this: " 166 | "'not (a OR b) => (not a) AND (not b)'.") 167 | 168 | # Recursively call the method for internal tree nodes, add a 169 | # filter for each leaf. 170 | for child in children: 171 | if isinstance(child, Node): 172 | self.add_filters(child) 173 | continue 174 | field, lookup_type, value = self._decode_child(child) 175 | self.add_filter(field, lookup_type, self._negated, value) 176 | 177 | if filters.negated: 178 | self._negated = not self._negated 179 | 180 | # ---------------------------------------------- 181 | # Internal API for reuse by subclasses 182 | # ---------------------------------------------- 183 | 184 | def _decode_child(self, child): 185 | """ 186 | Produces arguments suitable for add_filter from a WHERE tree 187 | leaf (a tuple). 188 | """ 189 | 190 | if django.VERSION < (1, 7): 191 | # TODO: Call get_db_prep_lookup directly, constraint.process 192 | # doesn't do much more. 193 | constraint, lookup_type, annotation, value = child 194 | packed, value = constraint.process(lookup_type, value, self.connection) 195 | alias, column, db_type = packed 196 | field = constraint.field 197 | else: 198 | rhs, rhs_params = child.process_rhs(self.compiler, self.connection) 199 | 200 | lookup_type = child.lookup_name 201 | 202 | # Since NoSql databases generally don't support aggregation or 203 | # annotation we simply pass true in this case unless the query has a 204 | # get_aggregation method defined. It's a little troubling however that 205 | # the _nomalize_lookup_value method seems to only use this value in the case 206 | # that the value is an iterable and the lookup_type equals isnull. 207 | if hasattr(self, 'get_aggregation'): 208 | annotation = self.get_aggregation(using=self.connection)[None] 209 | else: 210 | annotation = True 211 | 212 | value = rhs_params 213 | 214 | packed = child.lhs.get_group_by_cols()[0] 215 | 216 | if django.VERSION < (1, 8): 217 | alias, column = packed 218 | else: 219 | alias = packed.alias 220 | column = packed.target.column 221 | field = child.lhs.output_field 222 | 223 | opts = self.query.model._meta 224 | if alias and alias != opts.db_table: 225 | raise DatabaseError("This database doesn't support JOINs " 226 | "and multi-table inheritance.") 227 | 228 | # For parent.child_set queries the field held by the constraint 229 | # is the parent's primary key, while the field the filter 230 | # should consider is the child's foreign key field. 231 | if column != field.column: 232 | if not field.primary_key: 233 | raise DatabaseError("This database doesn't support filtering " 234 | "on non-primary key ForeignKey fields.") 235 | 236 | field = (f for f in opts.fields if f.column == column).next() 237 | assert field.rel is not None 238 | 239 | value = self._normalize_lookup_value( 240 | lookup_type, value, field, annotation) 241 | 242 | return field, lookup_type, value 243 | 244 | def _normalize_lookup_value(self, lookup_type, value, field, annotation): 245 | """ 246 | Undoes preparations done by `Field.get_db_prep_lookup` not 247 | suitable for nonrel back-ends and passes the lookup argument 248 | through nonrel's `value_for_db`. 249 | 250 | TODO: Blank `Field.get_db_prep_lookup` and remove this method. 251 | """ 252 | 253 | # Undo Field.get_db_prep_lookup putting most values in a list 254 | # (a subclass may override this, so check if it's a list) and 255 | # losing the (True / False) argument to the "isnull" lookup. 256 | if lookup_type not in ('in', 'range', 'year') and \ 257 | isinstance(value, (tuple, list)): 258 | if len(value) > 1: 259 | raise DatabaseError("Filter lookup type was %s; expected the " 260 | "filter argument not to be a list. Only " 261 | "'in'-filters can be used with lists." % 262 | lookup_type) 263 | elif lookup_type == 'isnull': 264 | value = annotation 265 | else: 266 | value = value[0] 267 | 268 | # Remove percents added by Field.get_db_prep_lookup (useful 269 | # if one were to use the value in a LIKE expression). 270 | if lookup_type in ('startswith', 'istartswith'): 271 | value = value[:-1] 272 | elif lookup_type in ('endswith', 'iendswith'): 273 | value = value[1:] 274 | elif lookup_type in ('contains', 'icontains'): 275 | value = value[1:-1] 276 | 277 | # Prepare the value for a database using the nonrel framework. 278 | return self.ops.value_for_db(value, field, lookup_type) 279 | 280 | def _get_children(self, children): 281 | """ 282 | Filters out nodes of the given contraint tree not needed for 283 | nonrel queries; checks that given constraints are supported. 284 | """ 285 | result = [] 286 | for child in children: 287 | 288 | if SubqueryConstraint is not None and isinstance(child, SubqueryConstraint): 289 | raise DatabaseError("Subqueries are not supported.") 290 | 291 | if isinstance(child, tuple): 292 | constraint, lookup_type, _, value = child 293 | 294 | # When doing a lookup using a QuerySet Django would use 295 | # a subquery, but this won't work for nonrel. 296 | # TODO: Add a supports_subqueries feature and let 297 | # Django evaluate subqueries instead of passing 298 | # them as SQL strings (QueryWrappers) to 299 | # filtering. 300 | if isinstance(value, QuerySet): 301 | raise DatabaseError("Subqueries are not supported.") 302 | 303 | # Remove leafs that were automatically added by 304 | # sql.Query.add_filter to handle negations of outer 305 | # joins. 306 | if lookup_type == 'isnull' and constraint.field is None: 307 | continue 308 | 309 | result.append(child) 310 | return result 311 | 312 | def _matches_filters(self, entity, filters): 313 | """ 314 | Checks if an entity returned by the database satisfies 315 | constraints in a WHERE tree (in-memory filtering). 316 | """ 317 | 318 | # Filters without rules match everything. 319 | if not filters.children: 320 | return True 321 | 322 | result = filters.connector == AND 323 | 324 | for child in filters.children: 325 | 326 | # Recursively check a subtree, 327 | if isinstance(child, Node): 328 | submatch = self._matches_filters(entity, child) 329 | 330 | # Check constraint leaf, emulating a database condition. 331 | else: 332 | field, lookup_type, lookup_value = self._decode_child(child) 333 | entity_value = entity[field.column] 334 | 335 | if entity_value is None: 336 | if isinstance(lookup_value, (datetime.datetime, datetime.date, 337 | datetime.time)): 338 | submatch = lookup_type in ('lt', 'lte') 339 | elif lookup_type in ( 340 | 'startswith', 'contains', 'endswith', 'iexact', 341 | 'istartswith', 'icontains', 'iendswith'): 342 | submatch = False 343 | else: 344 | submatch = EMULATED_OPS[lookup_type]( 345 | entity_value, lookup_value) 346 | else: 347 | submatch = EMULATED_OPS[lookup_type]( 348 | entity_value, lookup_value) 349 | 350 | if filters.connector == OR and submatch: 351 | result = True 352 | break 353 | elif filters.connector == AND and not submatch: 354 | result = False 355 | break 356 | 357 | if filters.negated: 358 | return not result 359 | return result 360 | 361 | def _order_in_memory(self, lhs, rhs): 362 | for field, ascending in self.compiler._get_ordering(): 363 | column = field.column 364 | 365 | # TOOD: cmp is removed in python 3. Rewrite this logic to leverage 366 | # the __eq__() special function. 367 | a = lhs.get(column) 368 | b = rhs.get(column) 369 | 370 | result = (a > b) - (a < b) 371 | if result != 0: 372 | return result if ascending else -result 373 | return 0 374 | 375 | 376 | class NonrelCompiler(SQLCompiler): 377 | """ 378 | Base class for data fetching back-end compilers. 379 | 380 | Note that nonrel compilers derive from sql.compiler.SQLCompiler and 381 | thus hold a reference to a sql.Query, not a NonrelQuery. 382 | 383 | TODO: Separate FetchCompiler from the abstract NonrelCompiler. 384 | """ 385 | 386 | def __init__(self, query, connection, using): 387 | """ 388 | Initializes the underlying SQLCompiler. 389 | """ 390 | super(NonrelCompiler, self).__init__(query, connection, using) 391 | self.ops = self.connection.ops 392 | 393 | # ---------------------------------------------- 394 | # Public API 395 | # ---------------------------------------------- 396 | 397 | def results_iter(self, results=None): 398 | """ 399 | Returns an iterator over the results from executing query given 400 | to this compiler. Called by QuerySet methods. 401 | """ 402 | 403 | if results is None: 404 | fields = self.get_fields() 405 | try: 406 | results = self.build_query(fields).fetch( 407 | self.query.low_mark, self.query.high_mark) 408 | except EmptyResultSet: 409 | results = [] 410 | 411 | for entity in results: 412 | yield self._make_result(entity, fields) 413 | 414 | def has_results(self): 415 | return self.get_count(check_exists=True) 416 | 417 | def execute_sql(self, result_type=MULTI): 418 | """ 419 | Handles SQL-like aggregate queries. This class only emulates COUNT 420 | by using abstract NonrelQuery.count method. 421 | """ 422 | self.pre_sql_setup() 423 | 424 | aggregates = self.query.aggregate_select.values() 425 | 426 | # Simulate a count(). 427 | if aggregates: 428 | assert len(aggregates) == 1 429 | aggregate = aggregates[0] 430 | if django.VERSION < (1, 8): 431 | if aggregate.sql_function != 'COUNT': 432 | raise NotImplementedError("The database backend only supports count() queries.") 433 | else: 434 | if aggregate.function != 'COUNT': 435 | raise NotImplementedError("The database backend only supports count() queries.") 436 | 437 | opts = self.query.get_meta() 438 | 439 | if django.VERSION < (1, 8): 440 | if aggregate.col != '*' and aggregate.col != (opts.db_table, opts.pk.column): 441 | raise DatabaseError("This database backend only supports " 442 | "count() queries on the primary key.") 443 | else: 444 | # Fair warning: the latter part of this or statement hasn't been tested 445 | if aggregate.input_field.value != '*' and aggregate.input_field != (opts.db_table, opts.pk.column): 446 | raise DatabaseError("This database backend only supports " 447 | "count() queries on the primary key.") 448 | 449 | count = self.get_count() 450 | if result_type is SINGLE: 451 | return [count] 452 | elif result_type is MULTI: 453 | return [[count]] 454 | 455 | # ---------------------------------------------- 456 | # Additional NonrelCompiler API 457 | # ---------------------------------------------- 458 | 459 | def _make_result(self, entity, fields): 460 | """ 461 | Decodes values for the given fields from the database entity. 462 | 463 | The entity is assumed to be a dict using field database column 464 | names as keys. Decodes values using `value_from_db` as well as 465 | the standard `convert_values`. 466 | """ 467 | result = [] 468 | for field in fields: 469 | value = entity.get(field.column, NOT_PROVIDED) 470 | if value is NOT_PROVIDED: 471 | value = field.get_default() 472 | else: 473 | value = self.ops.value_from_db(value, field) 474 | # This is the default behavior of ``query.convert_values`` 475 | # until django 1.8, where multiple converters are a thing. 476 | value = self.connection.ops.convert_values(value, field) 477 | if value is None and not field.null: 478 | raise IntegrityError("Non-nullable field %s can't be None!" % 479 | field.name) 480 | result.append(value) 481 | return result 482 | 483 | def check_query(self): 484 | """ 485 | Checks if the current query is supported by the database. 486 | 487 | In general, we expect queries requiring JOINs (many-to-many 488 | relations, abstract model bases, or model spanning filtering), 489 | using DISTINCT (through `QuerySet.distinct()`, which is not 490 | required in most situations) or using the SQL-specific 491 | `QuerySet.extra()` to not work with nonrel back-ends. 492 | """ 493 | if hasattr(self.query, 'is_empty') and self.query.is_empty(): 494 | raise EmptyResultSet() 495 | if (len([a for a in self.query.alias_map if self.query.alias_refcount[a]]) > 1 496 | or self.query.distinct or self.query.extra or self.query.having): 497 | raise DatabaseError("This query is not supported by the database.") 498 | 499 | def get_count(self, check_exists=False): 500 | """ 501 | Counts objects matching the current filters / constraints. 502 | 503 | :param check_exists: Only check if any object matches 504 | """ 505 | if check_exists: 506 | high_mark = 1 507 | else: 508 | high_mark = self.query.high_mark 509 | try: 510 | return self.build_query().count(high_mark) 511 | except EmptyResultSet: 512 | return 0 513 | 514 | def build_query(self, fields=None): 515 | """ 516 | Checks if the underlying SQL query is supported and prepares 517 | a NonrelQuery to be executed on the database. 518 | """ 519 | self.check_query() 520 | if fields is None: 521 | fields = self.get_fields() 522 | query = self.query_class(self, fields) 523 | query.add_filters(self.query.where) 524 | query.order_by(self._get_ordering()) 525 | 526 | # This at least satisfies the most basic unit tests. 527 | if django.VERSION < (1, 8): 528 | if connections[self.using].use_debug_cursor or (connections[self.using].use_debug_cursor is None and settings.DEBUG): 529 | self.connection.queries.append({'sql': repr(query)}) 530 | else: 531 | if connections[self.using].force_debug_cursor or (connections[self.using].force_debug_cursor is None and settings.DEBUG): 532 | self.connection.queries.append({'sql': repr(query)}) 533 | return query 534 | 535 | def get_fields(self): 536 | """ 537 | Returns fields which should get loaded from the back-end by the 538 | current query. 539 | """ 540 | 541 | # We only set this up here because related_select_fields isn't 542 | # populated until execute_sql() has been called. 543 | fields = get_selected_fields(self.query) 544 | 545 | # If the field was deferred, exclude it from being passed 546 | # into `resolve_columns` because it wasn't selected. 547 | only_load = self.deferred_to_columns() 548 | if only_load: 549 | db_table = self.query.model._meta.db_table 550 | only_load = dict((k, v) for k, v in only_load.items() 551 | if v or k == db_table) 552 | if len(only_load.keys()) > 1: 553 | raise DatabaseError("Multi-table inheritance is not " 554 | "supported by non-relational DBs %s." % 555 | repr(only_load)) 556 | fields = [f for f in fields if db_table in only_load and 557 | f.column in only_load[db_table]] 558 | 559 | query_model = self.query.model 560 | if query_model._meta.proxy: 561 | query_model = query_model._meta.proxy_for_model 562 | 563 | for field in fields: 564 | if field.model._meta != query_model._meta: 565 | raise DatabaseError("Multi-table inheritance is not " 566 | "supported by non-relational DBs.") 567 | return fields 568 | 569 | def _get_ordering(self): 570 | """ 571 | Returns a list of (field, ascending) tuples that the query 572 | results should be ordered by. If there is no field ordering 573 | defined returns just the standard_ordering (a boolean, needed 574 | for MongoDB "$natural" ordering). 575 | """ 576 | opts = self.query.get_meta() 577 | if not self.query.default_ordering: 578 | ordering = self.query.order_by 579 | else: 580 | ordering = self.query.order_by or opts.ordering 581 | 582 | if not ordering: 583 | return self.query.standard_ordering 584 | 585 | field_ordering = [] 586 | for order in ordering: 587 | if LOOKUP_SEP in order: 588 | raise DatabaseError("Ordering can't span tables on " 589 | "non-relational backends (%s)." % order) 590 | if order == '?': 591 | raise DatabaseError("Randomized ordering isn't supported by " 592 | "the backend.") 593 | 594 | ascending = not order.startswith('-') 595 | if not self.query.standard_ordering: 596 | ascending = not ascending 597 | 598 | name = order.lstrip('+-') 599 | if name == 'pk': 600 | name = opts.pk.name 601 | 602 | field_ordering.append((opts.get_field(name), ascending)) 603 | return field_ordering 604 | 605 | 606 | class NonrelInsertCompiler(NonrelCompiler): 607 | """ 608 | Base class for all compliers that create new entities or objects 609 | in the database. It has to define execute_sql method due to being 610 | used in place of a SQLInsertCompiler. 611 | 612 | TODO: Analyze if it's always true that when field is None we should 613 | use the PK from self.query (check if the column assertion 614 | below ever fails). 615 | """ 616 | 617 | def execute_sql(self, return_id=False): 618 | self.pre_sql_setup() 619 | 620 | to_insert = [] 621 | pk_field = self.query.get_meta().pk 622 | for obj in self.query.objs: 623 | field_values = {} 624 | for field in self.query.fields: 625 | value = field.get_db_prep_save( 626 | getattr(obj, field.attname) if self.query.raw else field.pre_save(obj, obj._state.adding), 627 | connection=self.connection 628 | ) 629 | if value is None and not field.null and not field.primary_key: 630 | raise IntegrityError("You can't set %s (a non-nullable " 631 | "field) to None!" % field.name) 632 | 633 | # Prepare value for database, note that query.values have 634 | # already passed through get_db_prep_save. 635 | value = self.ops.value_for_db(value, field) 636 | 637 | field_values[field.column] = value 638 | to_insert.append(field_values) 639 | 640 | key = self.insert(to_insert, return_id=return_id) 641 | 642 | # Pass the key value through normal database deconversion. 643 | return self.ops.convert_values(self.ops.value_from_db(key, pk_field), pk_field) 644 | 645 | def insert(self, values, return_id): 646 | """ 647 | Creates a new entity to represent a model. 648 | 649 | Note that the returned key will go through the same database 650 | deconversions that every value coming from the database does 651 | (`convert_values` and `value_from_db`). 652 | 653 | :param values: The model object as a list of (field, value) 654 | pairs; each value is already prepared for the 655 | database 656 | :param return_id: Whether to return the id or key of the newly 657 | created entity 658 | """ 659 | raise NotImplementedError 660 | 661 | 662 | class NonrelUpdateCompiler(NonrelCompiler): 663 | 664 | def execute_sql(self, result_type): 665 | self.pre_sql_setup() 666 | 667 | values = [] 668 | for field, _, value in self.query.values: 669 | if hasattr(value, 'prepare_database_save'): 670 | value = value.prepare_database_save(field) 671 | else: 672 | value = field.get_db_prep_save(value, 673 | connection=self.connection) 674 | value = self.ops.value_for_db(value, field) 675 | values.append((field, value)) 676 | return self.update(values) 677 | 678 | def update(self, values): 679 | """ 680 | Changes an entity that already exists in the database. 681 | 682 | :param values: A list of (field, new-value) pairs 683 | """ 684 | raise NotImplementedError 685 | 686 | 687 | class NonrelDeleteCompiler(NonrelCompiler): 688 | 689 | def execute_sql(self, result_type=MULTI): 690 | try: 691 | self.build_query([self.query.get_meta().pk]).delete() 692 | except EmptyResultSet: 693 | pass 694 | 695 | 696 | class NonrelAggregateCompiler(NonrelCompiler): 697 | pass 698 | 699 | 700 | class NonrelDateCompiler(NonrelCompiler): 701 | pass 702 | 703 | 704 | class NonrelDateTimeCompiler(NonrelCompiler): 705 | pass 706 | -------------------------------------------------------------------------------- /djangotoolbox/db/base.py: -------------------------------------------------------------------------------- 1 | from django.utils.six.moves import cPickle as pickle 2 | import datetime 3 | 4 | from django.conf import settings 5 | 6 | import django 7 | 8 | if django.VERSION < (1, 8): 9 | from django.db.backends import ( 10 | BaseDatabaseFeatures, 11 | BaseDatabaseOperations, 12 | BaseDatabaseWrapper, 13 | BaseDatabaseClient, 14 | BaseDatabaseValidation, 15 | BaseDatabaseIntrospection) 16 | else: 17 | from django.db.backends.base.base import BaseDatabaseWrapper 18 | from django.db.backends.base.client import BaseDatabaseClient 19 | from django.db.backends.base.features import BaseDatabaseFeatures 20 | from django.db.backends.base.validation import BaseDatabaseValidation 21 | from django.db.backends.base.introspection import BaseDatabaseIntrospection 22 | from django.db.backends.base.operations import BaseDatabaseOperations 23 | 24 | from django.db.utils import DatabaseError 25 | from django.utils import timezone 26 | from django.utils.functional import Promise 27 | 28 | if django.VERSION < (1, 5): 29 | from django.utils.encoding import (smart_unicode as smart_text, 30 | smart_str as smart_bytes) 31 | else: 32 | from django.utils.encoding import smart_text, smart_bytes 33 | 34 | if django.VERSION < (1, 5): 35 | from django.utils.safestring import (SafeString as SafeBytes, 36 | SafeUnicode as SafeText, 37 | EscapeString as EscapeBytes, 38 | EscapeUnicode as EscapeText) 39 | else: 40 | from django.utils.safestring import SafeBytes, SafeText, EscapeBytes, EscapeText 41 | 42 | from .creation import NonrelDatabaseCreation 43 | 44 | 45 | class NonrelDatabaseFeatures(BaseDatabaseFeatures): 46 | # Most NoSQL databases don't have true transaction support. 47 | supports_transactions = False 48 | 49 | # NoSQL databases usually return a key after saving a new object. 50 | can_return_id_from_insert = True 51 | 52 | # TODO: Doesn't seem necessary in general, move to back-ends. 53 | # Mongo: see PyMongo's FAQ; GAE: see: http://timezones.appspot.com/. 54 | supports_date_lookup_using_string = False 55 | supports_timezones = False 56 | 57 | # Features that are commonly not available on nonrel databases. 58 | supports_joins = False 59 | supports_select_related = False 60 | supports_deleting_related_objects = False 61 | 62 | # Having to decide whether to use an INSERT or an UPDATE query is 63 | # specific to SQL-based databases. 64 | distinguishes_insert_from_update = False 65 | 66 | # Can primary_key be used on any field? Without encoding usually 67 | # only a limited set of types is acceptable for keys. This is a set 68 | # of all field kinds (internal_types) for which the primary_key 69 | # argument may be used. 70 | # TODO: Use during model validation. 71 | # TODO: Move to core and use to skip unsuitable Django tests. 72 | supports_primary_key_on = set(NonrelDatabaseCreation.data_types.keys()) - \ 73 | set(('ForeignKey', 'OneToOneField', 'ManyToManyField', 'RawField', 74 | 'AbstractIterableField', 'ListField', 'SetField', 'DictField', 75 | 'EmbeddedModelField', 'BlobField')) 76 | 77 | # Django 1.4 compatibility 78 | def _supports_transactions(self): 79 | return False 80 | 81 | 82 | class NonrelDatabaseOperations(BaseDatabaseOperations): 83 | """ 84 | Override all database conversions normally done by fields (through 85 | `get_db_prep_value/save/lookup`) to make it possible to pass Python 86 | values directly to the database layer. On the other hand, provide a 87 | framework for making type-based conversions -- drivers of NoSQL 88 | database either can work with Python objects directly, sometimes 89 | representing one type using a another or expect everything encoded 90 | in some specific manner. 91 | 92 | Django normally handles conversions for the database by providing 93 | `BaseDatabaseOperations.value_to_db_*` / `convert_values` methods, 94 | but there are some problems with them: 95 | -- some preparations need to be done for all values or for values 96 | of a particular "kind" (e.g. lazy objects evaluation or casting 97 | strings wrappers to standard types); 98 | -- some conversions need more info about the field or model the 99 | value comes from (e.g. key conversions, embedded deconversion); 100 | -- there are no value_to_db_* methods for some value types (bools); 101 | -- we need to handle collecion fields (list, set, dict): they 102 | need to differentiate between deconverting from database and 103 | deserializing (so single to_python is inconvenient) and need to 104 | do some recursion, so a single `value_for_db` is better than one 105 | method for each field kind. 106 | Don't use these standard methods in nonrel, `value_for/from_db` are 107 | more elastic and keeping all conversions in one place makes the 108 | code easier to analyse. 109 | 110 | Please note, that after changes to type conversions, data saved 111 | using preexisting methods needs to be handled; and also that Django 112 | does not expect any special database driver exceptions, so any such 113 | exceptions should be reraised as django.db.utils.DatabaseError. 114 | 115 | TODO: Consider replacing all `value_to_db_*` and `convert_values` 116 | with just `BaseDatabaseOperations.value_for/from_db` and also 117 | moving there code from `Field.get_db_prep_lookup` (and maybe 118 | `RelatedField.get_db_prep_lookup`). 119 | """ 120 | 121 | def pk_default_value(self): 122 | """ 123 | Returns None, to be interpreted by back-ends as a request to 124 | generate a new key for an "inserted" object. 125 | """ 126 | return None 127 | 128 | def quote_name(self, name): 129 | """ 130 | Does not do any quoting, as it is not needed for most NoSQL 131 | databases. 132 | """ 133 | return name 134 | 135 | def prep_for_like_query(self, value): 136 | """ 137 | Does no conversion, parent string-cast is SQL specific. 138 | """ 139 | return value 140 | 141 | def prep_for_iexact_query(self, value): 142 | """ 143 | Does no conversion, parent string-cast is SQL specific. 144 | """ 145 | return value 146 | 147 | def value_to_db_auto(self, value): 148 | """ 149 | Assuming that the database has its own key type, leaves any 150 | conversions to the back-end. 151 | 152 | This method is added my nonrel to allow various types to be 153 | used for automatic primary keys. `AutoField.get_db_prep_value` 154 | calls it to prepare field's value for the database. 155 | 156 | Note that Django can pass a string representation of the value 157 | instead of the value itself (after receiving it as a query 158 | parameter for example), so you'll likely need to limit 159 | your `AutoFields` in a way that makes `str(value)` reversible. 160 | 161 | TODO: This could become a part of `value_for_db` if it makes 162 | to Django (with a `field_kind` condition). 163 | """ 164 | return value 165 | 166 | def value_to_db_date(self, value): 167 | """ 168 | Unlike with SQL database clients, it's better to assume that 169 | a date can be stored directly. 170 | """ 171 | return value 172 | 173 | def value_to_db_datetime(self, value): 174 | """ 175 | We may pass a datetime object to a database driver without 176 | casting it to a string. 177 | """ 178 | return value 179 | 180 | def value_to_db_time(self, value): 181 | """ 182 | Unlike with SQL database clients, we may assume that a time can 183 | be stored directly. 184 | """ 185 | return value 186 | 187 | def value_to_db_decimal(self, value, max_digits, decimal_places): 188 | """ 189 | We may assume that a decimal can be passed to a NoSQL database 190 | driver directly. 191 | """ 192 | return value 193 | 194 | # Django 1.4 compatibility 195 | def year_lookup_bounds(self, value): 196 | return self.year_lookup_bounds_for_datetime_field(value) 197 | 198 | def year_lookup_bounds_for_date_field(self, value): 199 | """ 200 | Converts year bounds to date bounds as these can likely be 201 | used directly, also adds one to the upper bound as it should be 202 | natural to use one strict inequality for BETWEEN-like filters 203 | for most nonrel back-ends. 204 | """ 205 | first = datetime.date(value, 1, 1) 206 | second = datetime.date(value + 1, 1, 1) 207 | return [first, second] 208 | 209 | def year_lookup_bounds_for_datetime_field(self, value): 210 | """ 211 | Converts year bounds to datetime bounds. 212 | """ 213 | first = datetime.datetime(value, 1, 1, 0, 0, 0, 0) 214 | second = datetime.datetime(value + 1, 1, 1, 0, 0, 0, 0) 215 | if settings.USE_TZ: 216 | tz = timezone.get_current_timezone() 217 | first = timezone.make_aware(first, tz) 218 | second = timezone.make_aware(second, tz) 219 | return [first, second] 220 | 221 | def convert_values(self, value, field): 222 | """ 223 | We may assume that values returned by the database are standard 224 | Python types suitable to be passed to fields. 225 | """ 226 | return value 227 | 228 | def check_aggregate_support(self, aggregate): 229 | """ 230 | Nonrel back-ends are only expected to implement COUNT in 231 | general. 232 | """ 233 | from django.db.models.sql.aggregates import Count 234 | if not isinstance(aggregate, Count): 235 | raise NotImplementedError("This database does not support %r " 236 | "aggregates." % type(aggregate)) 237 | 238 | def value_for_db(self, value, field, lookup=None): 239 | """ 240 | Does type-conversions needed before storing a value in the 241 | the database or using it as a filter parameter. 242 | 243 | This is a convience wrapper that only precomputes field's kind 244 | and a db_type for the field (or the primary key of the related 245 | model for ForeignKeys etc.) and knows that arguments to the 246 | `isnull` lookup (`True` or `False`) should not be converted, 247 | while some other lookups take a list of arguments. 248 | In the end, it calls `_value_for_db` to do the real work; you 249 | should typically extend that method, but only call this one. 250 | 251 | :param value: A value to be passed to the database driver 252 | :param field: A field the value comes from 253 | :param lookup: None if the value is being prepared for storage; 254 | lookup type name, when its going to be used as a 255 | filter argument 256 | """ 257 | field, field_kind, db_type = self._convert_as(field, lookup) 258 | 259 | # Argument to the "isnull" lookup is just a boolean, while some 260 | # other lookups take a list of values. 261 | if lookup == 'isnull': 262 | return value 263 | elif lookup in ('in', 'range', 'year'): 264 | return [self._value_for_db(subvalue, field, 265 | field_kind, db_type, lookup) 266 | for subvalue in value] 267 | else: 268 | return self._value_for_db(value, field, 269 | field_kind, db_type, lookup) 270 | 271 | def value_from_db(self, value, field): 272 | """ 273 | Performs deconversions defined by `_value_from_db`. 274 | 275 | :param value: A value received from the database client 276 | :param field: A field the value is meant for 277 | """ 278 | return self._value_from_db(value, *self._convert_as(field)) 279 | 280 | def _convert_as(self, field, lookup=None): 281 | """ 282 | Computes parameters that should be used for preparing the field 283 | for the database or deconverting a database value for it. 284 | """ 285 | # We need to compute db_type using the original field to allow 286 | # GAE to use different storage for primary and foreign keys. 287 | db_type = self.connection.creation.db_type(field) 288 | 289 | if field.rel is not None: 290 | field = field.rel.get_related_field() 291 | field_kind = field.get_internal_type() 292 | 293 | # Values for standard month / day queries are integers. 294 | if (field_kind in ('DateField', 'DateTimeField') and 295 | lookup in ('month', 'day')): 296 | db_type = 'integer' 297 | 298 | return field, field_kind, db_type 299 | 300 | def _value_for_db(self, value, field, field_kind, db_type, lookup): 301 | """ 302 | Converts a standard Python value to a type that can be stored 303 | or processed by the database driver. 304 | 305 | This implementation only converts elements of iterables passed 306 | by collection fields, evaluates Django's lazy objects and 307 | marked strings and handles embedded models. 308 | Currently, we assume that dict keys and column, model, module 309 | names (strings) of embedded models require no conversion. 310 | 311 | We need to know the field for two reasons: 312 | -- to allow back-ends having separate key spaces for different 313 | tables to create keys refering to the right table (which can 314 | be the field model's table or the table of the model of the 315 | instance a ForeignKey or other relation field points to). 316 | -- to know the field of values passed by typed collection 317 | fields and to use the proper fields when deconverting values 318 | stored for typed embedding field. 319 | Avoid using the field in any other way than by inspecting its 320 | properties, it may not hold any value or hold a value other 321 | than the one you're asked to convert. 322 | 323 | You may want to call this method before doing other back-end 324 | specific conversions. 325 | 326 | :param value: A value to be passed to the database driver 327 | :param field: A field having the same properties as the field 328 | the value comes from; instead of related fields 329 | you'll get the related model primary key, as the 330 | value usually needs to be converted using its 331 | properties 332 | :param field_kind: Equal to field.get_internal_type() 333 | :param db_type: Same as creation.db_type(field) 334 | :param lookup: None if the value is being prepared for storage; 335 | lookup type name, when its going to be used as a 336 | filter argument 337 | """ 338 | 339 | # Back-ends may want to store empty lists or dicts as None. 340 | if value is None: 341 | return None 342 | 343 | # Force evaluation of lazy objects (e.g. lazy translation 344 | # strings). 345 | # Some back-ends pass values directly to the database driver, 346 | # which may fail if it relies on type inspection and gets a 347 | # functional proxy. 348 | # This code relies on unicode cast in django.utils.functional 349 | # just evaluating the wrapped function and doing nothing more. 350 | # TODO: This has been partially fixed in vanilla with: 351 | # https://code.djangoproject.com/changeset/17698, however 352 | # still fails for proxies in lookups; reconsider in 1.4. 353 | # Also research cases of database operations not done 354 | # through the sql.Query. 355 | if isinstance(value, Promise): 356 | value = smart_text(value) 357 | 358 | # Django wraps strings marked as safe or needed escaping, 359 | # convert them to just strings for type-inspecting back-ends. 360 | if isinstance(value, (SafeBytes, EscapeBytes)): 361 | value = smart_bytes(value) 362 | elif isinstance(value, (SafeText, EscapeText)): 363 | value = smart_text(value) 364 | 365 | # Convert elements of collection fields. 366 | if field_kind in ('ListField', 'SetField', 'DictField',): 367 | value = self._value_for_db_collection(value, field, 368 | field_kind, db_type, lookup) 369 | 370 | # Store model instance fields' values. 371 | elif field_kind == 'EmbeddedModelField': 372 | value = self._value_for_db_model(value, field, 373 | field_kind, db_type, lookup) 374 | 375 | return value 376 | 377 | def _value_from_db(self, value, field, field_kind, db_type): 378 | """ 379 | Converts a database type to a type acceptable by the field. 380 | 381 | If you encoded a value for storage in the database, reverse the 382 | encoding here. This implementation only recursively deconverts 383 | elements of collection fields and handles embedded models. 384 | 385 | You may want to call this method after any back-end specific 386 | deconversions. 387 | 388 | :param value: A value to be passed to the database driver 389 | :param field: A field having the same properties as the field 390 | the value comes from 391 | :param field_kind: Equal to field.get_internal_type() 392 | :param db_type: Same as creation.db_type(field) 393 | 394 | Note: lookup values never get deconverted. 395 | """ 396 | 397 | # We did not convert Nones. 398 | if value is None: 399 | return None 400 | 401 | # Deconvert items or values of a collection field. 402 | if field_kind in ('ListField', 'SetField', 'DictField',): 403 | value = self._value_from_db_collection(value, field, 404 | field_kind, db_type) 405 | 406 | # Reinstatiate a serialized model. 407 | elif field_kind == 'EmbeddedModelField': 408 | value = self._value_from_db_model(value, field, 409 | field_kind, db_type) 410 | 411 | return value 412 | 413 | def _value_for_db_collection(self, value, field, field_kind, db_type, 414 | lookup): 415 | """ 416 | Recursively converts values from AbstractIterableFields. 417 | 418 | Note that collection lookup values are plain values rather than 419 | lists, sets or dicts, but they still should be converted as a 420 | collection item (assuming all items or values are converted in 421 | the same way). 422 | 423 | We base the conversion on field class / kind and assume some 424 | knowledge about field internals (e.g. that the field has an 425 | "item_field" property that gives the right subfield for any of 426 | its values), to avoid adding a framework for determination of 427 | parameters for items' conversions; we do the conversion here 428 | rather than inside get_db_prep_save/lookup for symmetry with 429 | deconversion (which can't be in to_python because the method is 430 | also used for data not coming from the database). 431 | 432 | Returns a list, set, dict, string or bytes according to the 433 | db_type given. 434 | If the "list" db_type used for DictField, a list with keys and 435 | values interleaved will be returned (list of pairs is not good, 436 | because lists / tuples may need conversion themselves; the list 437 | may still be nested for dicts containing collections). 438 | The "string" and "bytes" db_types use serialization with pickle 439 | protocol 0 or 2 respectively. 440 | If an unknown db_type is specified, returns a generator 441 | yielding converted elements / pairs with converted values. 442 | """ 443 | subfield, subkind, db_subtype = self._convert_as(field.item_field, 444 | lookup) 445 | 446 | # Do convert filter parameters. 447 | if lookup: 448 | # Special case where we are looking for an empty list 449 | if lookup == 'exact' and db_type == 'list' and value == u'[]': 450 | return [] 451 | value = self._value_for_db(value, subfield, 452 | subkind, db_subtype, lookup) 453 | 454 | # Convert list/set items or dict values. 455 | else: 456 | if field_kind == 'DictField': 457 | 458 | # Generator yielding pairs with converted values. 459 | value = ( 460 | (key, self._value_for_db(subvalue, subfield, 461 | subkind, db_subtype, lookup)) 462 | for key, subvalue in value.iteritems()) 463 | 464 | # Return just a dict, a once-flattened list; 465 | if db_type == 'dict': 466 | return dict(value) 467 | elif db_type == 'list': 468 | return list(item for pair in value for item in pair) 469 | 470 | else: 471 | 472 | # Generator producing converted items. 473 | value = ( 474 | self._value_for_db(subvalue, subfield, 475 | subkind, db_subtype, lookup) 476 | for subvalue in value) 477 | 478 | # "list" may be used for SetField. 479 | if db_type in 'list': 480 | return list(value) 481 | elif db_type == 'set': 482 | # assert field_kind != 'ListField' 483 | return set(value) 484 | 485 | # Pickled formats may be used for all collection fields, 486 | # the fields "natural" type is serialized (something 487 | # concrete is needed, pickle can't handle generators :-) 488 | if db_type == 'bytes': 489 | return pickle.dumps(field._type(value), protocol=2) 490 | elif db_type == 'string': 491 | return pickle.dumps(field._type(value)) 492 | 493 | # If nothing matched, pass the generator to the back-end. 494 | return value 495 | 496 | def _value_from_db_collection(self, value, field, field_kind, db_type): 497 | """ 498 | Recursively deconverts values for AbstractIterableFields. 499 | 500 | Assumes that all values in a collection can be deconverted 501 | using a single field (Field.item_field, possibly a RawField). 502 | 503 | Returns a value in a format proper for the field kind (the 504 | value will normally not go through to_python). 505 | """ 506 | subfield, subkind, db_subtype = self._convert_as(field.item_field) 507 | 508 | # Unpickle (a dict) if a serialized storage is used. 509 | if db_type == 'bytes' or db_type == 'string': 510 | value = pickle.loads(value) 511 | 512 | if field_kind == 'DictField': 513 | 514 | # Generator yielding pairs with deconverted values, the 515 | # "list" db_type stores keys and values interleaved. 516 | if db_type == 'list': 517 | value = zip(value[::2], value[1::2]) 518 | else: 519 | value = value.iteritems() 520 | 521 | # DictField needs to hold a dict. 522 | return dict( 523 | (key, self._value_from_db(subvalue, subfield, 524 | subkind, db_subtype)) 525 | for key, subvalue in value) 526 | else: 527 | 528 | # Generator yielding deconverted items. 529 | value = ( 530 | self._value_from_db(subvalue, subfield, 531 | subkind, db_subtype) 532 | for subvalue in value) 533 | 534 | # The value will be available from the field without any 535 | # further processing and it has to have the right type. 536 | if field_kind == 'ListField': 537 | return list(value) 538 | elif field_kind == 'SetField': 539 | return set(value) 540 | 541 | # A new field kind? Maybe it can take a generator. 542 | return value 543 | 544 | def _value_for_db_model(self, value, field, field_kind, db_type, lookup): 545 | """ 546 | Converts a field => value mapping received from an 547 | EmbeddedModelField the format chosen for the field storage. 548 | 549 | The embedded instance fields' values are also converted / 550 | deconverted using value_for/from_db, so any back-end 551 | conversions will be applied. 552 | 553 | Returns (field.column, value) pairs, possibly augmented with 554 | model info (to be able to deconvert the embedded instance for 555 | untyped fields) encoded according to the db_type chosen. 556 | If "dict" db_type is given a Python dict is returned. 557 | If "list db_type is chosen a list with columns and values 558 | interleaved will be returned. Note that just a single level of 559 | the list is flattened, so it still may be nested -- when the 560 | embedded instance holds other embedded models or collections). 561 | Using "bytes" or "string" pickles the mapping using pickle 562 | protocol 0 or 2 respectively. 563 | If an unknown db_type is used a generator yielding (column, 564 | value) pairs with values converted will be returned. 565 | 566 | TODO: How should EmbeddedModelField lookups work? 567 | """ 568 | if lookup: 569 | # raise NotImplementedError("Needs specification.") 570 | return value 571 | 572 | # Convert using proper instance field's info, change keys from 573 | # fields to columns. 574 | # TODO/XXX: Arguments order due to Python 2.5 compatibility. 575 | value = ( 576 | (subfield.column, self._value_for_db( 577 | subvalue, lookup=lookup, *self._convert_as(subfield, lookup))) 578 | for subfield, subvalue in value.iteritems()) 579 | 580 | # Cast to a dict, interleave columns with values on a list, 581 | # serialize, or return a generator. 582 | if db_type == 'dict': 583 | value = dict(value) 584 | elif db_type == 'list': 585 | value = list(item for pair in value for item in pair) 586 | elif db_type == 'bytes': 587 | value = pickle.dumps(dict(value), protocol=2) 588 | elif db_type == 'string': 589 | value = pickle.dumps(dict(value)) 590 | 591 | return value 592 | 593 | def _value_from_db_model(self, value, field, field_kind, db_type): 594 | """ 595 | Deconverts values stored for EmbeddedModelFields. 596 | 597 | Embedded instances are stored as a (column, value) pairs in a 598 | dict, a single-flattened list or a serialized dict. 599 | 600 | Returns a tuple with model class and field.attname => value 601 | mapping. 602 | """ 603 | 604 | # Separate keys from values and create a dict or unpickle one. 605 | if db_type == 'list': 606 | value = dict(zip(value[::2], value[1::2])) 607 | elif db_type == 'bytes' or db_type == 'string': 608 | value = pickle.loads(value) 609 | 610 | # Let untyped fields determine the embedded instance's model. 611 | embedded_model = field.stored_model(value) 612 | 613 | # Deconvert fields' values and prepare a dict that can be used 614 | # to initialize a model (by changing keys from columns to 615 | # attribute names). 616 | return embedded_model, dict( 617 | (subfield.attname, self._value_from_db( 618 | value[subfield.column], *self._convert_as(subfield))) 619 | for subfield in embedded_model._meta.fields 620 | if subfield.column in value) 621 | 622 | def _value_for_db_key(self, value, field_kind): 623 | """ 624 | Converts value to be used as a key to an acceptable type. 625 | On default we do no encoding, only allowing key values directly 626 | acceptable by the database for its key type (if any). 627 | 628 | The conversion has to be reversible given the field type, 629 | encoding should preserve comparisons. 630 | 631 | Use this to expand the set of fields that can be used as 632 | primary keys, return value suitable for a key rather than 633 | a key itself. 634 | """ 635 | raise DatabaseError( 636 | "%s may not be used as primary key field." % field_kind) 637 | 638 | def _value_from_db_key(self, value, field_kind): 639 | """ 640 | Decodes a value previously encoded for a key. 641 | """ 642 | return value 643 | 644 | 645 | class NonrelDatabaseClient(BaseDatabaseClient): 646 | pass 647 | 648 | 649 | class NonrelDatabaseValidation(BaseDatabaseValidation): 650 | pass 651 | 652 | 653 | class NonrelDatabaseIntrospection(BaseDatabaseIntrospection): 654 | 655 | def table_names(self, cursor=None): 656 | """ 657 | Returns a list of names of all tables that exist in the 658 | database. 659 | """ 660 | return self.django_table_names() 661 | 662 | 663 | class FakeCursor(object): 664 | 665 | def __getattribute__(self, name): 666 | raise Database.NotSupportedError("Cursors are not supported.") 667 | 668 | def __setattr__(self, name, value): 669 | raise Database.NotSupportedError("Cursors are not supported.") 670 | 671 | 672 | class FakeConnection(object): 673 | 674 | def commit(self): 675 | pass 676 | 677 | def rollback(self): 678 | pass 679 | 680 | def cursor(self): 681 | return FakeCursor() 682 | 683 | def close(self): 684 | pass 685 | 686 | 687 | class Database(object): 688 | class Error(Exception): 689 | pass 690 | 691 | class InterfaceError(Error): 692 | pass 693 | 694 | class DatabaseError(Error): 695 | pass 696 | 697 | class DataError(DatabaseError): 698 | pass 699 | 700 | class OperationalError(DatabaseError): 701 | pass 702 | 703 | class IntegrityError(DatabaseError): 704 | pass 705 | 706 | class InternalError(DatabaseError): 707 | pass 708 | 709 | class ProgrammingError(DatabaseError): 710 | pass 711 | 712 | class NotSupportedError(DatabaseError): 713 | pass 714 | 715 | 716 | class NonrelDatabaseWrapper(BaseDatabaseWrapper): 717 | 718 | Database = Database 719 | 720 | # These fake operators are required for SQLQuery.as_sql() support. 721 | operators = { 722 | 'exact': '= %s', 723 | 'iexact': '= UPPER(%s)', 724 | 'contains': 'LIKE %s', 725 | 'icontains': 'LIKE UPPER(%s)', 726 | 'regex': '~ %s', 727 | 'iregex': '~* %s', 728 | 'gt': '> %s', 729 | 'gte': '>= %s', 730 | 'lt': '< %s', 731 | 'lte': '<= %s', 732 | 'startswith': 'LIKE %s', 733 | 'endswith': 'LIKE %s', 734 | 'istartswith': 'LIKE UPPER(%s)', 735 | 'iendswith': 'LIKE UPPER(%s)', 736 | } 737 | 738 | def get_connection_params(self): 739 | return {} 740 | 741 | def get_new_connection(self, conn_params): 742 | return FakeConnection() 743 | 744 | def init_connection_state(self): 745 | pass 746 | 747 | def _set_autocommit(self, autocommit): 748 | pass 749 | 750 | def _cursor(self): 751 | return FakeCursor() 752 | -------------------------------------------------------------------------------- /djangotoolbox/tests.py: -------------------------------------------------------------------------------- 1 | from __future__ import with_statement 2 | from decimal import Decimal, InvalidOperation 3 | import time 4 | 5 | from django.core import serializers 6 | from django.db import models 7 | from django.db.models import Q 8 | from django.db.models.signals import post_save 9 | from django.db.utils import DatabaseError 10 | from django.dispatch.dispatcher import receiver 11 | from django.test import TestCase 12 | from django.utils.unittest import expectedFailure, skip 13 | 14 | from .fields import ListField, SetField, DictField, EmbeddedModelField 15 | 16 | 17 | def count_calls(func): 18 | 19 | def wrapper(*args, **kwargs): 20 | wrapper.calls += 1 21 | return func(*args, **kwargs) 22 | wrapper.calls = 0 23 | 24 | return wrapper 25 | 26 | 27 | class Target(models.Model): 28 | index = models.IntegerField() 29 | 30 | 31 | class Source(models.Model): 32 | target = models.ForeignKey(Target) 33 | index = models.IntegerField() 34 | 35 | 36 | class DecimalModel(models.Model): 37 | decimal = models.DecimalField(max_digits=9, decimal_places=2) 38 | 39 | 40 | class DecimalKey(models.Model): 41 | decimal = models.DecimalField(max_digits=9, decimal_places=2, primary_key=True) 42 | 43 | 44 | class DecimalParent(models.Model): 45 | child = models.ForeignKey(DecimalKey) 46 | 47 | 48 | class DecimalsList(models.Model): 49 | decimals = ListField(models.ForeignKey(DecimalKey)) 50 | 51 | 52 | class ListModel(models.Model): 53 | integer = models.IntegerField(primary_key=True) 54 | floating_point = models.FloatField() 55 | names = ListField(models.CharField) 56 | names_with_default = ListField(models.CharField(max_length=500), 57 | default=[]) 58 | names_nullable = ListField(models.CharField(max_length=500), null=True) 59 | 60 | 61 | class OrderedListModel(models.Model): 62 | ordered_ints = ListField(models.IntegerField(max_length=500), default=[], 63 | ordering=count_calls(lambda x: x), null=True) 64 | ordered_nullable = ListField(ordering=lambda x: x, null=True) 65 | 66 | 67 | class SetModel(models.Model): 68 | setfield = SetField(models.IntegerField()) 69 | 70 | 71 | class DictModel(models.Model): 72 | dictfield = DictField(models.IntegerField) 73 | dictfield_nullable = DictField(null=True) 74 | auto_now = DictField(models.DateTimeField(auto_now=True)) 75 | 76 | 77 | class EmbeddedModelFieldModel(models.Model): 78 | simple = EmbeddedModelField('EmbeddedModel', null=True) 79 | simple_untyped = EmbeddedModelField(null=True) 80 | decimal_parent = EmbeddedModelField(DecimalParent, null=True) 81 | typed_list = ListField(EmbeddedModelField('SetModel')) 82 | typed_list2 = ListField(EmbeddedModelField('EmbeddedModel')) 83 | untyped_list = ListField(EmbeddedModelField()) 84 | untyped_dict = DictField(EmbeddedModelField()) 85 | ordered_list = ListField(EmbeddedModelField(), 86 | ordering=lambda obj: obj.index) 87 | 88 | 89 | class EmbeddedModel(models.Model): 90 | some_relation = models.ForeignKey(DictModel, null=True) 91 | someint = models.IntegerField(db_column='custom') 92 | auto_now = models.DateTimeField(auto_now=True) 93 | auto_now_add = models.DateTimeField(auto_now_add=True) 94 | 95 | 96 | class IterableFieldsTest(TestCase): 97 | floats = [5.3, 2.6, 9.1, 1.58] 98 | names = [u'Kakashi', u'Naruto', u'Sasuke', u'Sakura'] 99 | unordered_ints = [4, 2, 6, 1] 100 | 101 | def setUp(self): 102 | for i, float in zip(range(1, 5), IterableFieldsTest.floats): 103 | ListModel(integer=i, floating_point=float, 104 | names=IterableFieldsTest.names[:i]).save() 105 | 106 | def test_startswith(self): 107 | self.assertEquals( 108 | dict([(entity.pk, entity.names) for entity in 109 | ListModel.objects.filter(names__startswith='Sa')]), 110 | dict([(3, ['Kakashi', 'Naruto', 'Sasuke']), 111 | (4, ['Kakashi', 'Naruto', 'Sasuke', 'Sakura']), ])) 112 | 113 | def test_options(self): 114 | self.assertEqual([entity.names_with_default for entity in 115 | ListModel.objects.filter(names__startswith='Sa')], 116 | [[], []]) 117 | 118 | self.assertEqual([entity.names_nullable for entity in 119 | ListModel.objects.filter(names__startswith='Sa')], 120 | [None, None]) 121 | 122 | def test_default_value(self): 123 | # Make sure default value is copied. 124 | ListModel().names_with_default.append(2) 125 | self.assertEqual(ListModel().names_with_default, []) 126 | 127 | def test_ordering(self): 128 | f = OrderedListModel._meta.fields[1] 129 | f.ordering.calls = 0 130 | 131 | # Ensure no ordering happens on assignment. 132 | obj = OrderedListModel() 133 | obj.ordered_ints = self.unordered_ints 134 | self.assertEqual(f.ordering.calls, 0) 135 | 136 | obj.save() 137 | self.assertEqual(OrderedListModel.objects.get().ordered_ints, 138 | sorted(self.unordered_ints)) 139 | # Ordering should happen only once, i.e. the order function may 140 | # be called N times at most (N being the number of items in the 141 | # list). 142 | self.assertLessEqual(f.ordering.calls, len(self.unordered_ints)) 143 | 144 | def test_gt(self): 145 | self.assertEquals( 146 | dict([(entity.pk, entity.names) for entity in 147 | ListModel.objects.filter(names__gt='Kakashi')]), 148 | dict([(2, [u'Kakashi', u'Naruto']), 149 | (3, [u'Kakashi', u'Naruto', u'Sasuke']), 150 | (4, [u'Kakashi', u'Naruto', u'Sasuke', u'Sakura']), ])) 151 | 152 | def test_lt(self): 153 | self.assertEquals( 154 | dict([(entity.pk, entity.names) for entity in 155 | ListModel.objects.filter(names__lt='Naruto')]), 156 | dict([(1, [u'Kakashi']), 157 | (2, [u'Kakashi', u'Naruto']), 158 | (3, [u'Kakashi', u'Naruto', u'Sasuke']), 159 | (4, [u'Kakashi', u'Naruto', u'Sasuke', u'Sakura']), ])) 160 | 161 | def test_gte(self): 162 | self.assertEquals( 163 | dict([(entity.pk, entity.names) for entity in 164 | ListModel.objects.filter(names__gte='Sakura')]), 165 | dict([(3, [u'Kakashi', u'Naruto', u'Sasuke']), 166 | (4, [u'Kakashi', u'Naruto', u'Sasuke', u'Sakura']), ])) 167 | 168 | def test_lte(self): 169 | self.assertEquals( 170 | dict([(entity.pk, entity.names) for entity in 171 | ListModel.objects.filter(names__lte='Kakashi')]), 172 | dict([(1, [u'Kakashi']), 173 | (2, [u'Kakashi', u'Naruto']), 174 | (3, [u'Kakashi', u'Naruto', u'Sasuke']), 175 | (4, [u'Kakashi', u'Naruto', u'Sasuke', u'Sakura']), ])) 176 | 177 | def test_equals(self): 178 | self.assertEquals([entity.names for entity in 179 | ListModel.objects.filter(names='Sakura')], 180 | [[u'Kakashi', u'Naruto', u'Sasuke', u'Sakura']]) 181 | 182 | # Test with additonal pk filter (for DBs that have special pk 183 | # queries). 184 | query = ListModel.objects.filter(names='Sakura') 185 | self.assertEquals(query.get(pk=query[0].pk).names, 186 | [u'Kakashi', u'Naruto', u'Sasuke', u'Sakura']) 187 | 188 | def test_is_null(self): 189 | self.assertEquals(ListModel.objects.filter( 190 | names__isnull=True).count(), 0) 191 | 192 | def test_exclude(self): 193 | self.assertEquals( 194 | dict([(entity.pk, entity.names) for entity in 195 | ListModel.objects.all().exclude(names__lt='Sakura')]), 196 | dict([(3, [u'Kakashi', u'Naruto', u'Sasuke']), 197 | (4, [u'Kakashi', u'Naruto', u'Sasuke', u'Sakura']), ])) 198 | 199 | def test_chained_filter(self): 200 | self.assertEquals( 201 | [entity.names for entity in ListModel.objects 202 | .filter(names='Sasuke').filter(names='Sakura')], 203 | [['Kakashi', 'Naruto', 'Sasuke', 'Sakura'], ]) 204 | 205 | self.assertEquals( 206 | [entity.names for entity in ListModel.objects 207 | .filter(names__startswith='Sa').filter(names='Sakura')], 208 | [['Kakashi', 'Naruto', 'Sasuke', 'Sakura']]) 209 | 210 | # Test across multiple columns. On app engine only one filter 211 | # is allowed to be an inequality filter. 212 | self.assertEquals( 213 | [entity.names for entity in ListModel.objects 214 | .filter(floating_point=9.1).filter(names__startswith='Sa')], 215 | [['Kakashi', 'Naruto', 'Sasuke'], ]) 216 | 217 | def test_setfield(self): 218 | setdata = [1, 2, 3, 2, 1] 219 | # At the same time test value conversion. 220 | SetModel(setfield=map(str, setdata)).save() 221 | item = SetModel.objects.filter(setfield=3)[0] 222 | self.assertEqual(item.setfield, set(setdata)) 223 | # This shouldn't raise an error because the default value is 224 | # an empty list. 225 | SetModel().save() 226 | 227 | def test_dictfield(self): 228 | DictModel(dictfield=dict(a=1, b='55', foo=3.14), 229 | auto_now={'a': None}).save() 230 | item = DictModel.objects.get() 231 | self.assertEqual(item.dictfield, {u'a': 1, u'b': 55, u'foo': 3}) 232 | 233 | dt = item.auto_now['a'] 234 | self.assertNotEqual(dt, None) 235 | item.save() 236 | time.sleep(0.5) # Sleep to avoid false positive failure on the assertion below 237 | self.assertGreater(DictModel.objects.get().auto_now['a'], dt) 238 | item.delete() 239 | 240 | # Saving empty dicts shouldn't throw errors. 241 | DictModel().save() 242 | # Regression tests for djangoappengine issue #39. 243 | DictModel.add_to_class('new_dict_field', DictField()) 244 | DictModel.objects.get() 245 | 246 | @skip("GAE specific?") 247 | def test_Q_objects(self): 248 | self.assertEquals( 249 | [entity.names for entity in ListModel.objects 250 | .exclude(Q(names__lt='Sakura') | Q(names__gte='Sasuke'))], 251 | [['Kakashi', 'Naruto', 'Sasuke', 'Sakura']]) 252 | 253 | def test_list_with_foreignkeys(self): 254 | 255 | class ReferenceList(models.Model): 256 | keys = ListField(models.ForeignKey('Model')) 257 | 258 | class Model(models.Model): 259 | pass 260 | 261 | model1 = Model.objects.create() 262 | model2 = Model.objects.create() 263 | ReferenceList.objects.create(keys=[model1.pk, model2.pk]) 264 | 265 | self.assertEqual(ReferenceList.objects.get().keys[0], model1.pk) 266 | self.assertEqual(ReferenceList.objects.filter(keys=model1.pk).count(), 1) 267 | 268 | def test_list_with_foreign_conversion(self): 269 | decimal = DecimalKey.objects.create(decimal=Decimal('1.5')) 270 | DecimalsList.objects.create(decimals=[decimal.pk]) 271 | 272 | @expectedFailure 273 | def test_nested_list(self): 274 | """ 275 | Some back-ends expect lists to be strongly typed or not contain 276 | other lists (e.g. GAE), this limits how the ListField can be 277 | used (unless the back-end were to serialize all lists). 278 | """ 279 | 280 | class UntypedListModel(models.Model): 281 | untyped_list = ListField() 282 | 283 | UntypedListModel.objects.create(untyped_list=[1, [2, 3]]) 284 | 285 | 286 | class Child(models.Model): 287 | pass 288 | 289 | 290 | class Parent(models.Model): 291 | id = models.IntegerField(primary_key=True) 292 | integer_list = ListField(models.IntegerField) 293 | integer_dict = DictField(models.IntegerField) 294 | embedded_list = ListField(EmbeddedModelField(Child)) 295 | embedded_dict = DictField(EmbeddedModelField(Child)) 296 | 297 | 298 | class EmbeddedModelFieldTest(TestCase): 299 | 300 | def assertEqualDatetime(self, d1, d2): 301 | """Compares d1 and d2, ignoring microseconds.""" 302 | self.assertEqual(d1.replace(microsecond=0), 303 | d2.replace(microsecond=0)) 304 | 305 | def assertNotEqualDatetime(self, d1, d2): 306 | self.assertNotEqual(d1.replace(microsecond=0), 307 | d2.replace(microsecond=0)) 308 | 309 | def _simple_instance(self): 310 | EmbeddedModelFieldModel.objects.create( 311 | simple=EmbeddedModel(someint='5')) 312 | return EmbeddedModelFieldModel.objects.get() 313 | 314 | def test_simple(self): 315 | instance = self._simple_instance() 316 | self.assertIsInstance(instance.simple, EmbeddedModel) 317 | # Make sure get_prep_value is called. 318 | self.assertEqual(instance.simple.someint, 5) 319 | # Primary keys should not be populated... 320 | self.assertEqual(instance.simple.id, None) 321 | # ... unless set explicitly. 322 | instance.simple.id = instance.id 323 | instance.save() 324 | instance = EmbeddedModelFieldModel.objects.get() 325 | self.assertEqual(instance.simple.id, instance.id) 326 | 327 | def _test_pre_save(self, instance, get_field): 328 | # Make sure field.pre_save is called for embedded objects. 329 | from time import sleep 330 | instance.save() 331 | auto_now = get_field(instance).auto_now 332 | auto_now_add = get_field(instance).auto_now_add 333 | self.assertNotEqual(auto_now, None) 334 | self.assertNotEqual(auto_now_add, None) 335 | 336 | sleep(1) # FIXME 337 | instance.save() 338 | self.assertNotEqualDatetime(get_field(instance).auto_now, 339 | get_field(instance).auto_now_add) 340 | 341 | instance = EmbeddedModelFieldModel.objects.get() 342 | instance.save() 343 | # auto_now_add shouldn't have changed now, but auto_now should. 344 | self.assertEqualDatetime(get_field(instance).auto_now_add, 345 | auto_now_add) 346 | self.assertGreater(get_field(instance).auto_now, auto_now) 347 | 348 | def test_pre_save(self): 349 | obj = EmbeddedModelFieldModel(simple=EmbeddedModel()) 350 | self._test_pre_save(obj, lambda instance: instance.simple) 351 | 352 | def test_pre_save_untyped(self): 353 | obj = EmbeddedModelFieldModel(simple_untyped=EmbeddedModel()) 354 | self._test_pre_save(obj, lambda instance: instance.simple_untyped) 355 | 356 | def test_pre_save_in_list(self): 357 | obj = EmbeddedModelFieldModel(untyped_list=[EmbeddedModel()]) 358 | self._test_pre_save(obj, lambda instance: instance.untyped_list[0]) 359 | 360 | def test_pre_save_in_dict(self): 361 | obj = EmbeddedModelFieldModel(untyped_dict={'a': EmbeddedModel()}) 362 | self._test_pre_save(obj, lambda instance: instance.untyped_dict['a']) 363 | 364 | def test_pre_save_list(self): 365 | # Also make sure auto_now{,add} works for embedded object *lists*. 366 | EmbeddedModelFieldModel.objects.create(typed_list2=[EmbeddedModel()]) 367 | instance = EmbeddedModelFieldModel.objects.get() 368 | 369 | auto_now = instance.typed_list2[0].auto_now 370 | auto_now_add = instance.typed_list2[0].auto_now_add 371 | self.assertNotEqual(auto_now, None) 372 | self.assertNotEqual(auto_now_add, None) 373 | 374 | instance.typed_list2.append(EmbeddedModel()) 375 | instance.save() 376 | instance = EmbeddedModelFieldModel.objects.get() 377 | 378 | self.assertEqualDatetime(instance.typed_list2[0].auto_now_add, 379 | auto_now_add) 380 | self.assertGreater(instance.typed_list2[0].auto_now, auto_now) 381 | self.assertNotEqual(instance.typed_list2[1].auto_now, None) 382 | self.assertNotEqual(instance.typed_list2[1].auto_now_add, None) 383 | 384 | def test_error_messages(self): 385 | for kwargs, expected in ( 386 | ({'simple': 42}, EmbeddedModel), 387 | ({'simple_untyped': 42}, models.Model), 388 | ({'typed_list': [EmbeddedModel()]}, SetModel)): 389 | self.assertRaisesRegexp( 390 | TypeError, "Expected instance of type %r." % expected, 391 | EmbeddedModelFieldModel(**kwargs).save) 392 | 393 | def test_typed_listfield(self): 394 | EmbeddedModelFieldModel.objects.create( 395 | typed_list=[SetModel(setfield=range(3)), 396 | SetModel(setfield=range(9))], 397 | ordered_list=[Target(index=i) for i in xrange(5, 0, -1)]) 398 | obj = EmbeddedModelFieldModel.objects.get() 399 | self.assertIn(5, obj.typed_list[1].setfield) 400 | self.assertEqual([target.index for target in obj.ordered_list], 401 | range(1, 6)) 402 | 403 | def test_untyped_listfield(self): 404 | EmbeddedModelFieldModel.objects.create(untyped_list=[ 405 | EmbeddedModel(someint=7), 406 | OrderedListModel(ordered_ints=range(5, 0, -1)), 407 | SetModel(setfield=[1, 2, 2, 3])]) 408 | instances = EmbeddedModelFieldModel.objects.get().untyped_list 409 | for instance, cls in zip(instances, 410 | [EmbeddedModel, OrderedListModel, SetModel]): 411 | self.assertIsInstance(instance, cls) 412 | self.assertNotEqual(instances[0].auto_now, None) 413 | self.assertEqual(instances[1].ordered_ints, range(1, 6)) 414 | 415 | def test_untyped_dict(self): 416 | EmbeddedModelFieldModel.objects.create(untyped_dict={ 417 | 'a': SetModel(setfield=range(3)), 418 | 'b': DictModel(dictfield={'a': 1, 'b': 2}), 419 | 'c': DictModel(dictfield={}, auto_now={'y': 1})}) 420 | data = EmbeddedModelFieldModel.objects.get().untyped_dict 421 | self.assertIsInstance(data['a'], SetModel) 422 | self.assertNotEqual(data['c'].auto_now['y'], None) 423 | 424 | def test_foreignkey_in_embedded_object(self): 425 | simple = EmbeddedModel(some_relation=DictModel.objects.create()) 426 | obj = EmbeddedModelFieldModel.objects.create(simple=simple) 427 | simple = EmbeddedModelFieldModel.objects.get().simple 428 | self.assertNotIn('some_relation', simple.__dict__) 429 | self.assertIsInstance(simple.__dict__['some_relation_id'], 430 | type(obj.id)) 431 | self.assertIsInstance(simple.some_relation, DictModel) 432 | 433 | def test_embedded_field_with_foreign_conversion(self): 434 | decimal = DecimalKey.objects.create(decimal=Decimal('1.5')) 435 | decimal_parent = DecimalParent.objects.create(child=decimal) 436 | EmbeddedModelFieldModel.objects.create(decimal_parent=decimal_parent) 437 | 438 | def test_update(self): 439 | """ 440 | Test that update can be used on an a subset of objects 441 | containing collections of embedded instances; see issue #13. 442 | Also ensure that updated values are coerced according to 443 | collection field. 444 | """ 445 | child1 = Child.objects.create() 446 | child2 = Child.objects.create() 447 | parent = Parent.objects.create(pk=1, 448 | integer_list=[1], integer_dict={'a': 2}, 449 | embedded_list=[child1], embedded_dict={'a': child2}) 450 | Parent.objects.filter(pk=1).update( 451 | integer_list=['3'], integer_dict={'b': '3'}, 452 | embedded_list=[child2], embedded_dict={'b': child1}) 453 | parent = Parent.objects.get() 454 | self.assertEqual(parent.integer_list, [3]) 455 | self.assertEqual(parent.integer_dict, {'b': 3}) 456 | self.assertEqual(parent.embedded_list, [child2]) 457 | self.assertEqual(parent.embedded_dict, {'b': child1}) 458 | 459 | 460 | class BaseModel(models.Model): 461 | pass 462 | 463 | 464 | class ExtendedModel(BaseModel): 465 | name = models.CharField(max_length=20) 466 | 467 | 468 | class BaseModelProxy(BaseModel): 469 | 470 | class Meta: 471 | proxy = True 472 | 473 | 474 | class ExtendedModelProxy(ExtendedModel): 475 | 476 | class Meta: 477 | proxy = True 478 | 479 | 480 | class ProxyTest(TestCase): 481 | 482 | def test_proxy(self): 483 | list(BaseModelProxy.objects.all()) 484 | 485 | def test_proxy_with_inheritance(self): 486 | self.assertRaises(DatabaseError, 487 | lambda: list(ExtendedModelProxy.objects.all())) 488 | 489 | 490 | class SignalTest(TestCase): 491 | 492 | def test_post_save(self): 493 | created = [] 494 | 495 | @receiver(post_save, sender=SetModel) 496 | def handle(**kwargs): 497 | created.append(kwargs['created']) 498 | 499 | SetModel().save() 500 | self.assertEqual(created, [True]) 501 | SetModel.objects.get().save() 502 | self.assertEqual(created, [True, False]) 503 | qs = SetModel.objects.all() 504 | list(qs)[0].save() 505 | self.assertEqual(created, [True, False, False]) 506 | list(qs)[0].save() 507 | self.assertEqual(created, [True, False, False, False]) 508 | list(qs.select_related())[0].save() 509 | self.assertEqual(created, [True, False, False, False, False]) 510 | 511 | 512 | class SelectRelatedTest(TestCase): 513 | 514 | def test_select_related(self): 515 | target = Target(index=5) 516 | target.save() 517 | Source(target=target, index=8).save() 518 | source = Source.objects.all().select_related()[0] 519 | self.assertEqual(source.target.pk, target.pk) 520 | self.assertEqual(source.target.index, target.index) 521 | source = Source.objects.all().select_related('target')[0] 522 | self.assertEqual(source.target.pk, target.pk) 523 | self.assertEqual(source.target.index, target.index) 524 | 525 | 526 | class DBColumn(models.Model): 527 | a = models.IntegerField(db_column='b') 528 | 529 | 530 | class OrderByTest(TestCase): 531 | 532 | def test_foreign_keys(self): 533 | target1 = Target.objects.create(index=1) 534 | target2 = Target.objects.create(index=2) 535 | source1 = Source.objects.create(target=target1, index=3) 536 | source2 = Source.objects.create(target=target2, index=4) 537 | self.assertEqual(list(Source.objects.all().order_by('target')), 538 | [source1, source2]) 539 | self.assertEqual(list(Source.objects.all().order_by('-target')), 540 | [source2, source1]) 541 | 542 | def test_db_column(self): 543 | model1 = DBColumn.objects.create(a=1) 544 | model2 = DBColumn.objects.create(a=2) 545 | self.assertEqual(list(DBColumn.objects.all().order_by('a')), 546 | [model1, model2]) 547 | self.assertEqual(list(DBColumn.objects.all().order_by('-a')), 548 | [model2, model1]) 549 | 550 | def test_reverse(self): 551 | model1 = DBColumn.objects.create(a=1) 552 | model2 = DBColumn.objects.create(a=2) 553 | self.assertEqual(list(DBColumn.objects.all().order_by('a').reverse()), 554 | [model2, model1]) 555 | self.assertEqual(list(DBColumn.objects.all().order_by('-a').reverse()), 556 | [model1, model2]) 557 | 558 | def test_chain(self): 559 | model1 = Target.objects.create(index=1) 560 | model2 = Target.objects.create(index=2) 561 | self.assertEqual( 562 | list(Target.objects.all().order_by('index').order_by('-index')), 563 | [model2, model1]) 564 | 565 | 566 | class SerializableSetModel(models.Model): 567 | setfield = SetField(models.IntegerField()) 568 | setcharfield = SetField(models.CharField(), null=True) 569 | 570 | 571 | class SerializationTest(TestCase): 572 | """ 573 | JSON doesn't support sets, so they need to be converted to lists 574 | for serialization; see issue #12. 575 | 576 | TODO: Check if the fix works with embedded models / nested sets. 577 | """ 578 | names = ['foo', 'bar', 'baz', 'monkey'] 579 | 580 | def test_json_listfield(self): 581 | for i in range(1, 5): 582 | ListModel(integer=i, floating_point=0, 583 | names=SerializationTest.names[:i]).save() 584 | objects = ListModel.objects.all() 585 | serialized = serializers.serialize('json', objects) 586 | deserialized = serializers.deserialize('json', serialized) 587 | for m in deserialized: 588 | integer = m.object.integer 589 | names = m.object.names 590 | self.assertEqual(names, SerializationTest.names[:integer]) 591 | 592 | def test_json_setfield(self): 593 | for i in range(1, 5): 594 | SerializableSetModel( 595 | setfield=set([i - 1]), 596 | setcharfield=set(SerializationTest.names[:i])).save() 597 | objects = SerializableSetModel.objects.all() 598 | serialized = serializers.serialize('json', objects) 599 | deserialized = serializers.deserialize('json', serialized) 600 | for m in deserialized: 601 | integer = m.object.setfield.pop() 602 | names = m.object.setcharfield 603 | self.assertEqual(names, set(SerializationTest.names[:integer + 1])) 604 | 605 | 606 | class String(models.Model): 607 | s = models.CharField(max_length=20) 608 | 609 | 610 | class LazyObjectsTest(TestCase): 611 | 612 | def test_translation(self): 613 | """ 614 | Using a lazy translation call should work just the same as 615 | a non-lazy one (or a plain string). 616 | """ 617 | from django.utils.translation import ugettext_lazy 618 | 619 | a = String.objects.create(s='a') 620 | b = String.objects.create(s=ugettext_lazy('b')) 621 | 622 | self.assertEqual(String.objects.get(s='a'), a) 623 | self.assertEqual(list(String.objects.filter(s='a')), [a]) 624 | self.assertEqual(list(String.objects.filter(s__lte='a')), [a]) 625 | self.assertEqual(String.objects.get(s=ugettext_lazy('a')), a) 626 | self.assertEqual( 627 | list(String.objects.filter(s__lte=ugettext_lazy('a'))), [a]) 628 | 629 | self.assertEqual(String.objects.get(s='b'), b) 630 | self.assertEqual(list(String.objects.filter(s='b')), [b]) 631 | self.assertEqual(list(String.objects.filter(s__gte='b')), [b]) 632 | self.assertEqual(String.objects.get(s=ugettext_lazy('b')), b) 633 | self.assertEqual( 634 | list(String.objects.filter(s__gte=ugettext_lazy('b'))), [b]) 635 | 636 | def test_marked_strings(self): 637 | """ 638 | Check that strings marked as safe or needing escaping do not 639 | confuse the back-end. 640 | """ 641 | from django.utils.safestring import mark_safe, mark_for_escaping 642 | 643 | a = String.objects.create(s='a') 644 | b = String.objects.create(s=mark_safe('b')) 645 | c = String.objects.create(s=mark_for_escaping('c')) 646 | 647 | self.assertEqual(String.objects.get(s='a'), a) 648 | self.assertEqual(list(String.objects.filter(s__startswith='a')), [a]) 649 | self.assertEqual(String.objects.get(s=mark_safe('a')), a) 650 | self.assertEqual( 651 | list(String.objects.filter(s__startswith=mark_safe('a'))), [a]) 652 | self.assertEqual(String.objects.get(s=mark_for_escaping('a')), a) 653 | self.assertEqual( 654 | list(String.objects.filter(s__startswith=mark_for_escaping('a'))), 655 | [a]) 656 | 657 | self.assertEqual(String.objects.get(s='b'), b) 658 | self.assertEqual(list(String.objects.filter(s__startswith='b')), [b]) 659 | self.assertEqual(String.objects.get(s=mark_safe('b')), b) 660 | self.assertEqual( 661 | list(String.objects.filter(s__startswith=mark_safe('b'))), [b]) 662 | self.assertEqual(String.objects.get(s=mark_for_escaping('b')), b) 663 | self.assertEqual( 664 | list(String.objects.filter(s__startswith=mark_for_escaping('b'))), 665 | [b]) 666 | 667 | self.assertEqual(String.objects.get(s='c'), c) 668 | self.assertEqual(list(String.objects.filter(s__startswith='c')), [c]) 669 | self.assertEqual(String.objects.get(s=mark_safe('c')), c) 670 | self.assertEqual( 671 | list(String.objects.filter(s__startswith=mark_safe('c'))), [c]) 672 | self.assertEqual(String.objects.get(s=mark_for_escaping('c')), c) 673 | self.assertEqual( 674 | list(String.objects.filter(s__startswith=mark_for_escaping('c'))), 675 | [c]) 676 | 677 | 678 | class FeaturesTest(TestCase): 679 | """ 680 | Some things are unlikely to cause problems for SQL back-ends, but 681 | require special handling in nonrel. 682 | """ 683 | 684 | def test_subqueries(self): 685 | """ 686 | Django includes SQL statements as WHERE tree values when 687 | filtering using a QuerySet -- this won't "just work" with 688 | nonrel back-ends. 689 | 690 | TODO: Subqueries handling may require a bit of Django 691 | changing, but should be easy to support. 692 | """ 693 | target = Target.objects.create(index=1) 694 | source = Source.objects.create(index=2, target=target) 695 | targets = Target.objects.all() 696 | with self.assertRaises(DatabaseError): 697 | Source.objects.get(target__in=targets) 698 | self.assertEqual( 699 | Source.objects.get(target__in=list(targets)), 700 | source) 701 | 702 | 703 | class DecimalFieldTest(TestCase): 704 | """ 705 | Some NoSQL databases can't handle Decimals, so respective back-ends 706 | convert them to strings or floats. This can cause some precision 707 | and sorting problems. 708 | """ 709 | 710 | def setUp(self): 711 | for d in (Decimal('12345.6789'), Decimal('5'), Decimal('345.67'), 712 | Decimal('45.6'), Decimal('2345.678'),): 713 | DecimalModel(decimal=d).save() 714 | 715 | def test_filter(self): 716 | d = DecimalModel.objects.get(decimal=Decimal('5.0')) 717 | 718 | self.assertTrue(isinstance(d.decimal, Decimal)) 719 | self.assertEquals(str(d.decimal), '5.00') 720 | 721 | d = DecimalModel.objects.get(decimal=Decimal('45.60')) 722 | self.assertEquals(str(d.decimal), '45.60') 723 | 724 | # Filter argument should be converted to Decimal with 2 decimal 725 | #_places. 726 | d = DecimalModel.objects.get(decimal='0000345.67333333333333333') 727 | self.assertEquals(str(d.decimal), '345.67') 728 | 729 | def test_order(self): 730 | """ 731 | Standard Django decimal-to-string conversion isn't monotonic 732 | (see `django.db.backends.util.format_number`). 733 | """ 734 | rows = DecimalModel.objects.all().order_by('decimal') 735 | values = list(d.decimal for d in rows) 736 | self.assertEquals(values, sorted(values)) 737 | 738 | def test_sign_extend(self): 739 | DecimalModel(decimal=Decimal('-0.0')).save() 740 | 741 | try: 742 | # If we've written a valid string we should be able to 743 | # retrieve the DecimalModel object without error. 744 | DecimalModel.objects.filter(decimal__lt=1)[0] 745 | except InvalidOperation: 746 | self.assertTrue(False) 747 | 748 | 749 | class DeleteModel(models.Model): 750 | key = models.IntegerField(primary_key=True) 751 | deletable = models.BooleanField() 752 | 753 | class BasicDeleteTest(TestCase): 754 | 755 | def setUp(self): 756 | for i in range(1, 10): 757 | DeleteModel(key=i, deletable=i % 2 == 0).save() 758 | 759 | def test_model_delete(self): 760 | d = DeleteModel.objects.get(pk=1) 761 | d.delete() 762 | 763 | with self.assertRaises(DeleteModel.DoesNotExist): 764 | DeleteModel.objects.get(pk=1) 765 | 766 | def test_delete_all(self): 767 | DeleteModel.objects.all().delete() 768 | 769 | self.assertEquals(0, DeleteModel.objects.all().count()) 770 | 771 | def test_delete_filtered(self): 772 | DeleteModel.objects.filter(deletable=True).delete() 773 | 774 | self.assertEquals(5, DeleteModel.objects.all().count()) 775 | 776 | 777 | class M2MDeleteChildModel(models.Model): 778 | key = models.IntegerField(primary_key=True) 779 | 780 | class M2MDeleteModel(models.Model): 781 | key = models.IntegerField(primary_key=True) 782 | deletable = models.BooleanField() 783 | children = models.ManyToManyField(M2MDeleteChildModel, blank=True) 784 | 785 | class ManyToManyDeleteTest(TestCase): 786 | """ 787 | Django-nonrel doesn't support many-to-many, but there may be 788 | models that are used which contain them, even if they're not 789 | accessed. This test ensures they can be deleted. 790 | """ 791 | 792 | def setUp(self): 793 | for i in range(1, 10): 794 | M2MDeleteModel(key=i, deletable=i % 2 == 0).save() 795 | 796 | def test_model_delete(self): 797 | d = M2MDeleteModel.objects.get(pk=1) 798 | d.delete() 799 | 800 | with self.assertRaises(M2MDeleteModel.DoesNotExist): 801 | M2MDeleteModel.objects.get(pk=1) 802 | 803 | @expectedFailure 804 | def test_delete_all(self): 805 | M2MDeleteModel.objects.all().delete() 806 | 807 | self.assertEquals(0, M2MDeleteModel.objects.all().count()) 808 | 809 | @expectedFailure 810 | def test_delete_filtered(self): 811 | M2MDeleteModel.objects.filter(deletable=True).delete() 812 | 813 | self.assertEquals(5, M2MDeleteModel.objects.all().count()) 814 | 815 | 816 | class QuerysetModel(models.Model): 817 | key = models.IntegerField(primary_key=True) 818 | 819 | class QuerysetTest(TestCase): 820 | """ 821 | Django 1.6 changes how 822 | """ 823 | 824 | def setUp(self): 825 | for i in range(10): 826 | QuerysetModel.objects.create(key=i + 1) 827 | 828 | def test_all(self): 829 | self.assertEqual(10, len(QuerysetModel.objects.all())) 830 | 831 | def test_none(self): 832 | self.assertEqual(0, len(QuerysetModel.objects.none())) 833 | --------------------------------------------------------------------------------