├── .gitignore ├── CHANGELOG.rst ├── LICENSE ├── MANIFEST.in ├── README.rst ├── dbindexer ├── __init__.py ├── api.py ├── backends.py ├── base.py ├── compiler.py ├── lookups.py ├── models.py ├── resolver.py └── tests.py └── setup.py /.gitignore: -------------------------------------------------------------------------------- 1 | *.pyc 2 | .DS_Store 3 | -------------------------------------------------------------------------------- /CHANGELOG.rst: -------------------------------------------------------------------------------- 1 | Changelog 2 | ========= 3 | 4 | Version 1.6.1 (Nov 29, 2013) 5 | ------------- 6 | 7 | * Fixed packaging issues 8 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) Waldemar Kornewald, Thomas Wanschik, and all contributors. 2 | All rights reserved. 3 | 4 | Redistribution and use in source and binary forms, with or without modification, 5 | are permitted provided that the following conditions are met: 6 | 7 | 1. Redistributions of source code must retain the above copyright notice, 8 | this list of conditions and the following disclaimer. 9 | 10 | 2. Redistributions in binary form must reproduce the above copyright 11 | notice, this list of conditions and the following disclaimer in the 12 | documentation and/or other materials provided with the distribution. 13 | 14 | 3. Neither the name of All Buttons Pressed nor 15 | the names of its contributors may be used to endorse or promote products 16 | derived from this software without specific prior written permission. 17 | 18 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND 19 | ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 20 | WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 21 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR 22 | ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 23 | (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 24 | LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON 25 | ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 26 | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 27 | SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include LICENSE 2 | include CHANGELOG.rst 3 | include README.rst 4 | -------------------------------------------------------------------------------- /README.rst: -------------------------------------------------------------------------------- 1 | With django-dbindexer_ you can emulate SQL features on NoSQL databases. For example, if your database doesn't support case-insensitive queries (iexact, istartswith, etc.) you can just tell the indexer which models and fields should support these queries and it'll take care of maintaining the required indexes for you. Magically, the previously unsupported queries will just work. Currently, this project is in a very early development stage. The long-term plan is to support JOINs and at least some simple aggregates, possibly even much more. 2 | 3 | Visit the `project site`_ for more information. 4 | 5 | .. _django-dbindexer: http://www.allbuttonspressed.com/projects/django-dbindexer 6 | .. _project site: django-dbindexer_ 7 | -------------------------------------------------------------------------------- /dbindexer/__init__.py: -------------------------------------------------------------------------------- 1 | def autodiscover(): 2 | from autoload import autodiscover as auto_discover 3 | auto_discover('dbindexes') 4 | 5 | def load_indexes(): 6 | from django.conf import settings 7 | from django.utils.importlib import import_module 8 | 9 | for name in getattr(settings, 'DB_INDEX_MODULES', ()): 10 | import_module(name) 11 | -------------------------------------------------------------------------------- /dbindexer/api.py: -------------------------------------------------------------------------------- 1 | from .lookups import LookupDoesNotExist, ExtraFieldLookup 2 | from . import lookups as lookups_module 3 | from .resolver import resolver 4 | import inspect 5 | 6 | # TODO: add possibility to add lookup modules 7 | def create_lookup(lookup_def): 8 | for _, cls in inspect.getmembers(lookups_module): 9 | if inspect.isclass(cls) and issubclass(cls, ExtraFieldLookup) and \ 10 | cls.matches_lookup_def(lookup_def): 11 | return cls() 12 | raise LookupDoesNotExist('No Lookup found for %s .' % lookup_def) 13 | 14 | def register_index(model, mapping): 15 | for field_name, lookups in mapping.items(): 16 | if not isinstance(lookups, (list, tuple)): 17 | lookups = (lookups, ) 18 | 19 | # create indexes and add model and field_name to lookups 20 | # create ExtraFieldLookup instances on the fly if needed 21 | for lookup in lookups: 22 | lookup_def = None 23 | if not isinstance(lookup, ExtraFieldLookup): 24 | lookup_def = lookup 25 | lookup = create_lookup(lookup_def) 26 | lookup.contribute(model, field_name, lookup_def) 27 | resolver.create_index(lookup) 28 | -------------------------------------------------------------------------------- /dbindexer/backends.py: -------------------------------------------------------------------------------- 1 | import django 2 | from django.core.exceptions import ImproperlyConfigured 3 | from django.db import models 4 | from django.db.models.fields import FieldDoesNotExist 5 | from django.utils.tree import Node 6 | 7 | try: 8 | from django.db.models.sql.where import SubqueryConstraint 9 | except ImportError: 10 | SubqueryConstraint = None 11 | 12 | from djangotoolbox.fields import ListField 13 | 14 | from dbindexer.lookups import StandardLookup 15 | 16 | if django.VERSION >= (1, 6): 17 | TABLE_NAME = 0 18 | RHS_ALIAS = 1 19 | JOIN_TYPE = 2 20 | LHS_ALIAS = 3 21 | 22 | def join_cols(join_info): 23 | return join_info.join_cols[0] 24 | elif django.VERSION >= (1, 5): 25 | TABLE_NAME = 0 26 | RHS_ALIAS = 1 27 | JOIN_TYPE = 2 28 | LHS_ALIAS = 3 29 | 30 | def join_cols(join_info): 31 | return (join_info.lhs_join_col, join_info.rhs_join_col) 32 | else: 33 | from django.db.models.sql.constants import (JOIN_TYPE, LHS_ALIAS, 34 | LHS_JOIN_COL, TABLE_NAME, RHS_JOIN_COL) 35 | 36 | def join_cols(join_info): 37 | return (join_info[LHS_JOIN_COL], join_info[RHS_JOIN_COL]) 38 | 39 | OR = 'OR' 40 | 41 | # TODO: optimize code 42 | class BaseResolver(object): 43 | def __init__(self): 44 | # mapping from lookups to indexes 45 | self.index_map = {} 46 | # mapping from column names to field names 47 | self.column_to_name = {} 48 | 49 | ''' API called by resolver''' 50 | 51 | def create_index(self, lookup): 52 | field_to_index = self.get_field_to_index(lookup.model, lookup.field_name) 53 | 54 | # backend doesn't now how to handle this index definition 55 | if not field_to_index: 56 | return 57 | 58 | index_field = lookup.get_field_to_add(field_to_index) 59 | config_field = index_field.item_field if \ 60 | isinstance(index_field, ListField) else index_field 61 | if field_to_index.max_length is not None and \ 62 | isinstance(config_field, models.CharField): 63 | config_field.max_length = field_to_index.max_length 64 | 65 | if isinstance(field_to_index, 66 | (models.DateField, models.DateTimeField, models.TimeField)): 67 | if field_to_index.auto_now or field_to_index.auto_now_add: 68 | raise ImproperlyConfigured('\'auto_now\' and \'auto_now_add\' ' 69 | 'on %s.%s is not supported by dbindexer.' % 70 | (lookup.model._meta.object_name, lookup.field_name)) 71 | 72 | # don't install a field if it already exists 73 | try: 74 | lookup.model._meta.get_field(self.index_name(lookup)) 75 | except: 76 | lookup.model.add_to_class(self.index_name(lookup), index_field) 77 | self.index_map[lookup] = index_field 78 | self.add_column_to_name(lookup.model, lookup.field_name) 79 | else: 80 | # makes dbindexer unit test compatible 81 | if lookup not in self.index_map: 82 | self.index_map[lookup] = lookup.model._meta.get_field( 83 | self.index_name(lookup)) 84 | self.add_column_to_name(lookup.model, lookup.field_name) 85 | 86 | def convert_insert_query(self, query): 87 | '''Converts a database saving query.''' 88 | 89 | for lookup in self.index_map.keys(): 90 | self._convert_insert_query(query, lookup) 91 | 92 | def _convert_insert_query(self, query, lookup): 93 | if not lookup.model == query.model: 94 | return 95 | 96 | position = self.get_query_position(query, lookup) 97 | if position is None: 98 | return 99 | 100 | value = self.get_value(lookup.model, lookup.field_name, query) 101 | 102 | if isinstance(value, list): 103 | for i in range(0, len(value)): 104 | setattr(query.objs[i], lookup.index_name, lookup.convert_value(value[i])) 105 | else: 106 | try: 107 | setattr(query.objs[0], lookup.index_name, lookup.convert_value(value)) 108 | except ValueError, e: 109 | ''' 110 | If lookup.index_name is a foreign key field, we need to set the actual 111 | referenced object, not just the id. When we try to set the id, we get an 112 | exception. 113 | ''' 114 | field_to_index = self.get_field_to_index(lookup.model, lookup.field_name) 115 | 116 | # backend doesn't now how to handle this index definition 117 | if not field_to_index: 118 | raise Exception('Unable to convert insert query because of unknown field' 119 | ' %s.%s' % (lookup.model._meta.object_name, lookup.field_name)) 120 | 121 | index_field = lookup.get_field_to_add(field_to_index) 122 | if isinstance(index_field, models.ForeignKey): 123 | setattr(query.objs[0], '%s_id' % lookup.index_name, lookup.convert_value(value)) 124 | else: 125 | raise 126 | 127 | def convert_filters(self, query): 128 | self._convert_filters(query, query.where) 129 | 130 | ''' helper methods ''' 131 | 132 | def _convert_filters(self, query, filters): 133 | for index, child in enumerate(filters.children[:]): 134 | if isinstance(child, Node): 135 | self._convert_filters(query, child) 136 | continue 137 | 138 | if SubqueryConstraint is not None and isinstance(child, SubqueryConstraint): 139 | continue 140 | 141 | self.convert_filter(query, filters, child, index) 142 | 143 | def convert_filter(self, query, filters, child, index): 144 | constraint, lookup_type, annotation, value = child 145 | 146 | if constraint.field is None: 147 | return 148 | 149 | field_name = self.column_to_name.get(constraint.field.column) 150 | if field_name and constraint.alias == \ 151 | query.table_map[query.model._meta.db_table][0]: 152 | for lookup in self.index_map.keys(): 153 | if lookup.matches_filter(query.model, field_name, lookup_type, 154 | value): 155 | new_lookup_type, new_value = lookup.convert_lookup(value, 156 | lookup_type) 157 | index_name = self.index_name(lookup) 158 | self._convert_filter(query, filters, child, index, 159 | new_lookup_type, new_value, index_name) 160 | 161 | def _convert_filter(self, query, filters, child, index, new_lookup_type, 162 | new_value, index_name): 163 | constraint, lookup_type, annotation, value = child 164 | lookup_type, value = new_lookup_type, new_value 165 | constraint.field = query.get_meta().get_field(index_name) 166 | constraint.col = constraint.field.column 167 | child = constraint, lookup_type, annotation, value 168 | filters.children[index] = child 169 | 170 | def index_name(self, lookup): 171 | return lookup.index_name 172 | 173 | def get_field_to_index(self, model, field_name): 174 | try: 175 | return model._meta.get_field(field_name) 176 | except: 177 | return None 178 | 179 | def get_value(self, model, field_name, query): 180 | field_to_index = self.get_field_to_index(model, field_name) 181 | 182 | if field_to_index in query.fields: 183 | values = [] 184 | for obj in query.objs: 185 | value = field_to_index.value_from_object(obj) 186 | values.append(value) 187 | if len(values): 188 | return values 189 | raise FieldDoesNotExist('Cannot find field in query.') 190 | 191 | def add_column_to_name(self, model, field_name): 192 | column_name = model._meta.get_field(field_name).column 193 | self.column_to_name[column_name] = field_name 194 | 195 | def get_index(self, lookup): 196 | return self.index_map[lookup] 197 | 198 | def get_query_position(self, query, lookup): 199 | for index, field in enumerate(query.fields): 200 | if field is self.get_index(lookup): 201 | return index 202 | return None 203 | 204 | def unref_alias(query, alias): 205 | table_name = query.alias_map[alias][TABLE_NAME] 206 | query.alias_refcount[alias] -= 1 207 | if query.alias_refcount[alias] < 1: 208 | # Remove all information about the join 209 | del query.alias_refcount[alias] 210 | if hasattr(query, 'rev_join_map'): 211 | # Django 1.4 compatibility 212 | del query.join_map[query.rev_join_map[alias]] 213 | del query.rev_join_map[alias] 214 | else: 215 | try: 216 | table, _, _, lhs, join_cols, _, _ = query.alias_map[alias] 217 | del query.join_map[(lhs, table, join_cols)] 218 | except KeyError: 219 | # Django 1.5 compatibility 220 | table, _, _, lhs, lhs_col, col, _ = query.alias_map[alias] 221 | del query.join_map[(lhs, table, lhs_col, col)] 222 | 223 | del query.alias_map[alias] 224 | query.tables.remove(alias) 225 | query.table_map[table_name].remove(alias) 226 | if len(query.table_map[table_name]) == 0: 227 | del query.table_map[table_name] 228 | query.used_aliases.discard(alias) 229 | 230 | class FKNullFix(BaseResolver): 231 | ''' 232 | Django doesn't generate correct code for ForeignKey__isnull. 233 | It becomes a JOIN with pk__isnull which won't work on nonrel DBs, 234 | so we rewrite the JOIN here. 235 | ''' 236 | 237 | def create_index(self, lookup): 238 | pass 239 | 240 | def convert_insert_query(self, query): 241 | pass 242 | 243 | def convert_filter(self, query, filters, child, index): 244 | constraint, lookup_type, annotation, value = child 245 | if constraint.field is not None and lookup_type == 'isnull' and \ 246 | isinstance(constraint.field, models.ForeignKey): 247 | self.fix_fk_null_filter(query, constraint) 248 | 249 | def unref_alias(self, query, alias): 250 | unref_alias(query, alias) 251 | 252 | def fix_fk_null_filter(self, query, constraint): 253 | alias = constraint.alias 254 | table_name = query.alias_map[alias][TABLE_NAME] 255 | lhs_join_col, rhs_join_col = join_cols(query.alias_map[alias]) 256 | if table_name != constraint.field.rel.to._meta.db_table or \ 257 | rhs_join_col != constraint.field.rel.to._meta.pk.column or \ 258 | lhs_join_col != constraint.field.column: 259 | return 260 | next_alias = query.alias_map[alias][LHS_ALIAS] 261 | if not next_alias: 262 | return 263 | self.unref_alias(query, alias) 264 | alias = next_alias 265 | constraint.col = constraint.field.column 266 | constraint.alias = alias 267 | 268 | class ConstantFieldJOINResolver(BaseResolver): 269 | def create_index(self, lookup): 270 | if '__' in lookup.field_name: 271 | super(ConstantFieldJOINResolver, self).create_index(lookup) 272 | 273 | def convert_insert_query(self, query): 274 | '''Converts a database saving query.''' 275 | 276 | for lookup in self.index_map.keys(): 277 | if '__' in lookup.field_name: 278 | self._convert_insert_query(query, lookup) 279 | 280 | def convert_filter(self, query, filters, child, index): 281 | constraint, lookup_type, annotation, value = child 282 | field_chain = self.get_field_chain(query, constraint) 283 | 284 | if field_chain is None: 285 | return 286 | 287 | for lookup in self.index_map.keys(): 288 | if lookup.matches_filter(query.model, field_chain, lookup_type, 289 | value): 290 | self.resolve_join(query, child) 291 | new_lookup_type, new_value = lookup.convert_lookup(value, 292 | lookup_type) 293 | index_name = self.index_name(lookup) 294 | self._convert_filter(query, filters, child, index, 295 | new_lookup_type, new_value, index_name) 296 | 297 | def get_field_to_index(self, model, field_name): 298 | model = self.get_model_chain(model, field_name)[-1] 299 | field_name = field_name.split('__')[-1] 300 | return super(ConstantFieldJOINResolver, self).get_field_to_index(model, 301 | field_name) 302 | 303 | def get_value(self, model, field_name, query): 304 | value = super(ConstantFieldJOINResolver, self).get_value(model, 305 | field_name.split('__')[0], 306 | query) 307 | 308 | if isinstance(value, list): 309 | value = value[0] 310 | if value is not None: 311 | value = self.get_target_value(model, field_name, value) 312 | return value 313 | 314 | def get_field_chain(self, query, constraint): 315 | if constraint.field is None: 316 | return 317 | 318 | column_index = self.get_column_index(query, constraint) 319 | return self.column_to_name.get(column_index) 320 | 321 | def get_model_chain(self, model, field_chain): 322 | model_chain = [model, ] 323 | for value in field_chain.split('__')[:-1]: 324 | model = model._meta.get_field(value).rel.to 325 | model_chain.append(model) 326 | return model_chain 327 | 328 | def get_target_value(self, start_model, field_chain, pk): 329 | fields = field_chain.split('__') 330 | foreign_key = start_model._meta.get_field(fields[0]) 331 | 332 | if not foreign_key.rel: 333 | # field isn't a related one, so return the value itself 334 | return pk 335 | 336 | target_model = foreign_key.rel.to 337 | foreignkey = target_model.objects.all().get(pk=pk) 338 | for value in fields[1:-1]: 339 | foreignkey = getattr(foreignkey, value) 340 | 341 | if isinstance(foreignkey._meta.get_field(fields[-1]), models.ForeignKey): 342 | return getattr(foreignkey, '%s_id' % fields[-1]) 343 | else: 344 | return getattr(foreignkey, fields[-1]) 345 | 346 | def add_column_to_name(self, model, field_name): 347 | model_chain = self.get_model_chain(model, field_name) 348 | column_chain = '' 349 | field_names = field_name.split('__') 350 | for model, name in zip(model_chain, field_names): 351 | column_chain += model._meta.get_field(name).column + '__' 352 | self.column_to_name[column_chain[:-2]] = field_name 353 | 354 | def unref_alias(self, query, alias): 355 | unref_alias(query, alias) 356 | 357 | def get_column_index(self, query, constraint): 358 | column_chain = [] 359 | if constraint.field: 360 | column_chain.append(constraint.col) 361 | alias = constraint.alias 362 | while alias: 363 | join = query.alias_map.get(alias) 364 | if join and join[JOIN_TYPE] == 'INNER JOIN': 365 | column_chain.insert(0, join_cols(join)[0]) 366 | alias = query.alias_map[alias][LHS_ALIAS] 367 | else: 368 | alias = None 369 | return '__'.join(column_chain) 370 | 371 | def resolve_join(self, query, child): 372 | constraint, lookup_type, annotation, value = child 373 | if not constraint.field: 374 | return 375 | 376 | alias = constraint.alias 377 | while True: 378 | next_alias = query.alias_map[alias][LHS_ALIAS] 379 | if not next_alias: 380 | break 381 | self.unref_alias(query, alias) 382 | alias = next_alias 383 | 384 | constraint.alias = alias 385 | 386 | # TODO: distinguish in memory joins from standard joins somehow 387 | class InMemoryJOINResolver(ConstantFieldJOINResolver): 388 | def __init__(self): 389 | self.field_chains = [] 390 | super(InMemoryJOINResolver, self).__init__() 391 | 392 | def create_index(self, lookup): 393 | if '__' in lookup.field_name: 394 | field_to_index = self.get_field_to_index(lookup.model, lookup.field_name) 395 | 396 | if not field_to_index: 397 | return 398 | 399 | # save old column_to_name so we can make in memory queries later on 400 | self.add_column_to_name(lookup.model, lookup.field_name) 401 | 402 | # don't add an extra field for standard lookups! 403 | if isinstance(lookup, StandardLookup): 404 | return 405 | 406 | # install lookup on target model 407 | model = self.get_model_chain(lookup.model, lookup.field_name)[-1] 408 | lookup.model = model 409 | lookup.field_name = lookup.field_name.split('__')[-1] 410 | super(ConstantFieldJOINResolver, self).create_index(lookup) 411 | 412 | def convert_insert_query(self, query): 413 | super(ConstantFieldJOINResolver, self).convert_insert_query(query) 414 | 415 | def _convert_filters(self, query, filters): 416 | # or queries are not supported for in-memory-JOINs 417 | if self.contains_OR(query.where, OR): 418 | return 419 | 420 | # start with the deepest JOIN level filter! 421 | all_filters = self.get_all_filters(filters) 422 | all_filters.sort(key=lambda item: self.get_field_chain(query, item[1][0]) and \ 423 | -len(self.get_field_chain(query, item[1][0])) or 0) 424 | 425 | for filters, child, index in all_filters: 426 | # check if convert_filter removed a given child from the where-tree 427 | if not self.contains_child(query.where, child): 428 | continue 429 | self.convert_filter(query, filters, child, index) 430 | 431 | def convert_filter(self, query, filters, child, index): 432 | constraint, lookup_type, annotation, value = child 433 | field_chain = self.get_field_chain(query, constraint) 434 | 435 | if field_chain is None: 436 | return 437 | 438 | if '__' not in field_chain: 439 | return super(ConstantFieldJOINResolver, self).convert_filter(query, 440 | filters, child, index) 441 | 442 | pks = self.get_pks(query, field_chain, lookup_type, value) 443 | self.resolve_join(query, child) 444 | self._convert_filter(query, filters, child, index, 'in', 445 | (pk for pk in pks), field_chain.split('__')[0]) 446 | 447 | def tree_contains(self, filters, to_find, func): 448 | result = False 449 | for child in filters.children[:]: 450 | if func(child, to_find): 451 | result = True 452 | break 453 | if isinstance(child, Node): 454 | result = self.tree_contains(child, to_find, func) 455 | if result: 456 | break 457 | return result 458 | 459 | def contains_OR(self, filters, or_): 460 | return self.tree_contains(filters, or_, 461 | lambda c, f: isinstance(c, Node) and c.connector == f) 462 | 463 | def contains_child(self, filters, to_find): 464 | return self.tree_contains(filters, to_find, lambda c, f: c is f) 465 | 466 | def get_all_filters(self, filters): 467 | all_filters = [] 468 | for index, child in enumerate(filters.children[:]): 469 | if isinstance(child, Node): 470 | all_filters.extend(self.get_all_filters(child)) 471 | continue 472 | 473 | all_filters.append((filters, child, index)) 474 | return all_filters 475 | 476 | def index_name(self, lookup): 477 | # use another index_name to avoid conflicts with lookups defined on the 478 | # target model which are handled by the BaseBackend 479 | return lookup.index_name + '_in_memory_join' 480 | 481 | def get_pks(self, query, field_chain, lookup_type, value): 482 | model_chain = self.get_model_chain(query.model, field_chain) 483 | 484 | first_lookup = {'%s__%s' %(field_chain.rsplit('__', 1)[-1], 485 | lookup_type): value} 486 | self.combine_with_same_level_filter(first_lookup, query, field_chain) 487 | pks = model_chain[-1].objects.all().filter(**first_lookup).values_list( 488 | 'id', flat=True) 489 | 490 | chains = [field_chain.rsplit('__', i+1)[0] 491 | for i in range(field_chain.count('__'))] 492 | lookup = {} 493 | for model, chain in reversed(zip(model_chain[1:-1], chains[:-1])): 494 | lookup.update({'%s__%s' %(chain.rsplit('__', 1)[-1], 'in'): 495 | (pk for pk in pks)}) 496 | self.combine_with_same_level_filter(lookup, query, chain) 497 | pks = model.objects.all().filter(**lookup).values_list('id', flat=True) 498 | return pks 499 | 500 | def combine_with_same_level_filter(self, lookup, query, field_chain): 501 | lookup_updates = {} 502 | field_chains = self.get_all_field_chains(query, query.where) 503 | 504 | for chain, child in field_chains.items(): 505 | if chain == field_chain: 506 | continue 507 | if field_chain.rsplit('__', 1)[0] == chain.rsplit('__', 1)[0]: 508 | lookup_updates ['%s__%s' %(chain.rsplit('__', 1)[1], child[1])] \ 509 | = child[3] 510 | 511 | self.remove_child(query.where, child) 512 | self.resolve_join(query, child) 513 | # TODO: update query.alias_refcount correctly! 514 | lookup.update(lookup_updates) 515 | 516 | def remove_child(self, filters, to_remove): 517 | ''' Removes a child object from filters. If filters doesn't contain 518 | children afterwoods, filters will be removed from its parent. ''' 519 | 520 | for child in filters.children[:]: 521 | if child is to_remove: 522 | self._remove_child(filters, to_remove) 523 | return 524 | elif isinstance(child, Node): 525 | self.remove_child(child, to_remove) 526 | 527 | if hasattr(child, 'children') and not child.children: 528 | self.remove_child(filters, child) 529 | 530 | def _remove_child(self, filters, to_remove): 531 | result = [] 532 | for child in filters.children[:]: 533 | if child is to_remove: 534 | continue 535 | result.append(child) 536 | filters.children = result 537 | 538 | def get_all_field_chains(self, query, filters): 539 | ''' Returns a dict mapping from field_chains to the corresponding child.''' 540 | 541 | field_chains = {} 542 | all_filters = self.get_all_filters(filters) 543 | for filters, child, index in all_filters: 544 | field_chain = self.get_field_chain(query, child[0]) 545 | # field_chain can be None if the user didn't specified an index for it 546 | if field_chain: 547 | field_chains[field_chain] = child 548 | return field_chains 549 | -------------------------------------------------------------------------------- /dbindexer/base.py: -------------------------------------------------------------------------------- 1 | from django.conf import settings 2 | from django.utils.importlib import import_module 3 | 4 | 5 | def merge_dicts(d1, d2): 6 | '''Update dictionary recursively. If values for a given key exist in both dictionaries and are dict-like they are merged.''' 7 | 8 | for k, v in d2.iteritems(): 9 | 10 | # Try to merge the values as if they were dicts. 11 | try: 12 | merge_dicts(d1[k], v) 13 | 14 | # Otherwise just overwrite the original value (if any). 15 | except (AttributeError, KeyError): 16 | d1[k] = v 17 | 18 | 19 | class DatabaseOperations(object): 20 | dbindexer_compiler_module = __name__.rsplit('.', 1)[0] + '.compiler' 21 | 22 | def __init__(self): 23 | self._dbindexer_cache = {} 24 | 25 | def compiler(self, compiler_name): 26 | if compiler_name not in self._dbindexer_cache: 27 | target = super(DatabaseOperations, self).compiler(compiler_name) 28 | base = getattr( 29 | import_module(self.dbindexer_compiler_module), compiler_name) 30 | class Compiler(base, target): 31 | pass 32 | self._dbindexer_cache[compiler_name] = Compiler 33 | return self._dbindexer_cache[compiler_name] 34 | 35 | class BaseDatabaseWrapper(object): 36 | def __init__(self, *args, **kwargs): 37 | super(BaseDatabaseWrapper, self).__init__(*args, **kwargs) 38 | class Operations(DatabaseOperations, self.ops.__class__): 39 | pass 40 | self.ops.__class__ = Operations 41 | self.ops.__init__() 42 | 43 | def DatabaseWrapper(settings_dict, *args, **kwargs): 44 | target_settings = settings_dict['TARGET'] 45 | if isinstance(target_settings, (str, unicode)): 46 | target_settings = settings.DATABASES[target_settings] 47 | engine = target_settings['ENGINE'] + '.base' 48 | target = import_module(engine).DatabaseWrapper 49 | class Wrapper(BaseDatabaseWrapper, target): 50 | pass 51 | 52 | # Update settings with target database settings (which can contain nested dicts). 53 | merged_settings = settings_dict.copy() 54 | merge_dicts(merged_settings, target_settings) 55 | 56 | return Wrapper(merged_settings, *args, **kwargs) 57 | -------------------------------------------------------------------------------- /dbindexer/compiler.py: -------------------------------------------------------------------------------- 1 | from .resolver import resolver 2 | from django.utils.importlib import import_module 3 | 4 | def __repr__(self): 5 | return '<%s, %s, %s, %s>' % (self.alias, self.col, self.field.name, 6 | self.field.model.__name__) 7 | 8 | from django.db.models.sql.where import Constraint 9 | Constraint.__repr__ = __repr__ 10 | 11 | # TODO: manipulate a copy of the query instead of the query itself. This has to 12 | # be done because the query can be reused afterwards by the user so that a 13 | # manipulated query can result in strange behavior for these cases! 14 | # TODO: Add watching layer which gives suggestions for indexes via query inspection 15 | # at runtime 16 | 17 | class BaseCompiler(object): 18 | def convert_filters(self): 19 | resolver.convert_filters(self.query) 20 | 21 | class SQLCompiler(BaseCompiler): 22 | def execute_sql(self, *args, **kwargs): 23 | self.convert_filters() 24 | return super(SQLCompiler, self).execute_sql(*args, **kwargs) 25 | 26 | def results_iter(self): 27 | self.convert_filters() 28 | return super(SQLCompiler, self).results_iter() 29 | 30 | def has_results(self): 31 | self.convert_filters() 32 | return super(SQLCompiler, self).has_results() 33 | 34 | class SQLInsertCompiler(BaseCompiler): 35 | def execute_sql(self, return_id=False): 36 | resolver.convert_insert_query(self.query) 37 | return super(SQLInsertCompiler, self).execute_sql(return_id=return_id) 38 | 39 | class SQLUpdateCompiler(BaseCompiler): 40 | pass 41 | 42 | class SQLDeleteCompiler(BaseCompiler): 43 | pass 44 | 45 | class SQLDateCompiler(BaseCompiler): 46 | pass 47 | 48 | class SQLDateTimeCompiler(BaseCompiler): 49 | pass 50 | 51 | class SQLAggregateCompiler(BaseCompiler): 52 | pass 53 | -------------------------------------------------------------------------------- /dbindexer/lookups.py: -------------------------------------------------------------------------------- 1 | from django.db import models 2 | from djangotoolbox.fields import ListField 3 | from copy import deepcopy 4 | 5 | import re 6 | regex = type(re.compile('')) 7 | 8 | class LookupDoesNotExist(Exception): 9 | pass 10 | 11 | class LookupBase(type): 12 | def __new__(cls, name, bases, attrs): 13 | new_cls = type.__new__(cls, name, bases, attrs) 14 | if not isinstance(new_cls.lookup_types, (list, tuple)): 15 | new_cls.lookup_types = (new_cls.lookup_types, ) 16 | return new_cls 17 | 18 | class ExtraFieldLookup(object): 19 | '''Default is to behave like an exact filter on an ExtraField.''' 20 | __metaclass__ = LookupBase 21 | lookup_types = 'exact' 22 | 23 | def __init__(self, model=None, field_name=None, lookup_def=None, 24 | new_lookup='exact', field_to_add=models.CharField( 25 | max_length=500, editable=False, null=True)): 26 | self.field_to_add = field_to_add 27 | self.new_lookup = new_lookup 28 | self.contribute(model, field_name, lookup_def) 29 | 30 | def contribute(self, model, field_name, lookup_def): 31 | self.model = model 32 | self.field_name = field_name 33 | self.lookup_def = lookup_def 34 | 35 | @property 36 | def index_name(self): 37 | return 'idxf_%s_l_%s' % (self.field_name, self.lookup_types[0]) 38 | 39 | def convert_lookup(self, value, lookup_type): 40 | # TODO: can value be a list or tuple? (in case of in yes) 41 | if isinstance(value, (tuple, list)): 42 | value = [self._convert_lookup(val, lookup_type)[1] for val in value] 43 | else: 44 | _, value = self._convert_lookup(value, lookup_type) 45 | return self.new_lookup, value 46 | 47 | def _convert_lookup(self, value, lookup_type): 48 | return lookup_type, value 49 | 50 | def convert_value(self, value): 51 | if value is not None: 52 | if isinstance(value, (tuple, list)): 53 | value = [self._convert_value(val) for val in value if val is not None] 54 | else: 55 | value = self._convert_value(value) 56 | return value 57 | 58 | def _convert_value(self, value): 59 | return value 60 | 61 | def matches_filter(self, model, field_name, lookup_type, value): 62 | return self.model == model and lookup_type in self.lookup_types \ 63 | and field_name == self.field_name 64 | 65 | @classmethod 66 | def matches_lookup_def(cls, lookup_def): 67 | if lookup_def in cls.lookup_types: 68 | return True 69 | return False 70 | 71 | def get_field_to_add(self, field_to_index): 72 | field_to_add = deepcopy(self.field_to_add) 73 | if isinstance(field_to_index, ListField): 74 | field_to_add = ListField(field_to_add, editable=False, null=True) 75 | return field_to_add 76 | 77 | class DateLookup(ExtraFieldLookup): 78 | # DateLookup is abstract so set lookup_types to None so it doesn't match 79 | lookup_types = None 80 | 81 | def __init__(self, *args, **kwargs): 82 | defaults = {'new_lookup': 'exact', 83 | 'field_to_add': models.IntegerField(editable=False, null=True)} 84 | defaults.update(kwargs) 85 | ExtraFieldLookup.__init__(self, *args, **defaults) 86 | 87 | def _convert_lookup(self, value, lookup_type): 88 | return self.new_lookup, value 89 | 90 | class Day(DateLookup): 91 | lookup_types = 'day' 92 | 93 | def _convert_value(self, value): 94 | return value.day 95 | 96 | class Month(DateLookup): 97 | lookup_types = 'month' 98 | 99 | def _convert_value(self, value): 100 | return value.month 101 | 102 | class Year(DateLookup): 103 | lookup_types = 'year' 104 | 105 | def _convert_value(self, value): 106 | return value.year 107 | 108 | class Weekday(DateLookup): 109 | lookup_types = 'week_day' 110 | 111 | def _convert_value(self, value): 112 | return value.isoweekday() 113 | 114 | class Contains(ExtraFieldLookup): 115 | lookup_types = 'contains' 116 | 117 | def __init__(self, *args, **kwargs): 118 | defaults = {'new_lookup': 'startswith', 119 | 'field_to_add': ListField(models.CharField(500), 120 | editable=False, null=True) 121 | } 122 | defaults.update(kwargs) 123 | ExtraFieldLookup.__init__(self, *args, **defaults) 124 | 125 | def get_field_to_add(self, field_to_index): 126 | # always return a ListField of CharFields even in the case of 127 | # field_to_index being a ListField itself! 128 | return deepcopy(self.field_to_add) 129 | 130 | def convert_value(self, value): 131 | new_value = [] 132 | if isinstance(value, (tuple, list)): 133 | for val in value: 134 | new_value.extend(self.contains_indexer(val)) 135 | else: 136 | new_value = self.contains_indexer(value) 137 | return new_value 138 | 139 | def _convert_lookup(self, value, lookup_type): 140 | return self.new_lookup, value 141 | 142 | def contains_indexer(self, value): 143 | # In indexing mode we add all postfixes ('o', 'lo', ..., 'hello') 144 | result = [] 145 | if value: 146 | result.extend([value[count:] for count in range(len(value))]) 147 | return result 148 | 149 | class Icontains(Contains): 150 | lookup_types = 'icontains' 151 | 152 | def convert_value(self, value): 153 | return [val.lower() for val in Contains.convert_value(self, value)] 154 | 155 | def _convert_lookup(self, value, lookup_type): 156 | return self.new_lookup, value.lower() 157 | 158 | class Iexact(ExtraFieldLookup): 159 | lookup_types = 'iexact' 160 | 161 | def _convert_lookup(self, value, lookup_type): 162 | return self.new_lookup, value.lower() 163 | 164 | def _convert_value(self, value): 165 | return value.lower() 166 | 167 | class Istartswith(ExtraFieldLookup): 168 | lookup_types = 'istartswith' 169 | 170 | def __init__(self, *args, **kwargs): 171 | defaults = {'new_lookup': 'startswith'} 172 | defaults.update(kwargs) 173 | ExtraFieldLookup.__init__(self, *args, **defaults) 174 | 175 | def _convert_lookup(self, value, lookup_type): 176 | return self.new_lookup, value.lower() 177 | 178 | def _convert_value(self, value): 179 | return value.lower() 180 | 181 | class Endswith(ExtraFieldLookup): 182 | lookup_types = 'endswith' 183 | 184 | def __init__(self, *args, **kwargs): 185 | defaults = {'new_lookup': 'startswith'} 186 | defaults.update(kwargs) 187 | ExtraFieldLookup.__init__(self, *args, **defaults) 188 | 189 | def _convert_lookup(self, value, lookup_type): 190 | return self.new_lookup, value[::-1] 191 | 192 | def _convert_value(self, value): 193 | return value[::-1] 194 | 195 | class Iendswith(Endswith): 196 | lookup_types = 'iendswith' 197 | 198 | def _convert_lookup(self, value, lookup_type): 199 | return self.new_lookup, value[::-1].lower() 200 | 201 | def _convert_value(self, value): 202 | return value[::-1].lower() 203 | 204 | class RegexLookup(ExtraFieldLookup): 205 | lookup_types = ('regex', 'iregex') 206 | 207 | def __init__(self, *args, **kwargs): 208 | defaults = {'field_to_add': models.NullBooleanField(editable=False, 209 | null=True) 210 | } 211 | defaults.update(kwargs) 212 | ExtraFieldLookup.__init__(self, *args, **defaults) 213 | 214 | def contribute(self, model, field_name, lookup_def): 215 | ExtraFieldLookup.contribute(self, model, field_name, lookup_def) 216 | if isinstance(lookup_def, regex): 217 | self.lookup_def = re.compile(lookup_def.pattern, re.S | re.U | 218 | (lookup_def.flags & re.I)) 219 | 220 | @property 221 | def index_name(self): 222 | return 'idxf_%s_l_%s' % (self.field_name, 223 | self.lookup_def.pattern.encode('hex')) 224 | 225 | def is_icase(self): 226 | return self.lookup_def.flags & re.I 227 | 228 | def _convert_lookup(self, value, lookup_type): 229 | return self.new_lookup, True 230 | 231 | def _convert_value(self, value): 232 | if self.lookup_def.match(value): 233 | return True 234 | return False 235 | 236 | def matches_filter(self, model, field_name, lookup_type, value): 237 | return self.model == model and lookup_type == \ 238 | '%sregex' % ('i' if self.is_icase() else '') and \ 239 | value == self.lookup_def.pattern and field_name == self.field_name 240 | 241 | @classmethod 242 | def matches_lookup_def(cls, lookup_def): 243 | if isinstance(lookup_def, regex): 244 | return True 245 | return False 246 | 247 | class StandardLookup(ExtraFieldLookup): 248 | ''' Creates a copy of the field_to_index in order to allow querying for 249 | standard lookup_types on a JOINed property. ''' 250 | # TODO: database backend can specify standardLookups 251 | lookup_types = ('exact', 'gt', 'gte', 'lt', 'lte', 'in', 'range', 'isnull') 252 | 253 | @property 254 | def index_name(self): 255 | return 'idxf_%s_l_%s' % (self.field_name, 'standard') 256 | 257 | def convert_lookup(self, value, lookup_type): 258 | return lookup_type, value 259 | 260 | def get_field_to_add(self, field_to_index): 261 | field_to_add = deepcopy(field_to_index) 262 | if isinstance(field_to_add, (models.DateTimeField, 263 | models.DateField, models.TimeField)): 264 | field_to_add.auto_now_add = field_to_add.auto_now = False 265 | field_to_add.name = self.index_name 266 | return field_to_add 267 | -------------------------------------------------------------------------------- /dbindexer/models.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/django-nonrel/django-dbindexer/445ca9f844bdd3d3136932aa14f1170c7640ae9c/dbindexer/models.py -------------------------------------------------------------------------------- /dbindexer/resolver.py: -------------------------------------------------------------------------------- 1 | from django.conf import settings 2 | from django.utils.importlib import import_module 3 | from django.core.exceptions import ImproperlyConfigured 4 | 5 | class Resolver(object): 6 | def __init__(self): 7 | self.backends = [] 8 | self.load_backends(getattr(settings, 'DBINDEXER_BACKENDS', 9 | ('dbindexer.backends.BaseResolver', 10 | 'dbindexer.backends.FKNullFix'))) 11 | 12 | def load_backends(self, backend_paths): 13 | for backend in backend_paths: 14 | self.backends.append(self.load_backend(backend)) 15 | 16 | def load_backend(self, path): 17 | module_name, attr_name = path.rsplit('.', 1) 18 | try: 19 | mod = import_module(module_name) 20 | except (ImportError, ValueError), e: 21 | raise ImproperlyConfigured('Error importing backend module %s: "%s"' 22 | % (module_name, e)) 23 | try: 24 | return getattr(mod, attr_name)() 25 | except AttributeError: 26 | raise ImproperlyConfigured('Module "%s" does not define a "%s" backend' 27 | % (module_name, attr_name)) 28 | 29 | def convert_filters(self, query): 30 | for backend in self.backends: 31 | backend.convert_filters(query) 32 | 33 | def create_index(self, lookup): 34 | for backend in self.backends: 35 | backend.create_index(lookup) 36 | 37 | def convert_insert_query(self, query): 38 | for backend in self.backends: 39 | backend.convert_insert_query(query) 40 | 41 | resolver = Resolver() 42 | -------------------------------------------------------------------------------- /dbindexer/tests.py: -------------------------------------------------------------------------------- 1 | from django.db import models 2 | from django.test import TestCase 3 | from .api import register_index 4 | from .lookups import StandardLookup 5 | from .resolver import resolver 6 | from djangotoolbox.fields import ListField 7 | from datetime import datetime 8 | import re 9 | 10 | class ForeignIndexed2(models.Model): 11 | name_fi2 = models.CharField(max_length=500) 12 | age = models.IntegerField() 13 | 14 | class ForeignIndexed(models.Model): 15 | title = models.CharField(max_length=500) 16 | name_fi = models.CharField(max_length=500) 17 | fk = models.ForeignKey(ForeignIndexed2, null=True) 18 | 19 | class Indexed(models.Model): 20 | name = models.CharField(max_length=500) 21 | published = models.DateTimeField(auto_now_add=True) 22 | foreignkey = models.ForeignKey(ForeignIndexed, null=True) 23 | foreignkey2 = models.ForeignKey(ForeignIndexed2, related_name='idx_set', null=True) 24 | tags = ListField(models.CharField(max_length=500, null=True)) 25 | 26 | class NullableCharField(models.Model): 27 | name = models.CharField(max_length=500, null=True) 28 | 29 | # TODO: add test for foreign key with multiple filters via different and equal paths 30 | # to do so we have to create some entities matching equal paths but not matching 31 | # different paths 32 | class IndexedTest(TestCase): 33 | def setUp(self): 34 | self.backends = list(resolver.backends) 35 | resolver.backends = [] 36 | resolver.load_backends(('dbindexer.backends.BaseResolver', 37 | 'dbindexer.backends.FKNullFix', 38 | # 'dbindexer.backends.InMemoryJOINResolver', 39 | 'dbindexer.backends.ConstantFieldJOINResolver', 40 | )) 41 | self.register_indexes() 42 | 43 | juubi = ForeignIndexed2(name_fi2='Juubi', age=2) 44 | juubi.save() 45 | rikudo = ForeignIndexed2(name_fi2='Rikudo', age=200) 46 | rikudo.save() 47 | 48 | kyuubi = ForeignIndexed(name_fi='Kyuubi', title='Bijuu', fk=juubi) 49 | hachibi= ForeignIndexed(name_fi='Hachibi', title='Bijuu', fk=rikudo) 50 | kyuubi.save() 51 | hachibi.save() 52 | 53 | Indexed(name='ItAchi', tags=('Sasuke', 'Madara'), foreignkey=kyuubi, 54 | foreignkey2=juubi).save() 55 | Indexed(name='YondAimE', tags=('Naruto', 'Jiraya'), foreignkey=kyuubi, 56 | foreignkey2=juubi).save() 57 | Indexed(name='Neji', tags=('Hinata'), foreignkey=hachibi, 58 | foreignkey2=juubi).save() 59 | Indexed(name='I1038593i', tags=('Sharingan'), foreignkey=hachibi, 60 | foreignkey2=rikudo).save() 61 | 62 | def tearDown(self): 63 | resolver.backends = self.backends 64 | 65 | def register_indexes(self): 66 | register_index(Indexed, { 67 | 'name': ('iexact', 'endswith', 'istartswith', 'iendswith', 'contains', 68 | 'icontains', re.compile('^i+', re.I), re.compile('^I+'), 69 | re.compile('^i\d*i$', re.I)), 70 | 'tags': ('iexact', 'icontains', StandardLookup() ), 71 | 'foreignkey__fk': (StandardLookup()), 72 | 'foreignkey__title': 'iexact', 73 | 'foreignkey__name_fi': 'iexact', 74 | 'foreignkey__fk__name_fi2': ('iexact', 'endswith'), 75 | 'foreignkey2__name_fi2': (StandardLookup(), 'iexact'), 76 | 'foreignkey2__age': (StandardLookup()) 77 | }) 78 | 79 | register_index(ForeignIndexed, { 80 | 'title': 'iexact', 81 | 'name_fi': ('iexact', 'icontains'), 82 | 'fk__name_fi2': ('iexact', 'endswith'), 83 | 'fk__age': (StandardLookup()), 84 | }) 85 | 86 | register_index(NullableCharField, { 87 | 'name': ('iexact', 'istartswith', 'endswith', 'iendswith',) 88 | }) 89 | 90 | # TODO: add tests for created indexes for all backends! 91 | # def test_model_fields(self): 92 | # field_list = [(item[0], item[0].column) 93 | # for item in Indexed._meta.get_fields_with_model()] 94 | # print field_list 95 | # x() 96 | # in-memory JOIN backend shouldn't create multiple indexes on the foreignkey side 97 | # for different paths or not even for index definition on different models. Test this! 98 | # standard JOIN backend should always add extra fields to registered model. Test this! 99 | 100 | def test_joins(self): 101 | self.assertEqual(2, len(Indexed.objects.all().filter( 102 | foreignkey__fk__name_fi2__iexact='juuBi', 103 | foreignkey__title__iexact='biJuu'))) 104 | 105 | self.assertEqual(0, len(Indexed.objects.all().filter( 106 | foreignkey__fk__name_fi2__iexact='juuBi', 107 | foreignkey2__name_fi2__iexact='Rikudo'))) 108 | 109 | self.assertEqual(1, len(Indexed.objects.all().filter( 110 | foreignkey__fk__name_fi2__endswith='udo', 111 | foreignkey2__name_fi2__iexact='Rikudo'))) 112 | 113 | self.assertEqual(2, len(Indexed.objects.all().filter( 114 | foreignkey__title__iexact='biJuu', 115 | foreignkey__name_fi__iexact='kyuuBi'))) 116 | 117 | self.assertEqual(2, len(Indexed.objects.all().filter( 118 | foreignkey__title__iexact='biJuu', 119 | foreignkey__name_fi__iexact='Hachibi'))) 120 | 121 | self.assertEqual(1, len(Indexed.objects.all().filter( 122 | foreignkey__title__iexact='biJuu', name__iendswith='iMe'))) 123 | 124 | # JOINs on one field only 125 | self.assertEqual(4, len(Indexed.objects.all().filter( 126 | foreignkey__title__iexact='biJuu'))) 127 | self.assertEqual(3, len(Indexed.objects.all().filter( 128 | foreignkey2__name_fi2='Juubi'))) 129 | 130 | # text endswith instead iexact all the time :) 131 | self.assertEqual(2, len(Indexed.objects.all().filter( 132 | foreignkey__fk__name_fi2__endswith='bi'))) 133 | 134 | # test JOINs via different paths targeting the same field 135 | self.assertEqual(2, len(Indexed.objects.all().filter( 136 | foreignkey__fk__name_fi2__iexact='juuBi'))) 137 | self.assertEqual(3, len(Indexed.objects.all().filter( 138 | foreignkey2__name_fi2__iexact='Juubi'))) 139 | 140 | # test standard lookups for foreign_keys 141 | self.assertEqual(3, len(Indexed.objects.all().filter( 142 | foreignkey2__age=2))) 143 | self.assertEqual(4, len(Indexed.objects.all().filter( 144 | foreignkey2__age__lt=201))) 145 | 146 | # test JOINs on different model 147 | # standard lookups JOINs 148 | self.assertEqual(1, len(ForeignIndexed.objects.all().filter( 149 | fk__age=2))) 150 | self.assertEqual(2, len(ForeignIndexed.objects.all().filter( 151 | fk__age__lt=210))) 152 | 153 | # other JOINs 154 | self.assertEqual(1, len(ForeignIndexed.objects.all().filter( 155 | fk__name_fi2__iexact='juUBI'))) 156 | self.assertEqual(1, len(ForeignIndexed.objects.all().filter( 157 | fk__name_fi2__endswith='bi'))) 158 | 159 | def test_fix_fk_isnull(self): 160 | self.assertEqual(0, len(Indexed.objects.filter(foreignkey=None))) 161 | self.assertEqual(4, len(Indexed.objects.exclude(foreignkey=None))) 162 | 163 | def test_iexact(self): 164 | self.assertEqual(1, len(Indexed.objects.filter(name__iexact='itaChi'))) 165 | self.assertEqual(1, Indexed.objects.filter(name__iexact='itaChi').count()) 166 | 167 | self.assertEqual(2, ForeignIndexed.objects.filter(title__iexact='BIJUU').count()) 168 | self.assertEqual(1, ForeignIndexed.objects.filter(name_fi__iexact='KYuubi').count()) 169 | 170 | # test on list field 171 | self.assertEqual(1, Indexed.objects.filter(tags__iexact='SasuKE').count()) 172 | 173 | def test_standard_lookups(self): 174 | self.assertEqual(1, Indexed.objects.filter(tags__exact='Naruto').count()) 175 | 176 | # test standard lookup on foreign_key 177 | juubi = ForeignIndexed2.objects.all().get(name_fi2='Juubi', age=2) 178 | self.assertEqual(2, Indexed.objects.filter(foreignkey__fk=juubi).count()) 179 | 180 | def test_delete(self): 181 | Indexed.objects.get(name__iexact='itaChi').delete() 182 | self.assertEqual(0, Indexed.objects.all().filter(name__iexact='itaChi').count()) 183 | 184 | def test_delete_query(self): 185 | Indexed.objects.all().delete() 186 | self.assertEqual(0, Indexed.objects.all().filter(name__iexact='itaChi').count()) 187 | 188 | def test_exists_query(self): 189 | self.assertTrue(Indexed.objects.filter(name__iexact='itaChi').exists()) 190 | 191 | def test_istartswith(self): 192 | self.assertEqual(1, len(Indexed.objects.all().filter(name__istartswith='iTa'))) 193 | 194 | def test_endswith(self): 195 | self.assertEqual(1, len(Indexed.objects.all().filter(name__endswith='imE'))) 196 | self.assertEqual(1, len(Indexed.objects.all().filter(name__iendswith='iMe'))) 197 | 198 | def test_regex(self): 199 | self.assertEqual(2, len(Indexed.objects.all().filter(name__iregex='^i+'))) 200 | self.assertEqual(2, len(Indexed.objects.all().filter(name__regex='^I+'))) 201 | self.assertEqual(1, len(Indexed.objects.all().filter(name__iregex='^i\d*i$'))) 202 | 203 | def test_null_strings(self): 204 | """Test indexing with nullable CharFields, see: https://github.com/django-nonrel/django-dbindexer/issues/3.""" 205 | NullableCharField.objects.create() 206 | 207 | def test_contains(self): 208 | self.assertEqual(1, len(Indexed.objects.all().filter(name__contains='Aim'))) 209 | self.assertEqual(1, len(Indexed.objects.all().filter(name__icontains='aim'))) 210 | 211 | self.assertEqual(1, ForeignIndexed.objects.filter(name_fi__icontains='Yu').count()) 212 | 213 | # test icontains on a list 214 | self.assertEqual(2, len(Indexed.objects.all().filter(tags__icontains='RA'))) 215 | 216 | 217 | class AutoNowIndexed(models.Model): 218 | published = models.DateTimeField(auto_now=True) 219 | 220 | class AutoNowAddIndexed(models.Model): 221 | published = models.DateTimeField(auto_now_add=True) 222 | 223 | class DateIndexed(models.Model): 224 | published = models.DateTimeField() 225 | 226 | class DateAutoNowTest(TestCase): 227 | def setUp(self): 228 | self.backends = list(resolver.backends) 229 | resolver.backends = [] 230 | resolver.load_backends(('dbindexer.backends.BaseResolver', 231 | 'dbindexer.backends.FKNullFix', 232 | # 'dbindexer.backends.InMemoryJOINResolver', 233 | 'dbindexer.backends.ConstantFieldJOINResolver', 234 | )) 235 | self.register_indexes() 236 | 237 | DateIndexed(published=datetime.now()).save() 238 | DateIndexed(published=datetime.now()).save() 239 | DateIndexed(published=datetime.now()).save() 240 | DateIndexed(published=datetime.now()).save() 241 | 242 | def tearDown(self): 243 | resolver.backends = self.backends 244 | 245 | def register_indexes(self): 246 | register_index(DateIndexed, { 247 | 'published': ('month', 'day', 'year', 'week_day'), 248 | }) 249 | 250 | def test_auto_now(self): 251 | from django.core.exceptions import ImproperlyConfigured 252 | 253 | self.assertRaises(ImproperlyConfigured, register_index, AutoNowIndexed, { 254 | 'published': ('month', 'day', 'year', 'week_day'), 255 | }) 256 | self.assertRaises(ImproperlyConfigured, register_index, AutoNowAddIndexed, { 257 | 'published': ('month', 'day', 'year', 'week_day'), 258 | }) 259 | 260 | def test_date_filters(self): 261 | now = datetime.now() 262 | self.assertEqual(4, len(DateIndexed.objects.all().filter(published__month=now.month))) 263 | self.assertEqual(4, len(DateIndexed.objects.all().filter(published__day=now.day))) 264 | self.assertEqual(4, len(DateIndexed.objects.all().filter(published__year=now.year))) 265 | self.assertEqual(4, len(DateIndexed.objects.all().filter( 266 | published__week_day=now.isoweekday()))) 267 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | from setuptools import setup, find_packages 2 | 3 | 4 | DESCRIPTION = 'Expressive NoSQL for Django' 5 | LONG_DESCRIPTION = None 6 | try: 7 | LONG_DESCRIPTION = open('README.rst').read() 8 | except: 9 | pass 10 | 11 | setup(name='django-dbindexer', 12 | version='1.6.1', 13 | description=DESCRIPTION, 14 | long_description=LONG_DESCRIPTION, 15 | author='Waldemar Kornewald, Thomas Wanschik', 16 | author_email='team@allbuttonspressed.com', 17 | url='https://github.com/django-nonrel/django-dbindexer', 18 | packages=find_packages(), 19 | license='3-clause BSD', 20 | classifiers=[ 21 | 'Development Status :: 4 - Beta', 22 | 'Environment :: Web Environment', 23 | 'Framework :: Django', 24 | 'Intended Audience :: Developers', 25 | 'License :: OSI Approved :: BSD License', 26 | 'Operating System :: OS Independent', 27 | 'Programming Language :: Python', 28 | 'Programming Language :: Python :: 2.5', 29 | 'Programming Language :: Python :: 2.6', 30 | 'Programming Language :: Python :: 2.7', 31 | 'Topic :: Software Development :: Libraries :: Application Frameworks', 32 | 'Topic :: Software Development :: Libraries :: Python Modules', 33 | ], 34 | ) 35 | --------------------------------------------------------------------------------