├── .gitignore ├── .pyup.yml ├── .travis.yml ├── LICENSE.txt ├── MANIFEST.in ├── README.md ├── manage.py ├── setup.cfg ├── setup.py ├── sphinxsearch ├── __init__.py ├── backend │ ├── __init__.py │ └── sphinx │ │ ├── __init__.py │ │ ├── base.py │ │ └── compiler.py ├── compat.py ├── fields.py ├── lookups.py ├── models.py ├── routers.py ├── sql.py └── utils.py ├── test-requires.txt ├── test_config └── sphinx.conf └── testproject ├── __init__.py ├── settings.py ├── testapp ├── __init__.py ├── models.py └── tests.py ├── urls.py └── wsgi.py /.gitignore: -------------------------------------------------------------------------------- 1 | *.pyc 2 | .idea 3 | *.egg-info 4 | -------------------------------------------------------------------------------- /.pyup.yml: -------------------------------------------------------------------------------- 1 | requirements: 2 | - test-requires.txt 3 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: python 2 | python: 3 | - "2.7" 4 | - "3.4" 5 | - "3.5" 6 | - "3.6" 7 | 8 | env: 9 | - DJANGO=1.8 10 | - DJANGO=1.9 11 | - DJANGO=1.10 12 | - DJANGO=1.11 13 | 14 | matrix: 15 | exclude: 16 | - python: "3.3" 17 | env: DJANGO=1.9 18 | - python: "3.3" 19 | env: DJANGO=1.10 20 | 21 | # command to install dependencies 22 | install: 23 | - sudo add-apt-repository ppa:builds/sphinxsearch-rel22 -y 24 | - sudo apt-get update 25 | - sudo apt-get install -y sphinxsearch 26 | - pip install Django==$DJANGO 27 | - pip install -r test-requires.txt 28 | - pip install codecov 29 | 30 | # command to run tests 31 | script: 32 | - searchd -c `pwd`/test_config/sphinx.conf 33 | - python -m coverage run ./manage.py test -v3 34 | 35 | after-script: kill `cat /tmp/searchd.pid` 36 | after_success: codecov 37 | -------------------------------------------------------------------------------- /LICENSE.txt: -------------------------------------------------------------------------------- 1 | "THE BEER-WARE LICENSE" (Revision 42): 2 | created project "django_sphinxsearch". 3 | As long as you retain this notice you can do whatever you want with this stuff. 4 | If we meet some day, and you think this stuff is worth it, you can buy me a 5 | beer in return. 6 | 7 | Sergey Tikhonov. 8 | 9 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include LICENSE.txt README.md 2 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Django-sphinxsearch 2 | 3 | [SphinxSearch](http://sphinxsearch.com) database backend for [Django](https://www.djangoproject.com/). 4 | 5 | [![Build Status](https://travis-ci.org/rutube/django_sphinxsearch.svg)](https://travis-ci.org/rutube/django_sphinxsearch) 6 | [![codecov](https://codecov.io/gh/rutube/django_sphinxsearch/branch/master/graph/badge.svg)](https://codecov.io/gh/rutube/django_sphinxsearch) 7 | [![PyPI version](https://badge.fury.io/py/django_sphinxsearch.svg)](http://badge.fury.io/py/django_sphinxsearch) 8 | 9 | * Not a [django_sphinx_db](https://github.com/smartfile/django-sphinx-db) fork 10 | * `Django>=1.8,<=1.11` supported 11 | 12 | ## Installation and usage 13 | 14 | 1. Install django-sphinxsearch package 15 | 16 | ```sh 17 | pip install django_sphinxsearch 18 | ``` 19 | 20 | 2. Configure Django settings 21 | 22 | ```python 23 | 24 | INSTALLED_APPS += ( 25 | 'sphinxsearch', 26 | ) 27 | 28 | SPHINX_DATABASE_NAME = 'sphinx' 29 | 30 | DATABASES[SPHINX_DATABASE_NAME] = { 31 | 'ENGINE': 'sphinxsearch.backend.sphinx', 32 | 'HOST': '127.0.0.1', 33 | 'PORT': 9306, 34 | } 35 | 36 | DATABASE_ROUTERS = ['sphinxsearch.routers.SphinxRouter'] 37 | ``` 38 | 39 | 3. Create index definitions in sphinx.conf 40 | 41 | ``` 42 | index testapp_testmodel 43 | { 44 | type = rt 45 | path = /data/sphinx/testapp/testmodel/ 46 | 47 | rt_field = sphinx_field 48 | rt_attr_uint = attr_uint 49 | rt_attr_bool = attr_bool 50 | rt_attr_bigint = attr_bigint 51 | rt_attr_float = attr_float 52 | rt_attr_multi = attr_multi 53 | rt_attr_multi_64 = attr_multi_64 54 | rt_attr_timestamp = attr_timestamp 55 | rt_attr_string = attr_string 56 | rt_attr_json = attr_json 57 | } 58 | ``` 59 | 60 | 4. Define Django model for index 61 | 62 | ```python 63 | import six 64 | from datetime import datetime 65 | from django.db import models 66 | 67 | from jsonfield.fields import JSONField 68 | 69 | from sphinxsearch import sql 70 | from sphinxsearch import models as spx_models 71 | 72 | 73 | class FieldMixin(spx_models.SphinxModel): 74 | # Note that NULL values are not allowed for sphinx rt-index. 75 | # Indexed text field. If no attribute with same name defined, can't be 76 | # retrieved from index. 77 | 78 | class Meta: 79 | abstract = True 80 | 81 | # Indexed text field. If no attribute with same name defined, can't be 82 | # retrieved from index. 83 | sphinx_field = spx_models.SphinxField(default='') 84 | other_field = spx_models.SphinxField(default='') 85 | 86 | # Numeric attributes 87 | attr_uint = spx_models.SphinxIntegerField(default=0, db_column='attr_uint_') 88 | attr_bigint = spx_models.SphinxBigIntegerField(default=0) 89 | attr_float = models.FloatField(default=0.0) 90 | attr_timestamp = spx_models.SphinxDateTimeField(default=datetime.now) 91 | attr_bool = models.BooleanField(default=False) 92 | 93 | # String attributes 94 | attr_string = models.CharField(max_length=32, default='') 95 | attr_json = JSONField(default={}) 96 | 97 | # Multi-value fields (sets of integer values) 98 | attr_multi = spx_models.SphinxMultiField(default=[]) 99 | attr_multi_64 = spx_models.SphinxMulti64Field(default=[]) 100 | 101 | 102 | class TestModel(FieldMixin, spx_models.SphinxModel): 103 | pass 104 | ``` 105 | 106 | 5. Query index from your app 107 | 108 | ```python 109 | 110 | # Numeric attributes filtering 111 | TestModel.objects.filter(attr_uint=0, attr_float__gte=10, attr_multi__in=[1, 2]) 112 | 113 | # For sphinxsearch>=2.2.7, string attr filtering enabled 114 | TestModel.objects.filter(attr_string='some test') 115 | 116 | # Use mysql-fulltext-search filtering: 117 | 118 | TestModel.objects.filter(sphinx_field__search='find me') 119 | 120 | # Run match queries 121 | TestModel.objects.match( 122 | 'find in all fields', 123 | sphinx_field='only in this field') 124 | 125 | # Insert and update documents to index 126 | 127 | obj = TestModel.objects.create(**values) 128 | obj.attr_uint = 1 129 | obj.save() 130 | 131 | TestModel.objects.filter(attr_bool=True).update(attr_uint=2) 132 | ``` 133 | 134 | ## Notes for production usage 135 | 136 | * Sphinxsearch engine has some issues with SQL-syntax support, and they vary 137 | from one version to another. I.e. float attributes are not comparable, 138 | string attributes were not comparible till v2.2.7. 139 | * Without limits sphinxsearch returns only 20 matched documents. 140 | * uint attributes accept -1 but return it as unsigned 32bit integer. 141 | * bigint accept 2**63 + 1 but return it as signed 64bit integer. 142 | * use SphinxIntegerField and SphinxBigIntegerField instead of IntegerField and 143 | BigIntegerField from django.db.models, because IN is an expression in 144 | SQL (`value IN column`), but a function (`IN(value, column)`) in sphinxsearch. 145 | -------------------------------------------------------------------------------- /manage.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | import os 3 | import sys 4 | 5 | sys.path.append(os.path.join(os.path.dirname(__file__), "testproject")) 6 | 7 | if __name__ == "__main__": 8 | os.environ.setdefault("DJANGO_SETTINGS_MODULE", "testproject.settings") 9 | 10 | from django.core.management import execute_from_command_line 11 | 12 | execute_from_command_line(sys.argv) 13 | -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [metadata] 2 | description-file = README.md 3 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | from distutils.core import setup 2 | import sys 3 | 4 | if sys.version_info < (3, 0): 5 | mysql = 'MySQL-python' 6 | else: 7 | mysql = 'PyMySQL' 8 | 9 | try: 10 | # noinspection PyPackageRequirements 11 | from pypandoc import convert 12 | read_md = lambda f: convert(f, 'rst') 13 | except ImportError: 14 | print("warning: pypandoc not found, could not convert Markdown to RST") 15 | read_md = lambda f: open(f, 'r').read() 16 | 17 | setup( 18 | name='django_sphinxsearch', 19 | version='0.8.1', 20 | long_description=read_md('README.md'), 21 | packages=[ 22 | 'sphinxsearch', 23 | 'sphinxsearch.backend', 24 | 'sphinxsearch.backend.sphinx', 25 | ], 26 | url='http://github.com/rutube/django_sphinxsearch', 27 | license='Beerware', 28 | author='tumbler', 29 | author_email='zimbler@gmail.com', 30 | description='Sphinxsearch database backend for django>=1.8', 31 | setup_requires=[ 32 | 'Django>=1.8', 33 | mysql 34 | ], 35 | ) 36 | -------------------------------------------------------------------------------- /sphinxsearch/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rutube/django_sphinxsearch/03fdcb485670f6bd462b56866941c6222ea16c69/sphinxsearch/__init__.py -------------------------------------------------------------------------------- /sphinxsearch/backend/__init__.py: -------------------------------------------------------------------------------- 1 | # coding: utf-8 2 | 3 | # $Id: $ 4 | 5 | 6 | -------------------------------------------------------------------------------- /sphinxsearch/backend/sphinx/__init__.py: -------------------------------------------------------------------------------- 1 | # coding: utf-8 2 | 3 | # $Id: $ 4 | 5 | 6 | -------------------------------------------------------------------------------- /sphinxsearch/backend/sphinx/base.py: -------------------------------------------------------------------------------- 1 | # coding: utf-8 2 | 3 | from django.db.backends.mysql import base, creation 4 | from django.db.backends.mysql.base import server_version_re 5 | from django.utils.functional import cached_property 6 | 7 | 8 | class SphinxOperations(base.DatabaseOperations): 9 | 10 | def regex_lookup(self, lookup_type): 11 | raise NotImplementedError() 12 | 13 | compiler_module = "sphinxsearch.backend.sphinx.compiler" 14 | 15 | def fulltext_search_sql(self, field_name): 16 | """ Formats full-text search expression.""" 17 | return 'MATCH (\'@%s "%%s"\')' % field_name 18 | 19 | def force_no_ordering(self): 20 | """ Fix unsupported syntax "ORDER BY NULL".""" 21 | return [] 22 | 23 | 24 | class SphinxValidation(base.DatabaseValidation): 25 | def _check_sql_mode(self, **kwargs): 26 | """ Disable sql_mode validation because it's unsupported 27 | >>> import django.db 28 | >>> cursor = django.db.connection 29 | >>> cursor.execute("SELECT @@sql_mode") 30 | # Error here after parsing searchd response 31 | """ 32 | return [] 33 | 34 | 35 | class SphinxCreation(creation.DatabaseCreation): 36 | 37 | def create_test_db(self, *args, **kwargs): 38 | # NOOP, test using regular sphinx database. 39 | if self.connection.settings_dict.get('TEST_NAME'): 40 | # initialize connection database name 41 | test_name = self.connection.settings_dict['TEST_NAME'] 42 | self.connection.close() 43 | self.connection.settings_dict['NAME'] = test_name 44 | self.connection.cursor() 45 | return test_name 46 | return self.connection.settings_dict['NAME'] 47 | 48 | def destroy_test_db(self, *args, **kwargs): 49 | # NOOP, we created nothing, nothing to destroy. 50 | return 51 | 52 | 53 | class SphinxFeatures(base.DatabaseFeatures): 54 | # The following can be useful for unit testing, with multiple databases 55 | # configured in Django, if one of them does not support transactions, 56 | # Django will fall back to using clear/create 57 | # (instead of begin...rollback) between each test. The method Django 58 | # uses to detect transactions uses CREATE TABLE and DROP TABLE, 59 | # which ARE NOT supported by Sphinx, even though transactions ARE. 60 | # Therefore, we can just set this to True, and Django will use 61 | # transactions for clearing data between tests when all OTHER backends 62 | # support it. 63 | supports_transactions = True 64 | allows_group_by_pk = False 65 | uses_savepoints = False 66 | supports_column_check_constraints = False 67 | is_sql_auto_is_null_enabled = False 68 | 69 | 70 | class DatabaseWrapper(base.DatabaseWrapper): 71 | def __init__(self, *args, **kwargs): 72 | super(DatabaseWrapper, self).__init__(*args, **kwargs) 73 | self.ops = SphinxOperations(self) 74 | self.creation = SphinxCreation(self) 75 | self.features = SphinxFeatures(self) 76 | self.validation = SphinxValidation(self) 77 | 78 | def _start_transaction_under_autocommit(self): 79 | raise NotImplementedError() 80 | 81 | @cached_property 82 | def mysql_version(self): 83 | # Django>=1.10 makes if differently 84 | with self.temporary_connection(): 85 | server_info = self.connection.get_server_info() 86 | match = server_version_re.match(server_info) 87 | if not match: 88 | raise Exception('Unable to determine MySQL version from version ' 89 | 'string %r' % server_info) 90 | return tuple(int(x) for x in match.groups()) 91 | -------------------------------------------------------------------------------- /sphinxsearch/backend/sphinx/compiler.py: -------------------------------------------------------------------------------- 1 | # coding: utf-8 2 | from collections import OrderedDict 3 | import re 4 | from django.core.exceptions import FieldError 5 | from django.db import models 6 | from django.db.models.expressions import Random 7 | from django.db.models.lookups import Search, Exact 8 | from django.db.models.sql import compiler, AND 9 | from django.db.models.sql.constants import ORDER_DIR 10 | from django.db.models.sql.query import get_order_dir 11 | 12 | from django.utils import six 13 | from sphinxsearch import sql as sqls 14 | from sphinxsearch.utils import sphinx_escape 15 | 16 | 17 | class SphinxQLCompiler(compiler.SQLCompiler): 18 | # Options names that are not escaped by compiler. Don't pass user input 19 | # there. 20 | safe_options = ('ranker', 'field_weights', 'index_weights') 21 | 22 | def compile(self, node, select_format=False): 23 | sql, params = super(SphinxQLCompiler, self).compile(node, select_format) 24 | 25 | # substitute MATCH() arguments with sphinx-escaped params 26 | if isinstance(node, Search): 27 | search_text = sphinx_escape(params[0]) 28 | sql = sql % search_text 29 | params = [] 30 | 31 | return sql, params 32 | 33 | def get_order_by(self): 34 | res = super(SphinxQLCompiler, self).get_order_by() 35 | 36 | order_by = [] 37 | for expr, params in res: 38 | if isinstance(expr.expression, Random): 39 | # Replacing ORDER BY RAND() ASC to ORDER BY RAND() 40 | assert params[0] == 'RAND() ASC', "Expected ordering clause" 41 | params = ('RAND()',) + params[1:] 42 | order_by.append((expr, params)) 43 | return order_by 44 | 45 | def get_group_by(self, select, order_by): 46 | res = super(SphinxQLCompiler, self).get_group_by(select, order_by) 47 | 48 | # override GROUP BY columns for sphinxsearch's "GROUP N BY" support 49 | group_by = getattr(self.query, 'group_by', None) 50 | if group_by: 51 | fields = self.query.model._meta.fields 52 | field_columns = [f.column for f in fields if f.attname in group_by] 53 | return [r for r in res if r[0] in field_columns] 54 | 55 | return res 56 | 57 | @staticmethod 58 | def _quote(s, negative=True): 59 | """ Adds quotes and negates to match lookup expressions.""" 60 | prefix = '-' if negative else '' 61 | if s.startswith('"'): 62 | return s 63 | negative = s.startswith('-') 64 | if not negative: 65 | return '"%s"' % s 66 | s = s[1:] 67 | if s.startswith('"'): 68 | return '%s%s' % (prefix, s) 69 | return '%s"%s"' % (prefix, s) 70 | 71 | def _serialize(self, values_list): 72 | """ Serializes list of sphinx MATCH lookup expressions 73 | 74 | :param values_list: list of match lookup expressions 75 | :type values_list: str, list, tuple 76 | :return: MATCH expression 77 | :rtype: str 78 | """"" 79 | if isinstance(values_list, six.string_types): 80 | return values_list 81 | ensure_list = lambda s: [s] if isinstance(s, six.string_types) else s 82 | values_list = [item for s in values_list for item in ensure_list(s)] 83 | positive_list = filter(lambda s: not s.startswith('-'), values_list) 84 | negative_list = filter(lambda s: s.startswith('-'), values_list) 85 | 86 | positive = "|".join(map(self._quote, positive_list)) 87 | if not positive_list: 88 | negative = '|'.join(self._quote(n, negative=False) 89 | for n in negative_list) 90 | template = '%s -(%s)' 91 | else: 92 | negative = ' '.join(map(self._quote, negative_list)) 93 | template = '%s %s' 94 | result = template % (positive.strip(' '), negative.strip(' ')) 95 | return result.strip(' ') 96 | 97 | def as_sql(self, with_limits=True, with_col_aliases=False, subquery=False): 98 | """ Patching final SQL query.""" 99 | where, self.query.where = self.query.where, sqls.SphinxWhereNode() 100 | match = getattr(self.query, 'match', None) 101 | if match: 102 | # add match extra where 103 | self._add_match_extra(match) 104 | 105 | connection = self.connection 106 | 107 | where_sql, where_params = where.as_sql(self, connection) 108 | # moving where conditions to SELECT clause because of better support 109 | # of SQL expressions in sphinxsearch. 110 | 111 | if where_sql: 112 | # Without annotation queryset.count() receives 1 as where_result 113 | # and count it as aggregation result. 114 | self.query.add_annotation( 115 | sqls.SphinxWhereExpression(where_sql, where_params), 116 | '__where_result') 117 | # almost all where conditions are now in SELECT clause, so 118 | # WHERE should contain only test against that conditions are true 119 | self.query.add_extra( 120 | None, None, 121 | ['__where_result = %s'], (True,), None, None) 122 | 123 | sql, args = super(SphinxQLCompiler, self).as_sql(with_limits, 124 | with_col_aliases) 125 | 126 | # empty SQL doesn't need patching 127 | if (sql, args) == ('', ()): 128 | return sql, args 129 | 130 | # removing unsupported by sphinxsearch OFFSET clause 131 | # replacing it with LIMIT , 132 | sql = re.sub(r'LIMIT (\d+) OFFSET (\d+)$', 'LIMIT \\2, \\1', sql) 133 | 134 | # patching GROUP BY clause 135 | group_limit = getattr(self.query, 'group_limit', '') 136 | group_by_ordering = self.get_group_ordering() 137 | if group_limit: 138 | # add GROUP BY expression 139 | group_by = 'GROUP %s BY \\1' % group_limit 140 | else: 141 | group_by = 'GROUP BY \\1' 142 | if group_by_ordering: 143 | # add WITHIN GROUP ORDER BY expression 144 | group_by += group_by_ordering 145 | sql = re.sub(r'GROUP BY ((\w+)(, \w+)*)', group_by, sql) 146 | 147 | # adding sphinxsearch OPTION clause 148 | options = getattr(self.query, 'options', None) 149 | if options: 150 | keys = sorted(options.keys()) 151 | values = [options[k] for k in keys if k not in self.safe_options] 152 | 153 | opts = [] 154 | for k in keys: 155 | if k in self.safe_options: 156 | opts.append("%s=%s" % (k, options[k])) 157 | else: 158 | opts.append("%s=%%s" % k) 159 | sql += ' OPTION %s' % ', '.join(opts) or '' 160 | args += tuple(values) 161 | return sql, args 162 | 163 | def get_group_ordering(self): 164 | """ Returns group ordering clause. 165 | 166 | Formats WITHIN GROUP ORDER BY expression 167 | with columns in query.group_order_by 168 | """ 169 | group_order_by = getattr(self.query, 'group_order_by', ()) 170 | asc, desc = ORDER_DIR['ASC'] 171 | if not group_order_by: 172 | return '' 173 | result = [] 174 | for order_by in group_order_by: 175 | col, order = get_order_dir(order_by, asc) 176 | result.append("%s %s" % (col, order)) 177 | return " WITHIN GROUP ORDER BY " + ", ".join(result) 178 | 179 | def _add_match_extra(self, match): 180 | """ adds MATCH clause to query.where """ 181 | expression = [] 182 | all_field_expr = [] 183 | all_fields_lookup = match.get('*') 184 | 185 | # format expression to MATCH against any indexed fields 186 | if all_fields_lookup: 187 | if isinstance(all_fields_lookup, six.string_types): 188 | expression.append(all_fields_lookup) 189 | all_field_expr.append(all_fields_lookup) 190 | else: 191 | for value in all_fields_lookup: 192 | value_str = self._serialize(value) 193 | expression.append(value_str) 194 | all_field_expr.append(value_str) 195 | 196 | # format expressions to MATCH against concrete fields 197 | for sphinx_attr, lookup in match.items(): 198 | if sphinx_attr == '*': 199 | continue 200 | # noinspection PyProtectedMember 201 | field = self.query.model._meta.get_field(sphinx_attr) 202 | db_column = field.db_column or field.attname 203 | expression.append('@' + db_column) 204 | expression.append("(%s)" % self._serialize(lookup)) 205 | 206 | # handle non-ascii characters in search expressions 207 | decode = lambda _: _.decode("utf-8") if type( 208 | _) is six.binary_type else _ 209 | match_expr = u"MATCH('%s')" % u' '.join(map(decode, expression)) 210 | 211 | # add MATCH() to query.where 212 | self.query.where.add(sqls.SphinxExtraWhere([match_expr], []), AND) 213 | 214 | 215 | # Set SQLCompiler appropriately, so queries will use the correct compiler. 216 | SQLCompiler = SphinxQLCompiler 217 | 218 | 219 | class SQLInsertCompiler(compiler.SQLInsertCompiler, SphinxQLCompiler): 220 | pass 221 | 222 | 223 | class SQLDeleteCompiler(compiler.SQLDeleteCompiler, SphinxQLCompiler): 224 | def as_sql(self): 225 | sql, params = super(SQLDeleteCompiler, self).as_sql() 226 | 227 | # empty SQL doesn't need patching 228 | if (sql, params) == ('', ()): 229 | return sql, params 230 | 231 | sql = re.sub(r'\(IN\((\w+),\s([\w\s\%,]+)\)\)', '\\1 IN (\\2)', sql) 232 | return sql, params 233 | 234 | 235 | class SQLUpdateCompiler(compiler.SQLUpdateCompiler, SphinxQLCompiler): 236 | 237 | # noinspection PyMethodOverriding 238 | def as_sql(self): 239 | node = self.is_single_row_update() 240 | # determine whether use UPDATE (only fixed-length fields) or 241 | # REPLACE (internal delete + insert) syntax 242 | need_replace = False 243 | if node: 244 | need_replace = self._has_string_fields() 245 | if node and need_replace: 246 | sql, args = self.as_replace(node) 247 | else: 248 | 249 | match = getattr(self.query, 'match', None) 250 | if match: 251 | # add match extra where 252 | self._add_match_extra(match) 253 | 254 | sql, args = super(SQLUpdateCompiler, self).as_sql() 255 | return sql, args 256 | 257 | def is_single_row_update(self): 258 | where = self.query.where 259 | match = getattr(self.query, 'match', {}) 260 | node = None 261 | if len(where.children) == 1: 262 | node = where.children[0] 263 | elif match: 264 | meta = self.query.model._meta 265 | pk_match = match.get(meta.pk.attname) 266 | if pk_match is not None: 267 | pk_value = list(pk_match.dict.keys())[0] 268 | return Exact(meta.pk.get_col(meta.db_table), pk_value) 269 | if not isinstance(node, Exact): 270 | node = None 271 | elif not node.lhs.field.primary_key: 272 | node = None 273 | return node 274 | 275 | def as_replace(self, where_node): 276 | """ 277 | Performs single-row UPDATE as REPLACE INTO query. 278 | 279 | Must be used to change string attributes or indexed fields. 280 | """ 281 | 282 | # It's a copy of compiler.SQLUpdateCompiler.as_sql method 283 | # that formats query more like INSERT than UPDATE 284 | self.pre_sql_setup() 285 | if not self.query.values: 286 | return '', () 287 | table = self.query.tables[0] 288 | qn = self.quote_name_unless_alias 289 | result = ['REPLACE INTO %s' % qn(table)] 290 | # noinspection PyProtectedMember 291 | meta = self.query.model._meta 292 | self.query.values.append((meta.pk, self.query.model, where_node.rhs)) 293 | columns, values, update_params = [], [], [] 294 | 295 | for field, model, val in self.query.values: 296 | if hasattr(val, 'resolve_expression'): 297 | val = val.resolve_expression(self.query, allow_joins=False, 298 | for_save=True) 299 | if val.contains_aggregate: 300 | raise FieldError( 301 | "Aggregate functions are not allowed in this query") 302 | elif hasattr(val, 'prepare_database_save'): 303 | if field.rel: 304 | val = field.get_db_prep_save( 305 | val.prepare_database_save(field), 306 | connection=self.connection, 307 | ) 308 | else: 309 | raise TypeError( 310 | "Database is trying to update a relational field " 311 | "of type %s with a value of type %s. Make sure " 312 | "you are setting the correct relations" % 313 | (field.__class__.__name__, val.__class__.__name__)) 314 | else: 315 | val = field.get_db_prep_save(val, connection=self.connection) 316 | 317 | # Getting the placeholder for the field. 318 | if hasattr(field, 'get_placeholder'): 319 | placeholder = field.get_placeholder(val, self, self.connection) 320 | else: 321 | placeholder = '%s' 322 | name = field.column 323 | columns.append(qn(name)) 324 | if hasattr(val, 'as_sql'): 325 | sql, params = self.compile(val) 326 | values.append(sql) 327 | update_params.extend(params) 328 | elif val is not None: 329 | values.append(placeholder) 330 | update_params.append(val) 331 | else: 332 | values.append('NULL') 333 | if not values: 334 | return '', () 335 | result.append('(') 336 | result.append(', '.join(columns)) 337 | result.append(') VALUES (') 338 | result.append(', '.join(values)) 339 | result.append(')') 340 | return ' '.join(result), tuple(update_params) 341 | 342 | def _has_string_fields(self): 343 | """ check whether query is updating text fields.""" 344 | _excluded_update_fields = ( 345 | models.CharField, 346 | models.TextField 347 | ) 348 | for field, model, val in self.query.values: 349 | if isinstance(field, _excluded_update_fields): 350 | return True 351 | return False 352 | 353 | 354 | class SQLAggregateCompiler(compiler.SQLAggregateCompiler, SphinxQLCompiler): 355 | pass 356 | -------------------------------------------------------------------------------- /sphinxsearch/compat.py: -------------------------------------------------------------------------------- 1 | # coding: utf-8 2 | 3 | 4 | import django 5 | 6 | DJ_11 = django.VERSION >= (1, 11, 0) 7 | -------------------------------------------------------------------------------- /sphinxsearch/fields.py: -------------------------------------------------------------------------------- 1 | # coding: utf-8 2 | 3 | # $Id: $ 4 | import datetime 5 | import time 6 | 7 | from sphinxsearch.lookups import sphinx_lookups 8 | from django.core import exceptions 9 | from django.db import models 10 | from django.utils import six 11 | 12 | 13 | class SphinxField(models.TextField): 14 | """ Non-selectable indexed string field 15 | 16 | In sphinxsearch config terms, sql_field_string or rt_field. 17 | """ 18 | class_lookups = sphinx_lookups.copy() 19 | 20 | 21 | class SphinxDateTimeField(models.FloatField): 22 | """ Sphinx timestamp field for sql_attr_timestamp and rt_attr_timestamp. 23 | 24 | NB: sphinxsearch doens't store microseconds, if necessary, describe 25 | field as sql_attr_float in config. 26 | """ 27 | 28 | def get_prep_value(self, value): 29 | if isinstance(value, (datetime.datetime, datetime.date)): 30 | return time.mktime(value.timetuple()) 31 | elif isinstance(value, six.integer_types + (float,)): 32 | return value 33 | else: 34 | raise ValueError("Invalid value for UNIX_TIMESTAMP") 35 | 36 | def from_db_value(self, value, expression, connection, context): 37 | return datetime.datetime.fromtimestamp(value) 38 | 39 | 40 | class SphinxIntegerField(models.IntegerField): 41 | class_lookups = sphinx_lookups.copy() 42 | 43 | 44 | class SphinxBigIntegerField(models.BigIntegerField): 45 | class_lookups = sphinx_lookups.copy() 46 | 47 | 48 | class SphinxMultiField(models.IntegerField): 49 | class_lookups = sphinx_lookups.copy() 50 | 51 | def get_prep_value(self, value): 52 | if value is None: 53 | return None 54 | if isinstance(value, six.integer_types): 55 | return value 56 | return [super(SphinxMultiField, self).get_prep_value(v) for v in value] 57 | 58 | def from_db_value(self, value, expression, connection, context): 59 | if value is None: 60 | return value 61 | if value == '': 62 | return [] 63 | try: 64 | return list(map(int, value.split(','))) 65 | except (TypeError, ValueError): 66 | raise exceptions.ValidationError( 67 | self.error_messages['invalid'], 68 | code='invalid', 69 | params={'value': value}, 70 | ) 71 | 72 | def to_python(self, value): 73 | if value is None: 74 | return value 75 | try: 76 | return list(map(int, value.split(','))) 77 | except (TypeError, ValueError): 78 | raise exceptions.ValidationError( 79 | self.error_messages['invalid'], 80 | code='invalid', 81 | params={'value': value}, 82 | ) 83 | 84 | 85 | class SphinxMulti64Field(SphinxMultiField): 86 | pass -------------------------------------------------------------------------------- /sphinxsearch/lookups.py: -------------------------------------------------------------------------------- 1 | # coding: utf-8 2 | 3 | from django.db.models.lookups import In 4 | from django.db.models.fields import Field 5 | 6 | sphinx_lookups = Field.class_lookups.copy() 7 | 8 | 9 | class SphinxIn(In): 10 | def as_sql(self, compiler, connection): 11 | lhs, lhs_params = self.process_lhs(compiler, connection) 12 | rhs, rhs_params = self.batch_process_rhs(compiler, connection) 13 | rhs_sql = ', '.join(['%s' for _ in range(len(rhs_params))]) 14 | return '(IN(%s, %s))' % (lhs, rhs_sql), rhs_params 15 | 16 | 17 | sphinx_lookups['in'] = SphinxIn 18 | -------------------------------------------------------------------------------- /sphinxsearch/models.py: -------------------------------------------------------------------------------- 1 | # coding: utf-8 2 | from copy import copy 3 | 4 | from django.conf import settings 5 | from django.db import connections 6 | from django.db.models import QuerySet 7 | from django.db.models.expressions import RawSQL 8 | 9 | from sphinxsearch import sql, compat 10 | from sphinxsearch.fields import * 11 | from sphinxsearch.utils import sphinx_escape 12 | 13 | 14 | class SphinxQuerySet(QuerySet): 15 | def __init__(self, model, **kwargs): 16 | kwargs.setdefault('query', sql.SphinxQuery(model)) 17 | super(SphinxQuerySet, self).__init__(model, **kwargs) 18 | 19 | def _filter_or_exclude(self, negate, *args, **kwargs): 20 | args = list(args) 21 | kwargs = copy(kwargs) 22 | for key, value in list(kwargs.items()): 23 | field, lookup = self.__get_field_lookup(key) 24 | if self.__check_search_lookup(field, lookup, value): 25 | kwargs.pop(key, None) 26 | elif self.__check_in_lookup(field, lookup, value, negate): 27 | kwargs.pop(key, None) 28 | elif self.__check_sphinx_field_exact(field, lookup, value, negate): 29 | kwargs.pop(key, None) 30 | elif self.__check_mva_field_lookup(field, lookup, value, negate): 31 | kwargs.pop(key, None) 32 | pass 33 | 34 | return super(SphinxQuerySet, self)._filter_or_exclude(negate, *args, 35 | **kwargs) 36 | 37 | def __get_field_lookup(self, key): 38 | tokens = key.split('__') 39 | if len(tokens) == 1: 40 | field_name, lookup = tokens[0], 'exact' 41 | elif len(tokens) == 2: 42 | field_name, lookup = tokens 43 | else: 44 | raise ValueError("Nested field lookup found") 45 | if field_name == 'pk': 46 | field = self.model._meta.pk 47 | else: 48 | field = self.model._meta.get_field(field_name) 49 | return field, lookup 50 | 51 | def _negate_expression(self, negate, lookup): 52 | if isinstance(lookup, (tuple, list)): 53 | result = [] 54 | for v in lookup: 55 | result.append(self._negate_expression(negate, v)) 56 | return result 57 | else: 58 | if not isinstance(lookup, six.string_types): 59 | lookup = six.text_type(lookup) 60 | 61 | if not lookup.startswith('"'): 62 | lookup = '"%s"' % lookup 63 | if negate: 64 | lookup = '-%s' % lookup 65 | return lookup 66 | 67 | def match(self, *args, **kwargs): 68 | """ Enables full-text searching in sphinx (MATCH expression). 69 | 70 | qs.match('sphinx_expression_1', 'sphinx_expression_2') 71 | compiles to 72 | MATCH('sphinx_expression_1 sphinx_expression_2) 73 | 74 | qs.match(field1='sphinx_loopup1',field2='sphinx_loopup2') 75 | compiles to 76 | MATCH('@field1 sphinx_lookup1 @field2 sphinx_lookup2') 77 | """ 78 | qs = self._clone() 79 | qs.query.add_match(*args, **kwargs) 80 | return qs 81 | 82 | def options(self, **kw): 83 | """ Setup OPTION clause for query.""" 84 | qs = self._clone() 85 | try: 86 | qs.query.options.update(kw) 87 | except AttributeError: 88 | qs.query.options = kw 89 | return qs 90 | 91 | def with_meta(self): 92 | """ Force call SHOW META after fetching queryset data from searchd.""" 93 | qs = self._clone() 94 | qs.query.with_meta = True 95 | return qs 96 | 97 | def group_by(self, *args, **kwargs): 98 | """ 99 | :param args: list of fields to group by 100 | :type args: list-like 101 | 102 | Keyword params: 103 | :param group_limit: (GROUP BY), limits number of group members to N 104 | :type group_limit: int 105 | :param group_order_by: (WITHIN GROUP ORDER BY), sort order within group 106 | :type group_order_by: list-like 107 | :return: new queryset with grouping 108 | :rtype: SphinxQuerySet 109 | """ 110 | group_limit = kwargs.get('group_limit', 0) 111 | group_order_by = kwargs.get('group_order_by', ()) 112 | 113 | if not isinstance(group_order_by, (list, tuple)): 114 | group_order_by = [group_order_by] 115 | 116 | def fix_arg_name(group_arg): 117 | if group_arg.startswith('-'): 118 | negate = True 119 | group_arg = group_arg[1:] 120 | # if group_arg isn't name of db_column, lets fix it 121 | try: 122 | fld = self.model._meta.get_field(group_arg) 123 | group_arg = fld.column 124 | except: 125 | pass 126 | else: 127 | negate = False 128 | 129 | if negate: 130 | group_arg = '-%s' % group_arg 131 | return group_arg 132 | 133 | group_order_by = list(map(fix_arg_name, group_order_by)) 134 | 135 | qs = self._clone() 136 | qs.query.group_by = qs.query.group_by or [] 137 | for field_name in args: 138 | if field_name not in qs.query.extra_select: 139 | field = self.model._meta.get_field(field_name) 140 | qs.query.group_by.append(field.attname) 141 | else: 142 | qs.query.group_by.append(RawSQL(field_name, [])) 143 | qs.query.group_limit = group_limit 144 | qs.query.group_order_by = group_order_by 145 | return qs 146 | 147 | def __check_mva_field_lookup(self, field, lookup, value, negate): 148 | """ Replaces some MVA field lookups with valid sphinx expressions.""" 149 | 150 | if not isinstance(field, (SphinxMultiField, SphinxMulti64Field)): 151 | return False 152 | 153 | transforms = { 154 | 'exact': "IN(%s, %%s)", 155 | 'gte': "LEAST(%s) >= %%s", 156 | 'ge': "LEAST(%s) > %%s", 157 | 'lt': "GREATEST(%s) < %%s", 158 | 'lte': "GREATEST(%s) <= %%s" 159 | } 160 | 161 | if lookup in transforms.keys(): 162 | tpl = transforms[lookup] 163 | condition = tpl % field.column 164 | if negate: 165 | condition = "NOT (%s)" % condition 166 | self.query.add_extra(None, None, [condition], [value], None, None) 167 | return True 168 | else: 169 | raise ValueError("Invalid lookup for MVA: %s" % lookup) 170 | 171 | def __check_search_lookup(self, field, lookup, value): 172 | """ Replaces field__search lookup with MATCH() call.""" 173 | if lookup != 'search': 174 | return False 175 | self.query.add_match(**{field.name: sphinx_escape(value)}) 176 | return True 177 | 178 | def __check_in_lookup(self, field, lookup, value, negate): 179 | if lookup != 'in': 180 | return False 181 | if not isinstance(value, (tuple, list)): 182 | value = [value] 183 | placeholders = ', '.join(['%s'] * len(value)) 184 | condition = "IN(%s, %s)" % (field.column, placeholders) 185 | value = [field.get_prep_value(v) for v in value] 186 | if negate: 187 | condition = "NOT (%s)" % condition 188 | self.query.add_extra(None, None, [condition], value, None, None) 189 | return True 190 | 191 | def __check_sphinx_field_exact(self, field, lookup, value, negate): 192 | if not isinstance(field, SphinxField): 193 | return False 194 | if lookup != 'exact': 195 | raise ValueError("Unsupported lookup for SphinxField") 196 | if negate: 197 | value = '-%s' % value 198 | self.query.add_match(**{field.name: value}) 199 | return True 200 | 201 | def _fetch_meta(self): 202 | c = connections[settings.SPHINX_DATABASE_NAME].cursor() 203 | try: 204 | c.execute("SHOW META") 205 | self.meta = dict([c.fetchone()]) 206 | except UnicodeDecodeError: 207 | self.meta = {} 208 | finally: 209 | c.close() 210 | 211 | def iterator(self): 212 | for row in super(SphinxQuerySet, self).iterator(): 213 | yield row 214 | if getattr(self.query, 'with_meta', False): 215 | self._fetch_meta() 216 | 217 | if compat.DJ_11: 218 | # Django-1.11 does not use iterator() call when materializing, so 219 | # with_meta() should be handled separately. 220 | 221 | def _fetch_all(self): 222 | super(SphinxQuerySet, self)._fetch_all() 223 | if getattr(self.query, 'with_meta', False): 224 | self._fetch_meta() 225 | 226 | 227 | class SphinxManager(models.Manager): 228 | use_for_related_fields = True 229 | 230 | def get_queryset(self): 231 | """ Creates new queryset for model. 232 | 233 | :return: model queryset 234 | :rtype: SphinxQuerySet 235 | """ 236 | 237 | # Determine which fields are sphinx fields (full-text data) and 238 | # defer loading them. Sphinx won't return them. 239 | # TODO: we probably need a way to keep these from being loaded 240 | # later if the attr is accessed. 241 | sphinx_fields = [field.name for field in self.model._meta.fields 242 | if isinstance(field, SphinxField)] 243 | 244 | return SphinxQuerySet(self.model).defer(*sphinx_fields) 245 | 246 | def options(self, **kw): 247 | return self.get_queryset().options(**kw) 248 | 249 | def match(self, expression): 250 | return self.get_queryset().match(expression) 251 | 252 | def group_by(self, *args, **kw): 253 | return self.get_queryset().group_by(*args, **kw) 254 | 255 | def get(self, *args, **kw): 256 | return self.get_queryset().get(*args, **kw) 257 | 258 | 259 | class SphinxModel(six.with_metaclass(sql.SphinxModelBase, models.Model)): 260 | class Meta: 261 | abstract = True 262 | 263 | objects = SphinxManager() 264 | 265 | _excluded_update_fields = ( 266 | models.CharField, 267 | models.TextField 268 | ) 269 | -------------------------------------------------------------------------------- /sphinxsearch/routers.py: -------------------------------------------------------------------------------- 1 | # coding: utf-8 2 | 3 | # $Id: $ 4 | 5 | 6 | from django.conf import settings 7 | 8 | 9 | class SphinxRouter(object): 10 | """ 11 | Routes database operations for Sphinx model to the sphinx database connection. 12 | """ 13 | 14 | def is_sphinx_model(self, model_or_obj): 15 | from sphinxsearch.models import SphinxModel 16 | from sphinxsearch.sql import SphinxModelBase 17 | if type(model_or_obj) is not SphinxModelBase: 18 | model = model_or_obj.__class__ 19 | else: 20 | model = model_or_obj 21 | is_sphinx_model = issubclass(model, SphinxModel) or type(model) is SphinxModelBase 22 | return is_sphinx_model 23 | 24 | def db_for_read(self, model, **kwargs): 25 | if self.is_sphinx_model(model): 26 | return getattr(settings, 'SPHINX_DATABASE_NAME', 'sphinx') 27 | 28 | def db_for_write(self, model, **kwargs): 29 | if self.is_sphinx_model(model): 30 | return getattr(settings, 'SPHINX_DATABASE_NAME', 'sphinx') 31 | 32 | def allow_relation(self, obj1, obj2, **kwargs): 33 | # Allow all relations... 34 | return True 35 | -------------------------------------------------------------------------------- /sphinxsearch/sql.py: -------------------------------------------------------------------------------- 1 | # coding: utf-8 2 | 3 | # $Id: $ 4 | from collections import OrderedDict 5 | import functools 6 | from django.db import models 7 | from django.db.models import Count, BooleanField 8 | from django.db.models.base import ModelBase 9 | from django.db.models.expressions import Col, Func, BaseExpression 10 | from django.db.models.sql import Query 11 | from django.db.models.sql.where import WhereNode, ExtraWhere 12 | from django.utils.datastructures import OrderedSet 13 | 14 | 15 | class SphinxCount(Count): 16 | """ Replaces Mysql-like COUNT('*') with COUNT(*) token.""" 17 | template = '%(function)s(*)' 18 | 19 | def as_sql(self, compiler, connection, function=None, template=None): 20 | sql, params = super(SphinxCount, self).as_sql( 21 | compiler, connection, function=function, template=template) 22 | try: 23 | params.remove('*') 24 | except ValueError: 25 | pass 26 | return sql, params 27 | 28 | 29 | class SphinxWhereExpression(BaseExpression): 30 | def __init__(self, where, where_params): 31 | self.where = where 32 | self.where_params = where_params 33 | super(SphinxWhereExpression, self).__init__(output_field=BooleanField()) 34 | 35 | def as_sql(self, compiler, connection): 36 | return self.where, self.where_params 37 | 38 | 39 | class SphinxExtraWhere(ExtraWhere): 40 | 41 | def as_sql(self, qn=None, connection=None): 42 | sqls = ["%s" % sql for sql in self.sqls] 43 | return " AND ".join(sqls), tuple(self.params or ()) 44 | 45 | 46 | class SphinxWhereNode(WhereNode): 47 | 48 | def make_atom(self, child, qn, connection): 49 | """ 50 | Transform search, the keyword should not be quoted. 51 | """ 52 | return super(WhereNode, self).make_atom(child, qn, connection) 53 | lvalue, lookup_type, value_annot, params_or_value = child 54 | sql, params = super(SphinxWhereNode, self).make_atom(child, qn, connection) 55 | if lookup_type == 'search': 56 | if hasattr(lvalue, 'process'): 57 | try: 58 | lvalue, params = lvalue.process(lookup_type, params_or_value, connection) 59 | except EmptyShortCircuit: 60 | raise EmptyResultSet 61 | if isinstance(lvalue, tuple): 62 | # A direct database column lookup. 63 | field_sql = self.sql_for_columns(lvalue, qn, connection) 64 | else: 65 | # A smart object with an as_sql() method. 66 | field_sql = lvalue.as_sql(qn, connection) 67 | # TODO: There are a couple problems here. 68 | # 1. The user _might_ want to search only a specific field. 69 | # 2. However, since Django requires a field name to use the __search operator 70 | # There is no way to do a search in _all_ fields. 71 | # 3. Because, using multiple __search operators is not supported. 72 | # So, we need to merge multiped __search operators into a single MATCH(), we 73 | # can't do that here, we have to do that one level up... 74 | # Ignore the field name, search all fields: 75 | params = ('@* %s' % params[0], ) 76 | # _OR_ respect the field name, and search on it: 77 | #params = ('@%s %s' % (field_sql, params[0]), ) 78 | if self._real_negated: 79 | col = lvalue.col 80 | if lookup_type == 'exact': 81 | sql = '%s <> %%s' % col 82 | elif lookup_type == 'in': 83 | params_placeholder = '(%s)' % (', '.join(['%s'] * len(params))) 84 | sql = '%s NOT IN %s' % (col, params_placeholder) 85 | else: 86 | raise ValueError("Negative '%s' lookup not supported" % lookup_type) 87 | return sql, params 88 | 89 | 90 | class SphinxQuery(Query): 91 | _clonable = ('options', 'match', 'group_limit', 'group_order_by', 92 | 'with_meta') 93 | 94 | def __init__(self, *args, **kwargs): 95 | kwargs.setdefault('where', SphinxWhereNode) 96 | super(SphinxQuery, self).__init__(*args, **kwargs) 97 | 98 | def clone(self, klass=None, memo=None, **kwargs): 99 | query = super(SphinxQuery, self).clone(klass=klass, memo=memo, **kwargs) 100 | for attr_name in self._clonable: 101 | value = getattr(self, attr_name, None) 102 | if value: 103 | setattr(query, attr_name, value) 104 | return query 105 | 106 | def add_match(self, *args, **kwargs): 107 | if not hasattr(self, 'match'): 108 | self.match = OrderedDict() 109 | for expression in args: 110 | self.match.setdefault('*', OrderedSet()) 111 | if isinstance(expression, (list, tuple)): 112 | self.match['*'].update(expression) 113 | else: 114 | self.match['*'].add(expression) 115 | for field, expression in kwargs.items(): 116 | self.match.setdefault(field, OrderedSet()) 117 | if isinstance(expression, (list, tuple, set)): 118 | self.match[field].update(expression) 119 | else: 120 | self.match[field].add(expression) 121 | 122 | def get_count(self, using): 123 | """ 124 | Performs a COUNT() query using the current filter constraints. 125 | """ 126 | obj = self.clone() 127 | obj.add_annotation(SphinxCount('*'), alias='__count', is_summary=True) 128 | number = obj.get_aggregation(using, ['__count'])['__count'] 129 | if number is None: 130 | number = 0 131 | return number 132 | 133 | 134 | class SphinxCol(Col): 135 | def as_sql(self, compiler, connection): 136 | # As column names in SphinxQL couldn't be escaped with `backticks`, 137 | # simply return column name 138 | return self.target.column, [] 139 | 140 | 141 | class SphinxModelBase(ModelBase): 142 | 143 | def __new__(cls, name, bases, attrs): 144 | # Each field must be monkey-patched with SphinxCol class to prevent 145 | # `tablename`.`attr` appearing in SQL 146 | for attr in attrs.values(): 147 | if isinstance(attr, models.Field): 148 | col_patched = getattr(attr, '_col_patched', False) 149 | if not col_patched: 150 | cls.patch_col_class(attr) 151 | 152 | new_class = super(SphinxModelBase, cls).__new__(cls, name, bases, attrs) 153 | 154 | # if have overriden primary key, it should be the first local field, 155 | # because of JSONField feature at jsonfield.subclassing.Creator.__set__ 156 | local_fields = new_class._meta.local_fields 157 | try: 158 | pk_idx = local_fields.index(new_class._meta.pk) 159 | if pk_idx > 0: 160 | local_fields.insert(0, local_fields.pop(pk_idx)) 161 | except ValueError: 162 | pass 163 | 164 | return new_class 165 | 166 | def add_to_class(cls, name, value): 167 | col_patched = getattr(value, '_col_patched', False) 168 | if not col_patched and isinstance(value, models.Field): 169 | cls.patch_col_class(value) 170 | super(SphinxModelBase, cls).add_to_class(name, value) 171 | 172 | @classmethod 173 | def patch_col_class(cls, field): 174 | @functools.wraps(field.get_col) 175 | def wrapper(alias, output_field=None): 176 | col = models.Field.get_col(field, alias, output_field=output_field) 177 | col.__class__ = SphinxCol 178 | return col 179 | field.get_col = wrapper 180 | -------------------------------------------------------------------------------- /sphinxsearch/utils.py: -------------------------------------------------------------------------------- 1 | # coding: utf-8 2 | 3 | # $Id: $ 4 | import re 5 | 6 | from django.utils import six 7 | 8 | 9 | def sphinx_escape(value): 10 | """ Escapes SphinxQL search expressions. """ 11 | 12 | if not isinstance(value, six.string_types): 13 | return value 14 | 15 | value = re.sub(r"([=<>()|!@~&/^$\-\'\"\\])", r'\\\\\\\1', value) 16 | value = re.sub(r'\b(SENTENCE|PARAGRAPH)\b', r'\\\\\\\1', value) 17 | return value 18 | -------------------------------------------------------------------------------- /test-requires.txt: -------------------------------------------------------------------------------- 1 | six==1.11.0 2 | PyMySQL==0.7.11 3 | jsonfield==2.0.2 4 | -------------------------------------------------------------------------------- /test_config/sphinx.conf: -------------------------------------------------------------------------------- 1 | index testapp_testmodel 2 | { 3 | type = rt 4 | path = /tmp/testmodel 5 | 6 | rt_field = sphinx_field 7 | rt_field = other_field 8 | rt_attr_uint = attr_uint_ 9 | rt_attr_bool = attr_bool 10 | rt_attr_bigint = attr_bigint 11 | rt_attr_float = attr_float 12 | rt_attr_multi = attr_multi 13 | rt_attr_multi_64 = attr_multi_64 14 | rt_attr_timestamp = attr_timestamp 15 | rt_attr_string = attr_string 16 | rt_attr_json = attr_json 17 | 18 | index_sp = 1 19 | html_strip = 1 20 | } 21 | 22 | index testapp_testmodel_aliased 23 | { 24 | type = rt 25 | path = /tmp/testmodel_aliased 26 | 27 | rt_field = _sphinx_field 28 | rt_field = _other_field 29 | rt_attr_uint = _attr_uint_ 30 | rt_attr_bool = _attr_bool 31 | rt_attr_bigint = _attr_bigint 32 | rt_attr_float = _attr_float 33 | rt_attr_multi = _attr_multi 34 | rt_attr_multi_64 = _attr_multi_64 35 | rt_attr_timestamp = _attr_timestamp 36 | rt_attr_string = _attr_string 37 | rt_attr_json = _attr_json 38 | } 39 | 40 | index testapp_overridensphinxmodel 41 | { 42 | type = rt 43 | path = /tmp/testoverridenmodel 44 | 45 | rt_field = sphinx_field 46 | rt_field = other_field 47 | rt_attr_uint = attr_uint_ 48 | rt_attr_bool = attr_bool 49 | rt_attr_bigint = attr_bigint 50 | rt_attr_float = attr_float 51 | rt_attr_multi = attr_multi 52 | rt_attr_multi_64 = attr_multi_64 53 | rt_attr_timestamp = attr_timestamp 54 | rt_attr_string = attr_string 55 | rt_attr_json = attr_json 56 | } 57 | 58 | index testapp_charpkmodel 59 | { 60 | type = rt 61 | path = /tmp/charpk 62 | 63 | rt_field = docid 64 | rt_field = sphinx_field 65 | rt_field = other_field 66 | rt_attr_string = docid 67 | rt_attr_uint = attr_uint_ 68 | rt_attr_bool = attr_bool 69 | rt_attr_bigint = attr_bigint 70 | rt_attr_float = attr_float 71 | rt_attr_multi = attr_multi 72 | rt_attr_multi_64 = attr_multi_64 73 | rt_attr_timestamp = attr_timestamp 74 | rt_attr_string = attr_string 75 | rt_attr_json = attr_json 76 | } 77 | 78 | ############################################################################# 79 | ## indexer settings 80 | ############################################################################# 81 | 82 | indexer 83 | { 84 | mem_limit = 128M 85 | } 86 | 87 | ############################################################################# 88 | ## searchd settings 89 | ############################################################################# 90 | 91 | searchd 92 | { 93 | listen = localhost:9307:mysql41 94 | log = /tmp/searchd.log 95 | query_log = /tmp/searchd_query.log 96 | query_log_format = sphinxql 97 | read_timeout = 5 98 | client_timeout = 300 99 | max_children = 30 100 | persistent_connections_limit = 30 101 | pid_file = /tmp/searchd.pid 102 | seamless_rotate = 1 103 | preopen_indexes = 1 104 | unlink_old = 1 105 | workers = threads # for RT to work 106 | binlog_path = # disable logging 107 | } 108 | 109 | ############################################################################# 110 | ## common settings 111 | ############################################################################# 112 | 113 | common 114 | { 115 | 116 | } 117 | 118 | # --eof-- 119 | -------------------------------------------------------------------------------- /testproject/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rutube/django_sphinxsearch/03fdcb485670f6bd462b56866941c6222ea16c69/testproject/__init__.py -------------------------------------------------------------------------------- /testproject/settings.py: -------------------------------------------------------------------------------- 1 | """ 2 | Django settings for django_sphinxsearch project. 3 | 4 | Generated by 'django-admin startproject' using Django 1.8.3. 5 | 6 | For more information on this file, see 7 | https://docs.djangoproject.com/en/1.8/topics/settings/ 8 | 9 | For the full list of settings and their values, see 10 | https://docs.djangoproject.com/en/1.8/ref/settings/ 11 | """ 12 | 13 | # Build paths inside the project like this: os.path.join(BASE_DIR, ...) 14 | import os 15 | 16 | BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) 17 | 18 | 19 | # Quick-start development settings - unsuitable for production 20 | # See https://docs.djangoproject.com/en/1.8/howto/deployment/checklist/ 21 | 22 | # SECURITY WARNING: keep the secret key used in production secret! 23 | SECRET_KEY = 'd*-^2xvr#^1ocw=4n=4%j*)7)b$eq$jdat+&p8e71^d&0zf_2!' 24 | 25 | # SECURITY WARNING: don't run with debug turned on in production! 26 | DEBUG = True 27 | 28 | ALLOWED_HOSTS = [] 29 | 30 | 31 | # Application definition 32 | 33 | INSTALLED_APPS = ( 34 | 'django.contrib.auth', 35 | 'django.contrib.contenttypes', 36 | 'django.contrib.sessions', 37 | 'django.contrib.messages', 38 | 'django.contrib.staticfiles', 39 | 'sphinxsearch', 40 | 'testapp', 41 | ) 42 | 43 | MIDDLEWARE_CLASSES = ( 44 | 'django.contrib.sessions.middleware.SessionMiddleware', 45 | 'django.middleware.common.CommonMiddleware', 46 | 'django.middleware.csrf.CsrfViewMiddleware', 47 | 'django.contrib.auth.middleware.AuthenticationMiddleware', 48 | 'django.contrib.auth.middleware.SessionAuthenticationMiddleware', 49 | 'django.contrib.messages.middleware.MessageMiddleware', 50 | 'django.middleware.clickjacking.XFrameOptionsMiddleware', 51 | 'django.middleware.security.SecurityMiddleware', 52 | ) 53 | 54 | ROOT_URLCONF = 'testproject.urls' 55 | 56 | TEMPLATES = [ 57 | { 58 | 'BACKEND': 'django.template.backends.django.DjangoTemplates', 59 | 'DIRS': [], 60 | 'APP_DIRS': True, 61 | 'OPTIONS': { 62 | 'context_processors': [ 63 | 'django.template.context_processors.debug', 64 | 'django.template.context_processors.request', 65 | 'django.contrib.auth.context_processors.auth', 66 | 'django.contrib.messages.context_processors.messages', 67 | ], 68 | }, 69 | }, 70 | ] 71 | 72 | WSGI_APPLICATION = 'testproject.wsgi.application' 73 | 74 | 75 | # Database 76 | # https://docs.djangoproject.com/en/1.8/ref/settings/#databases 77 | 78 | SPHINX_DATABASE_NAME = 'default' 79 | 80 | import pymysql 81 | pymysql.install_as_MySQLdb() 82 | 83 | DATABASES = { 84 | SPHINX_DATABASE_NAME: { 85 | 'ENGINE': 'sphinxsearch.backend.sphinx', 86 | 'HOST': '127.0.0.1', 87 | 'PORT': 9307, 88 | } 89 | } 90 | 91 | DATABASE_ROUTERS = ['sphinxsearch.routers.SphinxRouter'] 92 | 93 | 94 | # Internationalization 95 | # https://docs.djangoproject.com/en/1.8/topics/i18n/ 96 | 97 | LANGUAGE_CODE = 'en-us' 98 | 99 | TIME_ZONE = 'UTC' 100 | 101 | USE_I18N = True 102 | 103 | USE_L10N = True 104 | 105 | USE_TZ = True 106 | 107 | 108 | # Static files (CSS, JavaScript, Images) 109 | # https://docs.djangoproject.com/en/1.8/howto/static-files/ 110 | 111 | STATIC_URL = '/static/' 112 | -------------------------------------------------------------------------------- /testproject/testapp/__init__.py: -------------------------------------------------------------------------------- 1 | # coding: utf-8 2 | 3 | # $Id: $ 4 | 5 | 6 | -------------------------------------------------------------------------------- /testproject/testapp/models.py: -------------------------------------------------------------------------------- 1 | # coding: utf-8 2 | 3 | # $Id: $ 4 | import json 5 | 6 | import six 7 | from datetime import datetime 8 | from django.db import models 9 | 10 | from jsonfield.fields import JSONField 11 | 12 | from sphinxsearch import sql 13 | from sphinxsearch import models as spx_models 14 | 15 | 16 | class Django10CompatJSONField(JSONField): 17 | 18 | def from_db_value(self, value, expression, connection, context): 19 | # In Django-1.10 python value is loaded in this method 20 | if value is None: 21 | return None 22 | return json.loads(value) 23 | 24 | 25 | class FieldMixin(spx_models.SphinxModel): 26 | class Meta: 27 | abstract = True 28 | sphinx_field = spx_models.SphinxField(default='') 29 | other_field = spx_models.SphinxField(default='') 30 | attr_uint = spx_models.SphinxIntegerField(default=0, db_column='attr_uint_') 31 | attr_bigint = spx_models.SphinxBigIntegerField(default=0) 32 | attr_float = models.FloatField(default=0.0) 33 | attr_timestamp = spx_models.SphinxDateTimeField(default=datetime.now) 34 | attr_string = models.CharField(max_length=32, default='') 35 | attr_multi = spx_models.SphinxMultiField(default=[]) 36 | attr_multi_64 = spx_models.SphinxMulti64Field(default=[]) 37 | attr_json = Django10CompatJSONField(default={}) 38 | attr_bool = models.BooleanField(default=False) 39 | 40 | 41 | class TestModel(FieldMixin, spx_models.SphinxModel): 42 | pass 43 | 44 | 45 | class DefaultDjangoModel(models.Model): 46 | pass 47 | 48 | 49 | class OverridenSphinxModel(six.with_metaclass(sql.SphinxModelBase, models.Model)): 50 | class Meta: 51 | managed = False 52 | 53 | _excluded_update_fields = ( 54 | models.CharField, 55 | models.TextField 56 | ) 57 | 58 | objects = spx_models.SphinxManager() 59 | 60 | sphinx_field = spx_models.SphinxField(default='') 61 | other_field = spx_models.SphinxField(default='') 62 | attr_uint = spx_models.SphinxIntegerField(default=0, db_column='attr_uint_') 63 | attr_bigint = spx_models.SphinxBigIntegerField(default=0) 64 | attr_float = models.FloatField(default=0.0) 65 | attr_timestamp = spx_models.SphinxDateTimeField(default=datetime.now) 66 | attr_string = models.CharField(max_length=32, default='') 67 | attr_multi = spx_models.SphinxMultiField(default=[]) 68 | attr_multi_64 = spx_models.SphinxMulti64Field(default=[]) 69 | attr_json = Django10CompatJSONField(default={}) 70 | attr_bool = models.BooleanField(default=False) 71 | 72 | 73 | class ForcedPKModel(FieldMixin, spx_models.SphinxModel): 74 | 75 | class Meta: 76 | db_table = 'testapp_testmodel' 77 | 78 | id = models.BigIntegerField(primary_key=True) 79 | 80 | 81 | class ModelWithAllDbColumnFields(spx_models.SphinxModel): 82 | class Meta: 83 | db_table = 'testapp_testmodel_aliased' 84 | 85 | sphinx_field = spx_models.SphinxField(default='', db_column='_sphinx_field') 86 | other_field = spx_models.SphinxField(default='', db_column='_other_field') 87 | attr_uint = spx_models.SphinxIntegerField(default=0, db_column='_attr_uint_') 88 | attr_bigint = spx_models.SphinxBigIntegerField(default=0, db_column='_attr_bigint') 89 | attr_float = models.FloatField(default=0.0, db_column='_attr_float') 90 | attr_timestamp = spx_models.SphinxDateTimeField(default=datetime.now, 91 | db_column='_attr_timestamp') 92 | attr_string = models.CharField(max_length=32, default='', 93 | db_column='_attr_string') 94 | 95 | attr_multi = spx_models.SphinxMultiField(default=[], 96 | db_column='_attr_multi') 97 | attr_multi_64 = spx_models.SphinxMulti64Field(default=[], 98 | db_column='_attr_multi_64') 99 | attr_json = Django10CompatJSONField(default={}, db_column='_attr_json') 100 | attr_bool = models.BooleanField(default=False, db_column='_attr_bool') 101 | 102 | 103 | class CharPKModel(FieldMixin, spx_models.SphinxModel): 104 | 105 | docid = spx_models.SphinxField(primary_key=True) 106 | id = spx_models.SphinxBigIntegerField(unique=True) 107 | -------------------------------------------------------------------------------- /testproject/testapp/tests.py: -------------------------------------------------------------------------------- 1 | # coding: utf-8 2 | 3 | # $Id: $ 4 | import sys 5 | from datetime import datetime, timedelta 6 | 7 | from django.conf import settings 8 | from django.db import connections 9 | from django.db.models import Sum, Q 10 | from django.db.utils import ProgrammingError 11 | from django.test import TransactionTestCase 12 | from django.test.utils import CaptureQueriesContext 13 | from unittest import expectedFailure 14 | 15 | from sphinxsearch.utils import sphinx_escape 16 | from testapp import models 17 | from sphinxsearch.routers import SphinxRouter 18 | 19 | 20 | class SphinxModelTestCaseBase(TransactionTestCase): 21 | _id = 0 22 | 23 | model = models.TestModel 24 | 25 | def _fixture_teardown(self): 26 | # Prevent SHOW FULL TABLES call 27 | pass 28 | 29 | def truncate_model(self): 30 | c = connections[settings.SPHINX_DATABASE_NAME].cursor() 31 | c.execute("TRUNCATE RTINDEX %s" % self.model._meta.db_table) 32 | c.close() 33 | 34 | def setUp(self): 35 | c = connections[settings.SPHINX_DATABASE_NAME] 36 | self.no_string_compare = c.mysql_version < (2, 2, 7) 37 | self.truncate_model() 38 | self.now = datetime.now().replace(microsecond=0) 39 | self.defaults = self.get_model_defaults() 40 | self.spx_queries = CaptureQueriesContext( 41 | connections[settings.SPHINX_DATABASE_NAME]) 42 | self.spx_queries.__enter__() 43 | self.obj = self.model.objects.create(**self.defaults) 44 | 45 | def get_model_defaults(self): 46 | return { 47 | 'id': self.newid(), 48 | 'sphinx_field': "hello sphinx field", 49 | 'attr_uint': 100500, 50 | 'attr_bool': True, 51 | 'attr_bigint': 2 ** 33, 52 | 'attr_float': 1.2345, 53 | 'attr_multi': [1, 2, 3], 54 | 'attr_multi_64': [2 ** 33, 2 ** 34], 55 | 'attr_timestamp': self.now, 56 | 'attr_string': "hello sphinx attr", 57 | "attr_json": {"json": "test"}, 58 | } 59 | 60 | @classmethod 61 | def newid(cls): 62 | cls._id += 1 63 | return cls._id 64 | 65 | def reload_object(self, obj): 66 | return obj._meta.model.objects.get(pk=obj.pk) 67 | 68 | def assertObjectEqualsToDefaults(self, other, defaults=None): 69 | defaults = defaults or self.defaults 70 | result = {k: getattr(other, k) for k in defaults.keys() 71 | if k != 'sphinx_field'} 72 | for k in defaults.keys(): 73 | if k == 'sphinx_field': 74 | continue 75 | self.assertEqual(result[k], defaults[k]) 76 | 77 | def tearDown(self): 78 | self.spx_queries.__exit__(*sys.exc_info()) 79 | for query in self.spx_queries.captured_queries: 80 | print(query['sql']) 81 | 82 | 83 | class SphinxModelTestCase(SphinxModelTestCaseBase): 84 | 85 | def testInsertAttributes(self): 86 | other = self.reload_object(self.obj) 87 | self.assertObjectEqualsToDefaults(other) 88 | 89 | def testSelectByAttrs(self): 90 | exclude = ['attr_multi', 'attr_multi_64', 'attr_json', 'sphinx_field'] 91 | if self.no_string_compare: 92 | exclude.extend(['attr_string', 'attr_json']) 93 | for key in self.defaults.keys(): 94 | if key in exclude: 95 | continue 96 | value = getattr(self.obj, key) 97 | try: 98 | other = self.model.objects.get(**{key: value}) 99 | except self.model.DoesNotExist: 100 | self.fail("lookup failed for %s = %s" % (key, value)) 101 | self.assertObjectEqualsToDefaults(other) 102 | 103 | def testExtraWhere(self): 104 | qs = list(self.model.objects.extra(select={'const': 0}, where=['const=0'])) 105 | self.assertEqual(len(qs), 1) 106 | 107 | def testGroupByExtraSelect(self): 108 | qs = self.model.objects.all() 109 | 110 | column_name = None 111 | for fld in self.model._meta.get_fields(): 112 | if fld.name == 'attr_uint': 113 | column_name = fld.db_column 114 | break 115 | 116 | qs = qs.extra( 117 | select={'extra': 'CEIL(%s/3600)' % column_name}) 118 | 119 | qs = qs.group_by('extra') 120 | qs = list(qs) 121 | self.assertEqual(len(qs), 1) 122 | 123 | def testSelectByMulti(self): 124 | multi_lookups = dict( 125 | attr_multi=self.obj.attr_multi[0], 126 | attr_multi_64=self.obj.attr_multi_64[0], 127 | attr_multi__in=[self.obj.attr_multi[0], 100], 128 | attr_multi_64__in=[self.obj.attr_multi_64[0], 1] 129 | ) 130 | for k, v in multi_lookups.items(): 131 | other = self.model.objects.get(**{k: v}) 132 | self.assertObjectEqualsToDefaults(other) 133 | 134 | def testShowMeta(self): 135 | qs = self.model.objects.all().with_meta() 136 | self.assertEqual(len(list(qs)), 1) 137 | self.assertTrue(hasattr(qs, 'meta')) 138 | self.assertIsInstance(qs.meta, dict) 139 | self.assertDictEqual(qs.meta, {'total': '1'}) 140 | 141 | def testExcludeByAttrs(self): 142 | exclude = ['attr_multi', 'attr_multi_64', 'attr_json', 'sphinx_field', 143 | 'attr_float', 'docid'] 144 | if self.no_string_compare: 145 | exclude.extend(['attr_string']) 146 | for key in self.defaults.keys(): 147 | if key in exclude: 148 | continue 149 | value = getattr(self.obj, key) 150 | count = self.model.objects.exclude(**{key: value}).count() 151 | self.assertEqual(count, 0) 152 | 153 | def testExcludeAttrByList(self): 154 | exclude = ['attr_multi', 'attr_multi_64', 'attr_json', 'sphinx_field', 155 | 'attr_float', 'docid'] 156 | if self.no_string_compare: 157 | exclude.extend(['attr_string']) 158 | for key in self.defaults.keys(): 159 | if key in exclude: 160 | continue 161 | value = getattr(self.obj, key) 162 | filter_kwargs = {"%s__in" % key: [value]} 163 | count = self.model.objects.exclude(**filter_kwargs).count() 164 | self.assertEqual(count, 0) 165 | 166 | def testNumericAttrLookups(self): 167 | numeric_lookups = dict( 168 | attr_uint__gte=0, 169 | attr_timestamp__gte=self.now, 170 | attr_multi__gte=0 171 | ) 172 | 173 | for k, v in numeric_lookups.items(): 174 | other = self.model.objects.get(**{k: v}) 175 | self.assertObjectEqualsToDefaults(other) 176 | 177 | def testUpdates(self): 178 | new_values = { 179 | 'attr_uint': 200, 180 | 'attr_bool': False, 181 | 'attr_bigint': 2**35, 182 | 'attr_float': 5.4321, 183 | 'attr_multi': [6,7,8], 184 | 'attr_multi_64': [2**34, 2**35], 185 | 'attr_timestamp': self.now + timedelta(seconds=60), 186 | } 187 | 188 | for k, v in new_values.items(): 189 | setattr(self.obj, k, v) 190 | 191 | # Check UPDATE mode (string attributes are not updated) 192 | self.obj.save(update_fields=new_values.keys()) 193 | 194 | other = self.reload_object(self.obj) 195 | self.assertObjectEqualsToDefaults(other, defaults=new_values) 196 | 197 | # Check REPLACE mode (string and json attributes are updated by 198 | # replacing whole row only) 199 | string_defaults = { 200 | 'sphinx_field': "another_field", 201 | 'attr_string': "another string", 202 | 'attr_json': {"json": "other", 'add': 3}, 203 | } 204 | new_values.update(string_defaults) 205 | for k, v in string_defaults.items(): 206 | setattr(self.obj, k, v) 207 | 208 | self.obj.save() 209 | 210 | other = self.reload_object(self.obj) 211 | self.assertObjectEqualsToDefaults(other, defaults=new_values) 212 | 213 | def testBulkUpdate(self): 214 | qs = self.model.objects.filter(attr_uint=self.defaults['attr_uint']) 215 | qs.update(attr_bool=not self.defaults['attr_bool']) 216 | other = self.reload_object(self.obj) 217 | self.assertFalse(other.attr_bool) 218 | 219 | def testDelete(self): 220 | if self.no_string_compare: 221 | self.skipTest("searchd version is too low") 222 | self.assertEqual(self.model.objects.count(), 1) 223 | self.obj.delete() 224 | self.assertEqual(self.model.objects.count(), 0) 225 | 226 | def testDeleteWithIn(self): 227 | expected = self.create_multiple_models() 228 | delete_ids = expected[3:7] 229 | self.model.objects.filter(id__in=delete_ids).delete() 230 | qs = self.model.objects.filter(id__in=delete_ids) 231 | self.assertEqual(len(qs), 0) 232 | qs = self.model.objects.all().values_list('id', flat=True) 233 | self.assertListEqual(list(qs), expected[:3] + expected[7:]) 234 | 235 | 236 | def testDjangoSearch(self): 237 | other = self.model.objects.filter(sphinx_field__search="hello")[0] 238 | self.assertEqual(other.id, self.obj.id) 239 | 240 | def testDjangoSearchMultiple(self): 241 | list(self.model.objects.filter(sphinx_field__search="@sdfsff 'sdfdf'", 242 | other_field__search="sdf")) 243 | 244 | def testAdminSupportIssues(self): 245 | exclude = ['attr_multi', 'attr_multi_64', 'attr_json', 'sphinx_field'] 246 | if self.no_string_compare: 247 | exclude.extend(['attr_string', 'attr_json']) 248 | for key in self.defaults.keys(): 249 | if key in exclude: 250 | continue 251 | value = getattr(self.obj, key) 252 | try: 253 | key = '%s__exact' % key 254 | other = self.model.objects.get(**{key: value}) 255 | except self.model.DoesNotExist: 256 | self.fail("lookup failed for %s = %s" % (key, value)) 257 | self.assertObjectEqualsToDefaults(other) 258 | 259 | def test64BitNumerics(self): 260 | new_values = { 261 | # 32 bit unsigned int 262 | 'attr_uint': 2**31 + 1, 263 | 'attr_multi': [2**31 + 1], 264 | # 64 bit signed int 265 | 'attr_bigint': 2**63 + 1 - 2**64, 266 | 'attr_multi_64': [2**63 + 1 - 2**64] 267 | } 268 | for k, v in new_values.items(): 269 | setattr(self.obj, k, v) 270 | 271 | # Check UPDATE mode (string attributes are not updated) 272 | self.obj.save(update_fields=new_values.keys()) 273 | 274 | other = self.reload_object(self.obj) 275 | self.assertObjectEqualsToDefaults(other, defaults=new_values) 276 | 277 | def testOptionsClause(self): 278 | self.defaults['id'] = self.newid() 279 | self.model.objects.create(**self.defaults) 280 | 281 | qs = list(self.model.objects.options( 282 | max_matches=1, ranker='bm25').all()) 283 | self.assertEqual(len(qs), 1) 284 | 285 | def testLimit(self): 286 | expected = self.create_multiple_models() 287 | qs = list(self.model.objects.all()[2:4]) 288 | self.assertEqual([q.id for q in qs], expected[2:4]) 289 | 290 | def create_multiple_models(self): 291 | expected = [self.obj.id] 292 | for i in range(10): 293 | id = self.newid() 294 | self.model.objects.create(id=id, 295 | attr_json={}, 296 | attr_uint=i, 297 | attr_timestamp=self.now) 298 | expected.append(id) 299 | return expected 300 | 301 | def testExclude(self): 302 | attr_uint = self.defaults['attr_uint'] 303 | attr_bool = self.defaults['attr_bool'] 304 | not_bool = not attr_bool 305 | 306 | # check exclude works 307 | qs = list(self.model.objects.exclude( 308 | attr_uint=attr_uint, attr_bool=attr_bool)) 309 | self.assertEqual(len(qs), 0) 310 | # check that it's really NOT (a AND b) as in Django documentation 311 | qs = list(self.model.objects.exclude( 312 | attr_uint=attr_uint, attr_bool=not_bool)) 313 | self.assertEqual(len(qs), 1) 314 | 315 | def testExcludeByList(self): 316 | attr_multi = self.defaults['attr_multi'] 317 | qs = list(self.model.objects.exclude(attr_multi__in=attr_multi)) 318 | self.assertEqual(len(qs), 0) 319 | 320 | attr_uint = self.defaults['attr_uint'] 321 | qs = list(self.model.objects.exclude(attr_uint__in=[attr_uint])) 322 | self.assertEqual(len(qs), 0) 323 | 324 | def testNumericIn(self): 325 | attr_uint = self.defaults['attr_uint'] 326 | qs = list(self.model.objects.filter(attr_uint__in=[attr_uint])) 327 | self.assertEqual(len(qs), 1) 328 | 329 | def testMatchClause(self): 330 | qs = list(self.model.objects.match("doesnotexistinindex")) 331 | self.assertEqual(len(qs), 0) 332 | qs = list(self.model.objects.match("hello")) 333 | self.assertEqual(len(qs), 1) 334 | qs = list(self.model.objects.match("hello").match("world")) 335 | self.assertEqual(len(qs), 0) 336 | 337 | def testOptionClause(self): 338 | qs = list(self.model.objects.match("hello").options( 339 | ranker="expr('sum(lcs*user_weight)*1000+bm25')", 340 | field_weights="(sphinx_field=3,other_field=2)", 341 | index_weights="(testapp_testindex=2)", 342 | sort_method="kbuffer" 343 | )) 344 | self.assertEqual(len(qs), 1) 345 | 346 | def testOrderBy(self): 347 | expected = self.create_multiple_models() 348 | qs = list(self.model.objects.order_by('-attr_uint')) 349 | expected = [self.obj.id] + list(reversed(expected[1:])) 350 | self.assertEqual([q.id for q in qs], expected) 351 | list(self.model.objects.order_by()) 352 | 353 | def testOrderByRand(self): 354 | expected = self.create_multiple_models() 355 | query = str(self.model.objects.order_by('?').query) 356 | self.assertTrue(query.endswith("ORDER BY RAND()"), 357 | msg="invalid query: %s" % query) 358 | result = list(self.model.objects.order_by()) 359 | self.assertEqual(len(expected), len(result)) 360 | 361 | query = str(self.model.objects.order_by('?')[:2].query) 362 | self.assertTrue(query.endswith("ORDER BY RAND() LIMIT 2"), 363 | msg="invalid query: %s" % query) 364 | list(self.model.objects.order_by()) 365 | 366 | def testGroupBy(self): 367 | m1 = self.model.objects.create(id=self.newid(), 368 | attr_uint=10, attr_float=1) 369 | m2 = self.model.objects.create(id=self.newid(), 370 | attr_uint=10, attr_float=2) 371 | m3 = self.model.objects.create(id=self.newid(), 372 | attr_uint=20, attr_float=2) 373 | m4 = self.model.objects.create(id=self.newid(), 374 | attr_uint=10, attr_float=1) 375 | 376 | qs = self.model.objects.defer('attr_json', 'attr_multi', 'attr_multi_64') 377 | qs = list(qs.group_by('attr_uint', 378 | group_limit=1, 379 | group_order_by='-attr_float')) 380 | self.assertSetEqual({o.id for o in qs}, {self.obj.id, m2.id, m3.id}) 381 | 382 | def testAggregation(self): 383 | s = self.model.objects.aggregate(Sum('attr_uint')) 384 | self.assertEqual(s['attr_uint__sum'], self.defaults['attr_uint']) 385 | 386 | def testSphinxFieldExact(self): 387 | sphinx_field = self.defaults['sphinx_field'] 388 | other = self.model.objects.get(sphinx_field=sphinx_field) 389 | self.assertObjectEqualsToDefaults(other) 390 | 391 | def testSphinxFieldExactExclude(self): 392 | sphinx_field = self.defaults['sphinx_field'] 393 | qs = list(self.model.objects.match('hello').exclude(sphinx_field=sphinx_field)) 394 | self.assertEqual(len(qs), 0) 395 | 396 | def testCount(self): 397 | self.create_multiple_models() 398 | r = self.model.objects.filter(attr_uint__gte=-1).count() 399 | self.assertEqual(r, 11) 400 | 401 | def testCastToChar(self): 402 | if self.no_string_compare: 403 | self.skipTest("string compare not supported by server") 404 | self.obj.attr_string = 100500 405 | self.obj.save() 406 | self.defaults['attr_string'] = '100500' 407 | other = self.model.objects.get(attr_string=100500) 408 | self.assertObjectEqualsToDefaults(other) 409 | 410 | def testWorkWithRangeInQ(self): 411 | self.create_multiple_models() 412 | total = len(self.model.objects.all()[:1000]) 413 | self.assertGreater(total, 4) 414 | # simple Q 415 | result = self.model.objects.filter(Q(attr_uint__in=[2, 4, 0])) 416 | self.assertEqual(3, len(result)) 417 | self.assertEqual(0, result[0].attr_uint) 418 | self.assertEqual(2, result[1].attr_uint) 419 | self.assertEqual(4, result[2].attr_uint) 420 | 421 | # Q with negation 422 | result = self.model.objects.filter(~Q(attr_uint__in=[2, 4, 0])) 423 | self.assertEqual(total - 3, len(result)) 424 | for item in result: 425 | self.assertNotIn(item.attr_uint, [0, 2, 4]) 426 | 427 | # Q in exclude 428 | result = self.model.objects.exclude(Q(attr_uint__in=[2, 4, 0])) 429 | self.assertEqual(total - 3, len(result)) 430 | for item in result: 431 | self.assertNotIn(item.attr_uint, [0, 2, 4]) 432 | 433 | # complex Q 434 | result = self.model.objects.filter( 435 | Q(Q(attr_uint__in=[2]) | Q(attr_uint__in=[4, 0]))) 436 | 437 | self.assertEqual(3, len(result)) 438 | self.assertEqual(0, result[0].attr_uint) 439 | self.assertEqual(2, result[1].attr_uint) 440 | self.assertEqual(4, result[2].attr_uint) 441 | 442 | def testMVAWorkWithRangeInQFor(self): 443 | self.create_multiple_models() 444 | items = self.model.objects.all() 445 | total = len(items) 446 | 447 | for i, item in enumerate(items): 448 | mva_values = [_ + 1 for _ in range(i)] 449 | item.attr_multi = mva_values 450 | item.save() 451 | 452 | # simple Q 453 | result = self.model.objects.filter(Q(attr_multi__in=[100500, 777])) 454 | self.assertFalse(result) 455 | 456 | # all items excepts the first(attr_multi==[]) 457 | result = self.model.objects.filter(Q(attr_multi__in=[1, 3])) 458 | self.assertEqual(total - 1, len(result)) 459 | for item in result: 460 | self.assertTrue(set([1, 3]) and set(item.attr_multi)) 461 | 462 | # same result 463 | result = self.model.objects.filter(Q(attr_multi__in=[1, 3, 999])) 464 | self.assertEqual(total - 1, len(result)) 465 | 466 | items[0].attr_multi.append(999) 467 | items[0].save() 468 | # now all items in result 469 | result = self.model.objects.filter(Q(attr_multi__in=[1, 3, 999])) 470 | 471 | self.assertEqual(total, len(result)) 472 | self.assertIn(items[0], result) 473 | 474 | # complex Q 475 | result = self.model.objects.filter( 476 | Q(attr_multi__in=[1, 3]) | Q(attr_multi__in=[999])) 477 | 478 | self.assertEqual(total, len(result)) 479 | 480 | # Q with negation 481 | # result does not contains first and last items 482 | result = self.model.objects.filter(~Q(attr_multi__in=[total - 1, 999])) 483 | self.assertEqual(total - 2, len(result)) 484 | 485 | for item in result: 486 | self.assertNotIn(total - 1, item.attr_multi) 487 | self.assertNotIn(999, item.attr_multi) 488 | 489 | # Q in exclude 490 | # result does not contains first and last items 491 | result = self.model.objects.exclude(Q(attr_multi__in=[total - 1, 999])) 492 | self.assertEqual(total - 2, len(result)) 493 | 494 | self.assertNotIn(total - 1, item.attr_multi) 495 | self.assertNotIn(999, item.attr_multi) 496 | 497 | 498 | class ForcedPKTestCase(SphinxModelTestCase): 499 | model = models.ForcedPKModel 500 | 501 | 502 | class TestOverridenSphinxModel(SphinxModelTestCase): 503 | model = models.OverridenSphinxModel 504 | 505 | 506 | class TestModelWithAllDbColumnFields(SphinxModelTestCase): 507 | model = models.ModelWithAllDbColumnFields 508 | 509 | 510 | class CharPKTestCase(SphinxModelTestCase): 511 | model = models.CharPKModel 512 | 513 | def get_model_defaults(self): 514 | defaults = super(CharPKTestCase, self).get_model_defaults() 515 | defaults['docid'] = str(defaults['id']) 516 | return defaults 517 | 518 | @expectedFailure 519 | def testDelete(self): 520 | """ 521 | DELETE FROM `testapp_charpkmodel` WHERE (IN(docid, '1')) does not work 522 | :return: 523 | """ 524 | super(CharPKTestCase, self).testDelete() 525 | 526 | 527 | class TestSphinxRouter(SphinxModelTestCaseBase): 528 | def setUp(self): 529 | super(TestSphinxRouter, self).setUp() 530 | self.router = SphinxRouter() 531 | 532 | def testSphinxModelDetection(self): 533 | self.assertTrue(self.router.is_sphinx_model( 534 | models.TestModel)) 535 | 536 | self.assertTrue(self.router.is_sphinx_model( 537 | models.TestModel())) 538 | 539 | self.assertTrue(self.router.is_sphinx_model( 540 | models.OverridenSphinxModel)) 541 | 542 | self.assertTrue(self.router.is_sphinx_model( 543 | models.OverridenSphinxModel())) 544 | 545 | self.assertFalse(self.router.is_sphinx_model( 546 | models.DefaultDjangoModel)) 547 | 548 | self.assertFalse(self.router.is_sphinx_model( 549 | models.DefaultDjangoModel())) 550 | 551 | 552 | class EscapingTestCase(SphinxModelTestCaseBase): 553 | """ Checks escaping symbols""" 554 | 555 | def setUp(self): 556 | super(EscapingTestCase, self).setUp() 557 | self.obj.sphinx_field = 'sphinx' 558 | self.obj.save() 559 | 560 | def query(self, text, escape=True): 561 | escaped = sphinx_escape(text) if escape else text 562 | for c in text: 563 | self.assertIn(c, escaped) 564 | try: 565 | return list(self.model.objects.match(escaped)) 566 | except ProgrammingError as e: 567 | self.fail("Escaping text %s with %s failed: %s" % 568 | (text, escaped, e.args[1])) 569 | 570 | def testSphinxCharactersEscaping(self): 571 | """ 572 | Any sphinxql operator should not match document if escaped properly. 573 | """ 574 | operators = '=<>()|!@~&/^$\-\'\"\\' 575 | for o in operators: 576 | res = self.query("sphinx operators %s" % o) 577 | self.assertEqual(len(res), 0) 578 | text = sphinx_escape("sphinx operators %s" % o) 579 | res = self.query('"%s"/1' % text, escape=False) 580 | self.assertEqual(len(res), 1) 581 | 582 | def testSphinxKeywordsEscaping(self): 583 | """ 584 | a SENTENCE b means "a" and "b" in one sentence. 585 | a PARAGRAPH b means "a" and "b" in same html block. 586 | """ 587 | self.obj.sphinx_field = ('

Paragraph is not a word.

\n' 588 | '

Sentence also.

') 589 | self.obj.save() 590 | # text contains word paragraph 591 | res = self.query("PARAGRAPH") 592 | self.assertEqual(len(res), 1) 593 | text = sphinx_escape("PARAGRAPH") 594 | # "paragraph" and "is" in one sentence 595 | res = self.query('%s SENTENCE is' % text, escape=False) 596 | self.assertTrue(len(res), 1) 597 | 598 | # text contains word sentence 599 | res = self.query("SENTENCE") 600 | self.assertEqual(len(res), 1) 601 | text = sphinx_escape("SENTENCE") 602 | # "sentence" and "also" in one paragraph 603 | res = self.query('%s PARAGRAPH also' % text, escape=False) 604 | self.assertTrue(len(res), 1) 605 | 606 | # "not" and "sentence" in one paragraph (actually, false) 607 | res = self.query('not PARAGRAPH %s' % text, escape=False) 608 | self.assertEqual(len(res), 0) 609 | -------------------------------------------------------------------------------- /testproject/urls.py: -------------------------------------------------------------------------------- 1 | """django_sphinxsearch URL Configuration 2 | 3 | The `urlpatterns` list routes URLs to views. For more information please see: 4 | https://docs.djangoproject.com/en/1.8/topics/http/urls/ 5 | Examples: 6 | Function views 7 | 1. Add an import: from my_app import views 8 | 2. Add a URL to urlpatterns: url(r'^$', views.home, name='home') 9 | Class-based views 10 | 1. Add an import: from other_app.views import Home 11 | 2. Add a URL to urlpatterns: url(r'^$', Home.as_view(), name='home') 12 | Including another URLconf 13 | 1. Add an import: from blog import urls as blog_urls 14 | 2. Add a URL to urlpatterns: url(r'^blog/', include(blog_urls)) 15 | """ 16 | from django.conf.urls import include, url 17 | # from django.contrib import admin 18 | 19 | urlpatterns = [ 20 | # url(r'^admin/', include(admin.site.urls)), 21 | ] 22 | -------------------------------------------------------------------------------- /testproject/wsgi.py: -------------------------------------------------------------------------------- 1 | """ 2 | WSGI config for django_sphinxsearch project. 3 | 4 | It exposes the WSGI callable as a module-level variable named ``application``. 5 | 6 | For more information on this file, see 7 | https://docs.djangoproject.com/en/1.8/howto/deployment/wsgi/ 8 | """ 9 | 10 | import os 11 | 12 | from django.core.wsgi import get_wsgi_application 13 | 14 | os.environ.setdefault("DJANGO_SETTINGS_MODULE", "testproject.settings") 15 | 16 | application = get_wsgi_application() 17 | --------------------------------------------------------------------------------