├── example ├── __init__.py ├── movies │ ├── __init__.py │ ├── views.py │ ├── models.py │ ├── documents.py │ └── tests.py ├── urls.py ├── manage.py ├── tests.py ├── schema.xml └── settings.py ├── djangosolr ├── managment │ ├── __init__.py │ └── commands │ │ └── __init__.py ├── documents │ ├── __init__.py │ ├── manager.py │ ├── document.py │ ├── options.py │ ├── fields.py │ ├── queryset.py │ └── query.py ├── __init__.py ├── conf │ ├── __init__.py │ └── default_settings.py └── solr.py ├── .gitignore ├── MANIFEST.in ├── setup.py ├── LICENSE └── README.md /example/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /example/movies/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /djangosolr/managment/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /djangosolr/managment/commands/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.pyc 2 | .project 3 | .pydevproject 4 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include LICENSE 2 | include README 3 | -------------------------------------------------------------------------------- /example/movies/views.py: -------------------------------------------------------------------------------- 1 | # Create your views here. 2 | -------------------------------------------------------------------------------- /example/movies/models.py: -------------------------------------------------------------------------------- 1 | from django.db import models 2 | 3 | class Movie(models.Model): 4 | 5 | title = models.CharField() 6 | director = models.CharField() 7 | -------------------------------------------------------------------------------- /example/movies/documents.py: -------------------------------------------------------------------------------- 1 | from example.movies.models import Movie as MovieDB 2 | from djangosolr.documents import Document, TextField 3 | 4 | class Movie(Document): 5 | 6 | text = TextField(stored=False) 7 | 8 | class Meta: 9 | model = MovieDB 10 | type = 'movie' -------------------------------------------------------------------------------- /djangosolr/documents/__init__.py: -------------------------------------------------------------------------------- 1 | from djangosolr.documents.document import Document 2 | from djangosolr.documents.manager import Manager 3 | from djangosolr.documents.fields import Field, CharField, DateTimeField, DecimalField, FloatField, IntegerField, TextField, BooleanField 4 | from djangosolr.documents.query import Q -------------------------------------------------------------------------------- /example/movies/tests.py: -------------------------------------------------------------------------------- 1 | """ 2 | This file demonstrates writing tests using the unittest module. These will pass 3 | when you run "manage.py test". 4 | 5 | Replace this with more appropriate tests for your application. 6 | """ 7 | 8 | from django.test import TestCase 9 | 10 | 11 | class SimpleTest(TestCase): 12 | def test_basic_addition(self): 13 | """ 14 | Tests that 1 + 1 always equals 2. 15 | """ 16 | self.assertEqual(1 + 1, 2) 17 | -------------------------------------------------------------------------------- /example/urls.py: -------------------------------------------------------------------------------- 1 | from django.conf.urls.defaults import patterns, include, url 2 | 3 | # Uncomment the next two lines to enable the admin: 4 | # from django.contrib import admin 5 | # admin.autodiscover() 6 | 7 | urlpatterns = patterns('', 8 | # Examples: 9 | # url(r'^$', 'example.views.home', name='home'), 10 | # url(r'^example/', include('example.foo.urls')), 11 | 12 | # Uncomment the admin/doc line below to enable admin documentation: 13 | # url(r'^admin/doc/', include('django.contrib.admindocs.urls')), 14 | 15 | # Uncomment the next line to enable the admin: 16 | # url(r'^admin/', include(admin.site.urls)), 17 | ) 18 | -------------------------------------------------------------------------------- /example/manage.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | from django.core.management import execute_manager 3 | import sys, os 4 | 5 | sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '../'))) 6 | 7 | try: 8 | import settings # Assumed to be in the same directory. 9 | except ImportError: 10 | sys.stderr.write("Error: Can't find the file 'settings.py' in the directory containing %r. It appears you've customized things.\nYou'll have to run django-admin.py, passing it your settings module.\n(If the file settings.py does indeed exist, it's causing an ImportError somehow.)\n" % __file__) 11 | sys.exit(1) 12 | 13 | if __name__ == "__main__": 14 | execute_manager(settings) -------------------------------------------------------------------------------- /djangosolr/__init__.py: -------------------------------------------------------------------------------- 1 | """ 2 | Solr Search Engine ORM for Django 3 | 4 | Create, save, fetch, update and delete: 5 | 6 | mv = MovieDocument (title='Jurassic Park', director='Steven Spielberg') 7 | mv.save() 8 | 9 | mv = MovieDocument.docuemnts.get(1) 10 | 11 | mv.title = 'Jurassic Park I' 12 | mv.save() 13 | 14 | mv.delete() 15 | 16 | 17 | Search: 18 | 19 | MovieDocument.documents.q(Q('jurassic park') & Q('director', 'spielberg'))[:10] 20 | 21 | 22 | *Solr Not Included 23 | """ 24 | 25 | __version__ = (1, 0, 0) 26 | 27 | from djangosolr.conf import inject_defaults 28 | inject_defaults() 29 | -------------------------------------------------------------------------------- /djangosolr/conf/__init__.py: -------------------------------------------------------------------------------- 1 | 2 | #http://passingcuriosity.com/2010/default-settings-for-django-applications/ 3 | 4 | def inject_defaults(): 5 | import default_settings 6 | import sys 7 | 8 | _app_settings = sys.modules['djangosolr.conf.default_settings'] 9 | _def_settings = sys.modules['django.conf.global_settings'] 10 | _settings = sys.modules['django.conf'].settings 11 | for _k in dir(_app_settings): 12 | if _k.isupper(): 13 | # Add the value to the default settings module 14 | setattr(_def_settings, _k, getattr(_app_settings, _k)) 15 | 16 | # Add the value to the settings, if not already present 17 | if not hasattr(_settings, _k): 18 | setattr(_settings, _k, getattr(_app_settings, _k)) -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | from setuptools import setup, find_packages 2 | 3 | setup( 4 | name='django-solr', 5 | version='1.0.0alpha15', 6 | description='Solr Search Engine ORM for Django', 7 | author='Sophilabs', 8 | author_email='contact@sophilabs.com', 9 | url='https://github.com/sophilabs/django-solr', 10 | download_url='http://github.com/sophilabs/django-solr/tarball/v1.0.0alpha15#egg=django-solr-1.0.0alpha15', 11 | license='BSD', 12 | packages=[ 13 | 'djangosolr', 14 | 'djangosolr.conf', 15 | 'djangosolr.documents', 16 | ], 17 | classifiers=[ 18 | 'Development Status :: 3 - Alpha', 19 | 'Environment :: Web Environment', 20 | 'Intended Audience :: Developers', 21 | 'License :: OSI Approved :: BSD License', 22 | 'Operating System :: OS Independent', 23 | 'Programming Language :: Python', 24 | 'Framework :: Django', 25 | ], 26 | ) 27 | -------------------------------------------------------------------------------- /example/tests.py: -------------------------------------------------------------------------------- 1 | import sys, os 2 | sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '../'))) 3 | 4 | os.environ['DJANGO_SETTINGS_MODULE'] = 'example.settings' 5 | from example.movies.documents import Movie 6 | from djangosolr.documents import Q 7 | 8 | #Save some movies 9 | Movie(id="1", title='Jurassic Park I', director='Steven Spielberg').save() 10 | Movie(id="2", title='Jurassic Park III', director='Steven Spielberg').save() 11 | 12 | #Get and update 13 | m = Movie.documents.get(2) 14 | m.director = None 15 | m.save() 16 | 17 | #Get all movies 18 | ms = Movie.documents.all() 19 | 20 | #Get the first 10 Steven Spielberg's movies 21 | ms = Movie.documents.q(director__exact='Steven Spielberg').sort('title')[:10] 22 | print ms.count() 23 | for m in ms: 24 | print m.title 25 | 26 | #Get Spielberg's or Johnston's movies 27 | ms = Movie.documents.q(Q(text='spielberg')) 28 | for m in ms: 29 | print m.title, m.director, m.text 30 | 31 | #Delete a movie 32 | m = Movie.documents.get(1) 33 | m.delete() 34 | 35 | #Delete all movies 36 | Movie.documents.clear() 37 | -------------------------------------------------------------------------------- /djangosolr/conf/default_settings.py: -------------------------------------------------------------------------------- 1 | 2 | DJANGOSOLR_ID_FIELD = 'id' 3 | DJANGOSOLR_TYPE_FIELD = 'type' 4 | 5 | DJANGOSOLR_FIELD_MAPPING = { 6 | 'django.db.models.fields.AutoField': 'djangosolr.documents.fields.IntegerField', 7 | 'django.db.models.fields.IntegerField': 'djangosolr.documents.fields.IntegerField', 8 | 'django.db.models.fields.BigIntegerField': 'djangosolr.documents.fields.IntegerField', 9 | 'django.db.models.fields.FloatField': 'djangosolr.documents.fields.FloatField', 10 | 'django.db.models.fields.DecimalField': 'djangosolr.documents.fields.DecimalField', 11 | 'django.db.models.fields.TextField': 'djangosolr.documents.fields.CharField', 12 | 'django.db.models.fields.CharField': 'djangosolr.documents.fields.CharField', 13 | 'django.db.models.fields.DateField': 'djangosolr.documents.fields.DateTimeField', 14 | 'django.db.models.fields.DateTimeField': 'djangosolr.documents.fields.DateTimeField', 15 | 'django.db.models.fields.BooleanField': 'djangosolr.documents.fields.BooleanField', 16 | 'django.db.models.fields.NullBooleanField': 'djangosolr.documents.fields.BooleanField', 17 | } 18 | 19 | DJANGOSOLR_AUTOCOMMIT = True 20 | DJANGOSOLR_URL = 'http://localhost:8983/solr' 21 | DJANGOSOLR_SELECT_PATH = '/select' 22 | DJANGOSOLR_UPDATE_PATH = '/update/json' 23 | DJANGOSOLR_DELETE_PATH = '/update/json' 24 | 25 | DJANGOSOLR_ROWS_PER_QUERY = 2 26 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2011, Sophilabs 2 | All rights reserved. 3 | 4 | Redistribution and use in source and binary forms, with or without 5 | modification, are permitted provided that the following conditions are 6 | met: 7 | 8 | * Redistributions of source code must retain the above copyright 9 | notice, this list of conditions and the following disclaimer. 10 | * Redistributions in binary form must reproduce the above 11 | copyright notice, this list of conditions and the following 12 | disclaimer in the documentation and/or other materials provided 13 | with the distribution. 14 | * Neither the name of the author nor the names of other 15 | contributors may be used to endorse or promote products derived 16 | from this software without specific prior written permission. 17 | 18 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 19 | "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 20 | LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 21 | A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 22 | OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 23 | SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 24 | LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 25 | DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 26 | THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 27 | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 28 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 29 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Solr Search Engine ORM for Django 2 | 3 | [![No Maintenance Intended](http://unmaintained.tech/badge.svg)](http://unmaintained.tech/) 4 | 5 | ## Usage 6 | 7 | Define 8 | 9 | from djangosolr import documents 10 | 11 | class Movie(documents.Document): 12 | id = documents.IntegerField(primary_key=True) 13 | title = documents.CharField() 14 | director = documents.CharField() 15 | text = TextField() 16 | 17 | Define from an existing django model 18 | 19 | from djangosolr import documents 20 | from myapp import models 21 | 22 | class Movie(documents.Document): 23 | class Meta: 24 | model = models.Movie 25 | 26 | 27 | Save some movies 28 | 29 | Movie(id="1", title='Jurassic Park I', director='Steven Spielberg').save() 30 | Movie(id="2", title='Jurassic Park III', director='Steven Spielberg').save() 31 | 32 | Save many movies at once 33 | 34 | from djangosolr import solr 35 | 36 | solr.save([m1, m2]) 37 | 38 | Get and update 39 | 40 | m = Movie.documents.get(2) 41 | m.director = 'Joe Johnston' 42 | m.save() 43 | 44 | Get all movies 45 | 46 | ms = Movie.documents.all() 47 | 48 | Get the first 10 Steven Spielberg's movies 49 | 50 | ms = Movie.documents.q(director__exact='Steven Spielberg').sort('title')[:10] 51 | 52 | Get Spielberg's or Johnston's movies 53 | 54 | ms = Movie.documents.q(Q(text='spielberg') | Q(text='johnston')) 55 | 56 | Delete a movie 57 | 58 | m = Movie.documents.get(1) 59 | m.delete() 60 | 61 | Delete all movies 62 | 63 | Movie.documents.clear() 64 | 65 | ## Getting It 66 | 67 | You can get Django Solr by using pip or easy_install 68 | 69 | $ pip install django-solr 70 | or 71 | $ easy_install django-solr 72 | 73 | ## Comming Soon 74 | 75 | * Facet 76 | * More Like This 77 | 78 | -------------------------------------------------------------------------------- /djangosolr/documents/manager.py: -------------------------------------------------------------------------------- 1 | from djangosolr.documents.queryset import QuerySet 2 | from djangosolr.documents.query import Q 3 | from django.conf import settings 4 | from djangosolr import solr 5 | 6 | class ManagerDescriptor(object): 7 | 8 | def __init__(self, manager): 9 | self.manager = manager 10 | 11 | def __get__(self, instance, type=None): 12 | if instance != None: 13 | raise AttributeError("Manager isn't accessible via %s instances" % type.__name__) 14 | return self.manager 15 | 16 | class Manager(object): 17 | 18 | def _contribute_to_class(self, model, name): 19 | self._model = model 20 | setattr(model, name, ManagerDescriptor(self)) 21 | if not getattr(model, '_default_manager', None): 22 | model._default_manager = self 23 | 24 | def _get_query_set(self): 25 | return QuerySet(self._model) 26 | 27 | def all(self): 28 | return self._get_query_set() 29 | 30 | def raw(self, **kwargs): 31 | return self._get_query_set().raw(**kwargs) 32 | 33 | def q(self, *qs, **filters): 34 | return self._get_query_set().q(*qs, **filters) 35 | 36 | def fq(self, *qs, **filters): 37 | return self._get_query_set().fq(*qs, **filters) 38 | 39 | def fl(self, *fields): 40 | return self._get_query_set().fl(*fields) 41 | 42 | def sort(self, *fields): 43 | return self._get_query_set().sort(*fields) 44 | 45 | def get(self, id): 46 | pk = self._model._meta.pk 47 | return self._get_query_set().q(Q('%s:%s-%s' % (settings.DJANGOSOLR_ID_FIELD, self._model._meta.type, pk.prepare(id),)))[0] 48 | 49 | def delete(self, *qs, **filters): 50 | return self._get_query_set().delete(*qs, **filters) 51 | 52 | def clear(self): 53 | return solr.clear(self._model) 54 | 55 | def ensure_default_manager(cls): 56 | if not getattr(cls, '_default_manager', None): 57 | cls._add_to_class('documents', Manager()) 58 | cls._base_manager = cls.documents 59 | elif not getattr(cls, '_base_manager', None): 60 | default_mgr = cls._default_manager.__class__ 61 | if (default_mgr is Manager): 62 | cls._base_manager = cls._default_manager 63 | else: 64 | for base_class in default_mgr.mro()[1:]: 65 | if (base_class is Manager): 66 | cls.add_to_class('_base_manager', base_class()) 67 | return 68 | raise AssertionError("Should never get here. Please report a bug, including your model and model manager setup.") -------------------------------------------------------------------------------- /djangosolr/documents/document.py: -------------------------------------------------------------------------------- 1 | from djangosolr.documents.options import Options 2 | from djangosolr.documents.manager import ensure_default_manager 3 | from djangosolr import solr 4 | 5 | class DocumentBase(type): 6 | 7 | def __new__(cls, name, bases, attrs): 8 | super_new = super(DocumentBase, cls).__new__ 9 | new_class = super_new(cls, name, bases, {'__module__': attrs.pop('__module__')}) 10 | 11 | attr_meta = attrs.pop('Meta', None) 12 | if not attr_meta: 13 | meta = getattr(new_class, 'Meta', None) 14 | else: 15 | meta = attr_meta 16 | new_class._add_to_class('_meta', Options(meta)) 17 | 18 | if getattr(new_class, '_default_manager', None): 19 | new_class._default_manager = None 20 | new_class._base_manager = None 21 | 22 | for obj_name, obj in attrs.items(): 23 | new_class._add_to_class(obj_name, obj) 24 | 25 | new_class._prepare_class() 26 | 27 | return new_class 28 | 29 | def _add_to_class(cls, name, value): 30 | if hasattr(value, '_contribute_to_class'): 31 | value._contribute_to_class(cls, name) 32 | else: 33 | setattr(cls, name, value) 34 | 35 | def _prepare_class(cls): 36 | opts = cls._meta 37 | opts._prepare_class(cls) 38 | ensure_default_manager(cls) 39 | 40 | class Document(object): 41 | 42 | __metaclass__ = DocumentBase 43 | 44 | def __init__(self, **kwargs): 45 | for field in self._meta.fields: 46 | if field.name in kwargs: 47 | setattr(self, field.name, kwargs.pop(field.name)) 48 | else: 49 | setattr(self, field.name, field.get_default()) 50 | if kwargs: 51 | raise KeyError(kwargs.keys()[0]) 52 | 53 | @classmethod 54 | def create(cls, om): 55 | document = cls() 56 | if isinstance(om, dict): 57 | for field in cls._meta.fields: 58 | name = cls._meta.get_solr_field_name(field) 59 | if om.has_key(name): 60 | setattr(document, field.name, field.convert(om[name])) 61 | else: 62 | for field in cls._meta.fields: 63 | if hasattr(om, field.name): 64 | setattr(document, field.name, getattr(om, field.name)) 65 | return document 66 | 67 | def save(self): 68 | return solr.save([self]) 69 | 70 | def pre_save(self): 71 | pass 72 | 73 | def delete(self): 74 | return solr.delete([self]) 75 | 76 | def pre_delete(self): 77 | pass 78 | -------------------------------------------------------------------------------- /djangosolr/solr.py: -------------------------------------------------------------------------------- 1 | from django.conf import settings 2 | import httplib2, json, re, urllib 3 | 4 | #http://fragmentsofcode.wordpress.com/2010/03/10/escape-special-characters-for-solrlucene-query/ 5 | ESCAPE_CHARS_RE = re.compile(r'(?[&|+\-!(){}[\]^"~*?:])') 6 | 7 | def escape(value): 8 | return ESCAPE_CHARS_RE.sub(r'\\\g', value) 9 | 10 | def urlencode(query): 11 | l = [] 12 | for k, v in query: 13 | k = urllib.quote(unicode(k).encode('utf8')) 14 | v = urllib.quote(unicode(v).encode('utf8')) 15 | l.append(k + '=' + v) 16 | return '&'.join(l) 17 | 18 | def request(method, path, query=None, body=None): 19 | try: 20 | uri = '%s%s?wt=json' % (settings.DJANGOSOLR_URL, path,) 21 | if query: 22 | uri += '&' + urlencode(query) 23 | if body: 24 | body = json.dumps(body) 25 | headers, body = httplib2.Http().request(uri=uri, method=method, body=body, headers={'Content-type': 'application/json'}) 26 | if headers['status'] == '200': 27 | return json.loads(body) 28 | raise Exception(body) 29 | except: 30 | raise 31 | 32 | def select(query): 33 | return request('GET', settings.DJANGOSOLR_SELECT_PATH, query) 34 | 35 | def save(docs, commit=True, overwrite=True): 36 | ddocs = [] 37 | for doc in docs: 38 | m = doc._meta 39 | doc.pre_save() 40 | ddoc = { m.get_solr_id_field(): m.get_solr_id_value(doc), 41 | m.get_solr_type_field(): m.get_solr_type_value()} 42 | for field in doc._meta.fields: 43 | value = field.prepare(getattr(doc, field.name)) 44 | if value is None: 45 | ddoc[m.get_solr_field_name(field)] = [] #BUG: https://issues.apache.org/jira/browse/SOLR-2714 46 | else: 47 | ddoc[m.get_solr_field_name(field)] = value 48 | ddocs.append(ddoc) 49 | return request('POST', settings.DJANGOSOLR_UPDATE_PATH, [('commit', str(commit).lower(),), ('overwrite', str(overwrite).lower())], { 'add': ddocs }) 50 | 51 | def delete(query, commit=True): 52 | if not isinstance(query, basestring): 53 | queries = [] 54 | for doc in query: 55 | m = doc._meta 56 | doc.pre_delete() 57 | queries.append(u'%s:%s' % (m.get_solr_id_field(), escape(m.get_solr_id_value(doc)))) 58 | query = ' OR '.join(queries) 59 | return request('POST', settings.DJANGOSOLR_DELETE_PATH, [('commit', str(commit).lower(),)], {'delete': { 'query': query }}) 60 | 61 | def clear(model, commit=True): 62 | return request('POST', settings.DJANGOSOLR_DELETE_PATH, [('commit', str(commit).lower(),)], {'delete': { 'query': settings.DJANGOSOLR_TYPE_FIELD + ':' + model._meta.type}}) 63 | -------------------------------------------------------------------------------- /djangosolr/documents/options.py: -------------------------------------------------------------------------------- 1 | from django.utils.importlib import import_module 2 | from djangosolr.solr import escape 3 | from django.conf import settings 4 | 5 | class Options(object): 6 | 7 | def __init__(self, meta): 8 | self.meta = meta 9 | self.model = None 10 | self.type = None 11 | self.fields = [] 12 | self.pk = None 13 | 14 | def get_field(self, name): 15 | for field in self.fields: 16 | if field.name == name: 17 | return field 18 | raise IndexError(name) 19 | 20 | def get_solr_field_name(self, field): 21 | if isinstance(field, basestring): 22 | field = self.get_field(field) 23 | return escape(u'%s__%s' % (self.type, field.name,)) 24 | 25 | def get_solr_id_field(self): 26 | return settings.DJANGOSOLR_ID_FIELD 27 | 28 | def get_solr_id_value(self, document): 29 | return u'%s-%s' % (self.type, self.pk.prepare(getattr(document, self.pk.name)),) 30 | 31 | def get_solr_type_field(self): 32 | return settings.DJANGOSOLR_TYPE_FIELD 33 | 34 | def get_solr_type_value(self): 35 | return self.type 36 | 37 | 38 | def add_field(self, field): 39 | self.fields.append(field) 40 | if field.primary_key: 41 | self.pk = field 42 | 43 | def _contribute_to_class(self, cls, name): 44 | cls._meta = self 45 | self.name = cls.__name__.lower() 46 | if self.meta: 47 | meta_attrs = self.meta.__dict__.copy() 48 | for name in self.meta.__dict__: 49 | if name.startswith('_'): 50 | del meta_attrs[name] 51 | for attr_name in ['model', 'type']: 52 | if attr_name in meta_attrs: 53 | setattr(self, attr_name, meta_attrs.pop(attr_name)) 54 | elif hasattr(self.meta, attr_name): 55 | setattr(self, attr_name, getattr(self.meta, attr_name)) 56 | 57 | if meta_attrs != {}: 58 | raise TypeError("'class Meta' got invalid attribute(s): %s" % ','.join(meta_attrs.keys())) 59 | if not self.type: 60 | self.type = self.name 61 | del self.meta 62 | 63 | def _prepare_class(self, model): 64 | mapping = settings.DJANGOSOLR_FIELD_MAPPING 65 | if model._meta.model: 66 | for df in model._meta.model._meta.local_fields: 67 | kwargs = dict(name=df.name, stored=True, indexed=True, multivalued=False, primary_key=df.primary_key) 68 | sc = df.__class__.__module__ + '.' + df.__class__.__name__ 69 | f_module, f_classname = mapping[sc].rsplit('.', 1) 70 | f = getattr(import_module(f_module), f_classname)(**kwargs) 71 | model._add_to_class(f.name, f) -------------------------------------------------------------------------------- /djangosolr/documents/fields.py: -------------------------------------------------------------------------------- 1 | import datetime, decimal 2 | from django.utils.encoding import force_unicode 3 | from djangosolr.solr import escape 4 | 5 | class Field(): 6 | 7 | def __init__(self, type='string', name=None, stored=True, indexed=True, multivalued=False, primary_key=False): 8 | self.type = type 9 | self.name = name 10 | self.stored = stored 11 | self.indexed = indexed 12 | self.multivalued = multivalued 13 | self.primary_key = primary_key 14 | 15 | def _contribute_to_class(self, cls, name): 16 | self.name = name 17 | self._model = cls 18 | cls._meta.add_field(self) 19 | 20 | def get_default(self): 21 | return None 22 | 23 | def prepare(self, value): 24 | return value 25 | 26 | def convert(self, value): 27 | return value 28 | 29 | def prepare_to_query(self, value): 30 | value = self.prepare(value) 31 | if isinstance(value, basestring): 32 | return escape(value) 33 | else: 34 | return escape(str(value)) 35 | 36 | class BooleanField(Field): 37 | pass 38 | 39 | class IntegerField(Field): 40 | 41 | def __init__(self, **kwargs): 42 | kwargs.setdefault('type', 'int') 43 | Field.__init__(self, **kwargs) 44 | 45 | class CharField(Field): 46 | 47 | def __init__(self, **kwargs): 48 | kwargs.setdefault('type', 'string') 49 | Field.__init__(self, **kwargs) 50 | 51 | def prepare(self, value): 52 | if value is None: 53 | return None 54 | elif hasattr(value, '__iter__'): 55 | return [self.prepare(v) for v in value] 56 | else: 57 | return force_unicode(value) 58 | 59 | class DateTimeField(Field): 60 | 61 | def __init__(self, **kwargs): 62 | kwargs.setdefault('type', 'date') 63 | Field.__init__(self, **kwargs) 64 | 65 | def prepare(self, value): 66 | if value is None: 67 | return None 68 | if hasattr(value, '__iter__'): 69 | return [self.prepare(v) for v in value] 70 | else: 71 | return value.strftime('%Y-%m-%dT%H:%M:%S.%fZ') 72 | 73 | def convert(self, value): 74 | try: 75 | return datetime.datetime.strptime(value, '%Y-%m-%dT%H:%M:%SZ') 76 | except ValueError: 77 | try: 78 | return datetime.datetime.strptime(value, '%Y-%m-%dT%H:%M:%S.%fZ') 79 | except ValueError: 80 | return datetime.datetime.strptime(value, '%Y-%m-%dT00:00:00Z').date() 81 | 82 | class FloatField(Field): 83 | 84 | def __init__(self, **kwargs): 85 | kwargs.setdefault('type', 'float') 86 | Field.__init__(self, **kwargs) 87 | 88 | class TextField(Field): 89 | 90 | def __init__(self, **kwargs): 91 | kwargs.setdefault('type', 'text') 92 | Field.__init__(self, **kwargs) 93 | 94 | class DecimalField(Field): 95 | 96 | def __init__(self, **kwargs): 97 | kwargs.setdefault('type', 'float') 98 | Field.__init__(self, **kwargs) 99 | 100 | def prepare(self, value): 101 | if value is None: 102 | return value 103 | elif hasattr(value, '__iter__'): 104 | return [self.prepare(v) for v in value] 105 | else: 106 | return float(str(value)) 107 | 108 | def convert(self, value): 109 | return decimal.Decimal(value) 110 | -------------------------------------------------------------------------------- /example/schema.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 45 | 46 | 47 | 48 | 49 | 50 | 51 | 52 | 53 | 54 | 55 | 56 | 57 | 58 | 59 | 60 | 61 | 62 | 63 | 64 | 65 | 66 | 67 | 68 | 69 | 70 | 71 | id 72 | id 73 | 74 | 75 | 76 | 77 | 78 | 79 | -------------------------------------------------------------------------------- /example/settings.py: -------------------------------------------------------------------------------- 1 | # Django settings for example project. 2 | 3 | DEBUG = True 4 | TEMPLATE_DEBUG = DEBUG 5 | 6 | ADMINS = ( 7 | # ('Your Name', 'your_email@example.com'), 8 | ) 9 | 10 | MANAGERS = ADMINS 11 | 12 | DATABASES = { 13 | 'default': { 14 | 'ENGINE': 'django.db.backends.', # Add 'postgresql_psycopg2', 'postgresql', 'mysql', 'sqlite3' or 'oracle'. 15 | 'NAME': '', # Or path to database file if using sqlite3. 16 | 'USER': '', # Not used with sqlite3. 17 | 'PASSWORD': '', # Not used with sqlite3. 18 | 'HOST': '', # Set to empty string for localhost. Not used with sqlite3. 19 | 'PORT': '', # Set to empty string for default. Not used with sqlite3. 20 | } 21 | } 22 | 23 | # Local time zone for this installation. Choices can be found here: 24 | # http://en.wikipedia.org/wiki/List_of_tz_zones_by_name 25 | # although not all choices may be available on all operating systems. 26 | # On Unix systems, a value of None will cause Django to use the same 27 | # timezone as the operating system. 28 | # If running in a Windows environment this must be set to the same as your 29 | # system time zone. 30 | TIME_ZONE = 'America/Chicago' 31 | 32 | # Language code for this installation. All choices can be found here: 33 | # http://www.i18nguy.com/unicode/language-identifiers.html 34 | LANGUAGE_CODE = 'en-us' 35 | 36 | SITE_ID = 1 37 | 38 | # If you set this to False, Django will make some optimizations so as not 39 | # to load the internationalization machinery. 40 | USE_I18N = True 41 | 42 | # If you set this to False, Django will not format dates, numbers and 43 | # calendars according to the current locale 44 | USE_L10N = True 45 | 46 | # Absolute filesystem path to the directory that will hold user-uploaded files. 47 | # Example: "/home/media/media.lawrence.com/media/" 48 | MEDIA_ROOT = '' 49 | 50 | # URL that handles the media served from MEDIA_ROOT. Make sure to use a 51 | # trailing slash. 52 | # Examples: "http://media.lawrence.com/media/", "http://example.com/media/" 53 | MEDIA_URL = '' 54 | 55 | # Absolute path to the directory static files should be collected to. 56 | # Don't put anything in this directory yourself; store your static files 57 | # in apps' "static/" subdirectories and in STATICFILES_DIRS. 58 | # Example: "/home/media/media.lawrence.com/static/" 59 | STATIC_ROOT = '' 60 | 61 | # URL prefix for static files. 62 | # Example: "http://media.lawrence.com/static/" 63 | STATIC_URL = '/static/' 64 | 65 | # URL prefix for admin static files -- CSS, JavaScript and images. 66 | # Make sure to use a trailing slash. 67 | # Examples: "http://foo.com/static/admin/", "/static/admin/". 68 | ADMIN_MEDIA_PREFIX = '/static/admin/' 69 | 70 | # Additional locations of static files 71 | STATICFILES_DIRS = ( 72 | # Put strings here, like "/home/html/static" or "C:/www/django/static". 73 | # Always use forward slashes, even on Windows. 74 | # Don't forget to use absolute paths, not relative paths. 75 | ) 76 | 77 | # List of finder classes that know how to find static files in 78 | # various locations. 79 | STATICFILES_FINDERS = ( 80 | 'django.contrib.staticfiles.finders.FileSystemFinder', 81 | 'django.contrib.staticfiles.finders.AppDirectoriesFinder', 82 | # 'django.contrib.staticfiles.finders.DefaultStorageFinder', 83 | ) 84 | 85 | # Make this unique, and don't share it with anybody. 86 | SECRET_KEY = '7yh44x!g$1ru4!s-j)!l%$4unqk586b2-i8ub%j49d#6zrt5u7' 87 | 88 | # List of callables that know how to import templates from various sources. 89 | TEMPLATE_LOADERS = ( 90 | 'django.template.loaders.filesystem.Loader', 91 | 'django.template.loaders.app_directories.Loader', 92 | # 'django.template.loaders.eggs.Loader', 93 | ) 94 | 95 | MIDDLEWARE_CLASSES = ( 96 | 'django.middleware.common.CommonMiddleware', 97 | 'django.contrib.sessions.middleware.SessionMiddleware', 98 | 'django.middleware.csrf.CsrfViewMiddleware', 99 | 'django.contrib.auth.middleware.AuthenticationMiddleware', 100 | 'django.contrib.messages.middleware.MessageMiddleware', 101 | ) 102 | 103 | ROOT_URLCONF = 'example.urls' 104 | 105 | TEMPLATE_DIRS = ( 106 | # Put strings here, like "/home/html/django_templates" or "C:/www/django/templates". 107 | # Always use forward slashes, even on Windows. 108 | # Don't forget to use absolute paths, not relative paths. 109 | ) 110 | 111 | INSTALLED_APPS = ( 112 | 'django.contrib.auth', 113 | 'django.contrib.contenttypes', 114 | 'django.contrib.sessions', 115 | 'django.contrib.sites', 116 | 'django.contrib.messages', 117 | 'django.contrib.staticfiles', 118 | 'djangosolr', 119 | 'movies' 120 | # Uncomment the next line to enable the admin: 121 | # 'django.contrib.admin', 122 | # Uncomment the next line to enable admin documentation: 123 | # 'django.contrib.admindocs', 124 | ) 125 | 126 | # A sample logging configuration. The only tangible logging 127 | # performed by this configuration is to send an email to 128 | # the site admins on every HTTP 500 error. 129 | # See http://docs.djangoproject.com/en/dev/topics/logging for 130 | # more details on how to customize your logging configuration. 131 | LOGGING = { 132 | 'version': 1, 133 | 'disable_existing_loggers': False, 134 | 'handlers': { 135 | 'mail_admins': { 136 | 'level': 'ERROR', 137 | 'class': 'django.utils.log.AdminEmailHandler' 138 | } 139 | }, 140 | 'loggers': { 141 | 'django.request': { 142 | 'handlers': ['mail_admins'], 143 | 'level': 'ERROR', 144 | 'propagate': True, 145 | }, 146 | } 147 | } 148 | -------------------------------------------------------------------------------- /djangosolr/documents/queryset.py: -------------------------------------------------------------------------------- 1 | from djangosolr.documents.query import Query, Q 2 | from djangosolr import solr 3 | 4 | class QuerySet(object): 5 | 6 | def __init__(self, model, query=None): 7 | self._model = model 8 | self._query = query or Query() 9 | self._responses = [] 10 | self._responses_more = True 11 | self._result_cache = None 12 | self._iter = None 13 | 14 | def _get_responses(self): 15 | for response in self._responses: 16 | yield response 17 | rows = 10 if self._query._rows is None else self._query._rows 18 | start = len(self._responses) * rows if self._query._start is None else self._query._start 19 | while self._responses_more: 20 | query = self._query.clone() 21 | query.set_limits(start, start + rows) 22 | response = solr.select(query.get_query_string(self._model._meta)) 23 | start += rows 24 | self._responses.append(response) 25 | self._responses_more = self._query._start is None and self._query._rows is None and len(response['response']['docs']) == rows 26 | yield response 27 | 28 | def _get_response(self): 29 | return self._get_responses().next() 30 | response = property(_get_response) 31 | 32 | def _clone(self): 33 | return QuerySet(self._model, self._query.clone()) 34 | 35 | def __len__(self): 36 | if self._result_cache is None: 37 | if self._iter: 38 | self._result_cache = list(self._iter) 39 | else: 40 | self._result_cache = list(self.iterator()) 41 | elif self._iter: 42 | self._result_cache.extend(self._iter) 43 | return len(self._result_cache) 44 | 45 | def __iter__(self): 46 | if self._result_cache is None: 47 | self._iter = self.iterator() 48 | self._result_cache = [] 49 | if self._iter: 50 | return self._result_iter() 51 | return iter(self._result_cache) 52 | 53 | def _result_iter(self): 54 | pos = 0 55 | while 1: 56 | upper = len(self._result_cache) 57 | while pos < upper: 58 | yield self._result_cache[pos] 59 | pos = pos + 1 60 | if not self._iter: 61 | raise StopIteration 62 | if len(self._result_cache) <= pos: 63 | self._fill_cache() 64 | 65 | def _fill_cache(self, num=None): 66 | if self._iter: 67 | try: 68 | for _ in range(num or 20): 69 | self._result_cache.append(self._iter.next()) 70 | except StopIteration: 71 | self._iter = None 72 | 73 | def iterator(self): 74 | for response in self._get_responses(): 75 | for doc in response['response']['docs']: 76 | yield self._model.create(doc) 77 | 78 | def __nonzero__(self): 79 | if self._result_cache is not None: 80 | return bool(self._result_cache) 81 | try: 82 | iter(self).next() 83 | except StopIteration: 84 | return False 85 | return True 86 | 87 | def __getitem__(self, k): 88 | if not isinstance(k, (slice, int, long)): 89 | raise TypeError 90 | assert ((not isinstance(k, slice) and (k >= 0)) 91 | or (isinstance(k, slice) and (k.start is None or k.start >= 0) 92 | and (k.stop is None or k.stop >= 0))), \ 93 | "Negative indexing is not supported." 94 | 95 | if self._result_cache is not None: 96 | if self._iter is not None: 97 | if isinstance(k, slice): 98 | if k.stop is not None: 99 | bound = int(k.stop) 100 | else: 101 | bound = None 102 | else: 103 | bound = k + 1 104 | if len(self._result_cache) < bound: 105 | self._fill_cache(bound - len(self._result_cache)) 106 | return self._result_cache[k] 107 | 108 | if isinstance(k, slice): 109 | qs = self._clone() 110 | if k.start is not None: 111 | start = int(k.start) 112 | else: 113 | start = None 114 | if k.stop is not None: 115 | stop = int(k.stop) 116 | else: 117 | stop = None 118 | qs._query.set_limits(start, stop) 119 | return k.step and list(qs)[::k.step] or qs 120 | 121 | try: 122 | qs = self._clone() 123 | qs._query.set_limits(k, k + 1) 124 | return list(qs)[0] 125 | except: 126 | raise IndexError(0) 127 | 128 | def sort(self, *fields): 129 | clone = self._clone() 130 | clone._query.sort(*fields) 131 | return clone 132 | 133 | def fl(self, *fields): 134 | clone = self._clone() 135 | clone._query.fl(*fields) 136 | return clone 137 | 138 | def count(self): 139 | return self.response['response']['numFound'] 140 | 141 | def raw(self, **kwargs): 142 | clone = self._clone() 143 | clone._query.raw(**kwargs) 144 | return clone 145 | 146 | def q(self, *qs, **filters): 147 | clone = self._clone() 148 | clone._query.q(*qs, **filters) 149 | return clone 150 | 151 | def fq(self, *qs, **filters): 152 | clone = self._clone() 153 | clone._query.fq(*qs, **filters) 154 | return clone 155 | 156 | def delete(self, *qs, **filters): 157 | if qs or filters: 158 | return self.q(*qs, **filters).delete() 159 | else: 160 | return solr.delete((self._query._q or Q('*:*')).get_query_string(self._model._meta)) -------------------------------------------------------------------------------- /djangosolr/documents/query.py: -------------------------------------------------------------------------------- 1 | from django.utils import tree 2 | import re 3 | 4 | FILTER_CONTAINS = u'%(field)s:%(value)s' 5 | FILTER_EXACT = u'%(field)s:"%(value)s"' 6 | FILTER_COMPARE = { 7 | 'gt': u'%(field)s:{%(value)s TO *}', 8 | 'gte': u'%(field)s:[%(value)s TO *]', 9 | 'lt': u'%(field)s:{* TO %(value)s}', 10 | 'lte': u'%(field)s:[* TO %(value)s]', 11 | } 12 | FILTER_RANGE = { 13 | 'range': u'%(field)s:[%(from)s TO %(to)s]', 14 | 'rangecc': u'%(field)s:[%(from)s TO %(to)s]', 15 | 'rangeoc': u'(%(field)s:{%(from)s TO *} AND %(field)s:[* TO %(to)s])', 16 | 'rangeco': u'(%(field)s:[%(from)s TO *] AND %(field)s:{* TO %(to)s})', 17 | 'rangeoo': u'%(field)s:{%(from)s TO %(to)s}' 18 | } 19 | WHITESPACE_RE = re.compile(r'\s+') 20 | 21 | class Q(tree.Node): 22 | 23 | AND = 'AND' 24 | OR = 'OR' 25 | default = AND 26 | 27 | def __init__(self, *args, **kwargs): 28 | super(Q, self).__init__(children=list(args) + kwargs.items()) 29 | 30 | def _combine(self, other, conn): 31 | if not isinstance(other, Q): 32 | raise TypeError(other) 33 | obj = Q() 34 | obj.add(self, conn) 35 | obj.add(other, conn) 36 | return obj 37 | 38 | def __or__(self, other): 39 | return self._combine(other, self.OR) 40 | 41 | def __and__(self, other): 42 | return self._combine(other, self.AND) 43 | 44 | def __invert__(self): 45 | obj = Q() 46 | obj.add(self, self.AND) 47 | obj.negate() 48 | 49 | def get_query_string(self, meta): 50 | query = [] 51 | for child in self.children: 52 | if isinstance(child, basestring): 53 | query.append(child) 54 | elif hasattr(child, 'get_query_string'): 55 | query.append(child.get_query_string(meta)) 56 | else: 57 | filterx, value = child 58 | fn, _, ft = filterx.partition('__') 59 | f = meta.get_field(fn) 60 | fn = meta.get_solr_field_name(fn) 61 | if not ft or ft == 'contains': 62 | if isinstance(value, basestring): 63 | queryt = [] 64 | for value in WHITESPACE_RE.split(value): 65 | queryt.append(FILTER_CONTAINS % {'field': fn, 'value': f.prepare_to_query(value)}) 66 | s = u' AND '.join(queryt) 67 | if len(queryt) > 1: 68 | s = u'(%s)' % (s,) 69 | query.append(s) 70 | else: 71 | query.append(FILTER_CONTAINS % {'field':fn, 'value': f.prepare_to_query(value)}) 72 | elif ft == 'exact': 73 | query.append(FILTER_EXACT % {'field': fn, 'value': f.prepare_to_query(value)}) 74 | elif ft in FILTER_COMPARE: 75 | value = u'"%s"' % (f.prepare_to_query(value),) if isinstance(value, basestring) else f.prepare_to_query(value) 76 | query.append(FILTER_COMPARE[ft] % {'field': fn, 'value': value}) 77 | elif ft in FILTER_RANGE: 78 | value1, value2 = value 79 | value1 = u'"%s"' % (f.prepare_to_query(value1),) if isinstance(value1, basestring) else f.prepare_to_query(value1) 80 | value2 = u'"%s"' % (f.prepare_to_query(value2),) if isinstance(value2, basestring) else f.prepare_to_query(value2) 81 | query.append(FILTER_RANGE[ft] % {'field': fn, 'from': value1, 'to': value2}) 82 | elif ft == 'in': 83 | query.append(u'(%s)' % (' OR '.join([u'%s:%s' % (fn, f.prepare_to_query(v),) for v in value]),)) 84 | else: 85 | raise NotImplementedError 86 | s = (u' %s ' % (self.connector,)).join(filter(lambda y: y, query)) 87 | if self.negated: 88 | s = u'NOT (%s)' % (s,) 89 | elif len(self.children) > 1: 90 | s = u'(%s)' % (s,) 91 | return s 92 | 93 | class Query(object): 94 | 95 | def __init__(self): 96 | self._q = Q() 97 | self._fq = Q() 98 | self._sort = [] 99 | self._fl = [] 100 | self._start = None 101 | self._rows = None 102 | self._params = [] 103 | 104 | def clone(self): 105 | clone = Query() 106 | clone._q = self._q 107 | clone._fq = self._fq 108 | clone._sort.extend(self._sort) 109 | clone._fl.extend(self._fl) 110 | clone._start = self._start 111 | clone._rows = self._rows 112 | clone._params.extend(self._params) 113 | return clone 114 | 115 | def q(self, *qs, **filters): 116 | for q in qs: 117 | self._q &= q 118 | if filters: 119 | self._q &= Q(**filters) 120 | 121 | def fq(self, *qs, **filters): 122 | for q in qs: 123 | self._fq &= q 124 | if filters: 125 | self._fq &= Q(**filters) 126 | 127 | def fl(self, *fields): 128 | self._fl.extend(fields) 129 | 130 | def sort(self, *fields): 131 | self._sort.extend(fields) 132 | 133 | def raw(self, **kwargs): 134 | for k, v in kwargs.items(): 135 | self._params.append((k,v,)) 136 | 137 | def set_limits(self, start, stop): 138 | self._start = start 139 | if stop is not None: 140 | self._rows = stop - (self._start or 0) 141 | else: 142 | self._rows = None 143 | 144 | def get_query_string(self, meta): 145 | query = [] 146 | 147 | #start/rows 148 | if self._start is not None: 149 | query.append(('start', self._start,)) 150 | if self._rows is not None: 151 | query.append(('rows', self._rows,)) 152 | 153 | #q 154 | if not self._q: 155 | self._q = Q('*:*') 156 | query.append(('q', self._q.get_query_string(meta),)) 157 | 158 | #fq 159 | self._fq &= Q('%s:%s' % (meta.get_solr_type_field(), meta.get_solr_type_value(),)) 160 | query.append(('fq', self._fq.get_query_string(meta),)) 161 | 162 | #sort 163 | if self._sort: 164 | sort = ','.join(['%s desc' % (meta.get_solr_field_name(field[1:]),) if field.startswith('-') 165 | else '%s asc' % (meta.get_solr_field_name(field),) 166 | for field in self._sort]) 167 | query.append(('sort', sort,)) 168 | 169 | #fl 170 | if self._fl: 171 | query.append(('fl', ','.join([meta.get_solr_field_name(f) for f in self._fl]),)) 172 | 173 | #raw params 174 | query.extend(self._params) 175 | 176 | return query 177 | --------------------------------------------------------------------------------