├── docs ├── requirements.txt ├── authors.rst ├── history.rst ├── contributing.rst ├── index.rst ├── usage.rst ├── make.bat ├── Makefile └── conf.py ├── setup.cfg ├── simple_elasticsearch ├── management │ ├── __init__.py │ └── commands │ │ ├── __init__.py │ │ └── es_manage.py ├── exceptions.py ├── __init__.py ├── signals.py ├── settings.py ├── models.py ├── mixins.py ├── search.py ├── utils.py └── tests.py ├── AUTHORS.rst ├── MANIFEST.in ├── .editorconfig ├── .travis.yml ├── .gitignore ├── tox.ini ├── runtests.py ├── LICENSE ├── test_settings.py ├── Makefile ├── setup.py ├── README.rst ├── CONTRIBUTING.rst └── HISTORY.rst /docs/requirements.txt: -------------------------------------------------------------------------------- 1 | alabaster==0.6.1 -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [wheel] 2 | universal = 1 3 | -------------------------------------------------------------------------------- /simple_elasticsearch/management/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /docs/authors.rst: -------------------------------------------------------------------------------- 1 | .. include:: ../AUTHORS.rst 2 | -------------------------------------------------------------------------------- /docs/history.rst: -------------------------------------------------------------------------------- 1 | .. include:: ../HISTORY.rst 2 | -------------------------------------------------------------------------------- /simple_elasticsearch/management/commands/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /docs/contributing.rst: -------------------------------------------------------------------------------- 1 | .. include:: ../CONTRIBUTING.rst 2 | -------------------------------------------------------------------------------- /simple_elasticsearch/exceptions.py: -------------------------------------------------------------------------------- 1 | 2 | class MissingObjectError(Exception): 3 | pass 4 | -------------------------------------------------------------------------------- /simple_elasticsearch/__init__.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | __author__ = 'James Addison' 4 | __email__ = 'addi00+github.com@gmail.com' 5 | __version__ = '2.2.1' 6 | -------------------------------------------------------------------------------- /AUTHORS.rst: -------------------------------------------------------------------------------- 1 | ======= 2 | Credits 3 | ======= 4 | 5 | Development Lead 6 | ---------------- 7 | 8 | * James Addison 9 | 10 | Contributors 11 | ------------ 12 | 13 | None yet. Why not be the first? 14 | -------------------------------------------------------------------------------- /simple_elasticsearch/signals.py: -------------------------------------------------------------------------------- 1 | import django.dispatch 2 | 3 | 4 | post_indices_create = django.dispatch.Signal(providing_args=["indices", "aliases_set"]) 5 | post_indices_rebuild = django.dispatch.Signal(providing_args=["indices", "aliases_set"]) 6 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include AUTHORS.rst 2 | include CONTRIBUTING.rst 3 | include HISTORY.rst 4 | include LICENSE 5 | include README.rst 6 | 7 | recursive-include docs *.rst conf.py Makefile make.bat 8 | recursive-include tests * 9 | global-exclude *.py[cod] __pycache__ *.so 10 | 11 | -------------------------------------------------------------------------------- /.editorconfig: -------------------------------------------------------------------------------- 1 | # http://editorconfig.org 2 | 3 | root = true 4 | 5 | [*] 6 | indent_style = space 7 | indent_size = 4 8 | trim_trailing_whitespace = true 9 | insert_final_newline = true 10 | charset = utf-8 11 | end_of_line = lf 12 | 13 | [*.bat] 14 | indent_style = tab 15 | end_of_line = crlf 16 | 17 | [LICENSE] 18 | insert_final_newline = false 19 | 20 | [Makefile] 21 | indent_style = tab 22 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | sudo: false 2 | language: python 3 | python: 4 | - "2.7" 5 | - "3.2" 6 | - "3.3" 7 | - "3.4" 8 | - "3.5" 9 | - "3.6" 10 | install: 11 | # needed for python 3.2 tests - virtualenv>=14.0.0 dropped python 3.2 support 12 | # coverage>=4.0 doesn't support python 3.2 properly either 13 | - travis_retry pip install "virtualenv<14.0.0" tox-travis 14 | script: tox 15 | after_success: coveralls 16 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.py[cod] 2 | 3 | # C extensions 4 | *.so 5 | 6 | # Packages 7 | *.egg 8 | *.egg-info 9 | dist 10 | build 11 | eggs 12 | parts 13 | bin 14 | var 15 | sdist 16 | develop-eggs 17 | .installed.cfg 18 | lib 19 | lib64 20 | 21 | # Installer logs 22 | pip-log.txt 23 | 24 | # Unit test / coverage reports 25 | .coverage 26 | .tox 27 | nosetests.xml 28 | htmlcov 29 | 30 | .Python 31 | .DS_Store 32 | .idea 33 | 34 | # Sphinx 35 | docs/_build 36 | -------------------------------------------------------------------------------- /tox.ini: -------------------------------------------------------------------------------- 1 | [tox] 2 | envlist = 3 | py{27,32,33,34,35}-django18, 4 | py{27,34,35}-django{19,110,111} 5 | py{36}-django{111} 6 | 7 | [tox:travis] 8 | 2.7 = py27 9 | 3.2 = py32 10 | 3.3 = py33 11 | 3.4 = py34 12 | 3.5 = py35 13 | 3.6 = py36 14 | 15 | [testenv] 16 | deps= 17 | django18: Django>=1.8,<1.9 18 | django19: Django>=1.9,<1.10 19 | django110: Django>=1.10,<1.11 20 | django111: Django>=1.11,<2.0 21 | coverage<4.0 22 | commands = coverage run --source=simple_elasticsearch setup.py test 23 | -------------------------------------------------------------------------------- /docs/index.rst: -------------------------------------------------------------------------------- 1 | .. django-simple-elasticsearch documentation master file, created by 2 | sphinx-quickstart on Tue Jul 9 22:26:36 2013. 3 | You can adapt this file completely to your liking, but it should at least 4 | contain the root `toctree` directive. 5 | 6 | .. include:: ../README.rst 7 | 8 | Jump to a section 9 | ----------------- 10 | 11 | .. toctree:: 12 | :maxdepth: 2 13 | 14 | usage 15 | contributing 16 | authors 17 | history 18 | 19 | Indices and tables 20 | ================== 21 | 22 | * :ref:`genindex` 23 | * :ref:`modindex` 24 | * :ref:`search` 25 | 26 | -------------------------------------------------------------------------------- /runtests.py: -------------------------------------------------------------------------------- 1 | # This file mainly exists to allow python setup.py test to work. 2 | import os 3 | import sys 4 | 5 | os.environ['DJANGO_SETTINGS_MODULE'] = 'test_settings' 6 | test_dir = os.path.dirname(__file__) 7 | sys.path.insert(0, test_dir) 8 | 9 | import django 10 | from django.test.utils import get_runner 11 | from django.conf import settings 12 | 13 | 14 | def runtests(): 15 | TestRunner = get_runner(settings) 16 | test_runner = TestRunner(verbosity=1, interactive=True) 17 | if hasattr(django, 'setup'): 18 | django.setup() 19 | failures = test_runner.run_tests(['simple_elasticsearch']) 20 | sys.exit(bool(failures)) 21 | 22 | if __name__ == '__main__': 23 | runtests() 24 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2014, James Addison 2 | All rights reserved. 3 | 4 | Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 5 | 6 | * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 7 | 8 | * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. 9 | 10 | * Neither the name of Django Simple Elasticsearch nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. 11 | 12 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -------------------------------------------------------------------------------- /test_settings.py: -------------------------------------------------------------------------------- 1 | """ 2 | Django test settings for django-simple-elasticsearch app. 3 | """ 4 | 5 | import os 6 | import sys 7 | 8 | # Build paths inside the project like this: os.path.join(BASE_DIR, ...) 9 | # BASE_DIR = os.path.dirname(os.path.dirname(__file__)) 10 | 11 | IS_DEV = False 12 | IS_STAGING = False 13 | IS_PROD = False 14 | IS_TEST = 'test' in sys.argv or 'test_coverage' in sys.argv 15 | 16 | SECRET_KEY = 'c!66li$a&_7q6o)-*va5#mbdmdizyvi4qo6!u90=l%$5pk68+-' 17 | ALLOWED_HOSTS = [] 18 | 19 | # Application definition 20 | 21 | INSTALLED_APPS = ( 22 | 'django.contrib.admin', 23 | 'django.contrib.auth', 24 | 'django.contrib.contenttypes', 25 | 'django.contrib.sessions', 26 | 'django.contrib.messages', 27 | 'django.contrib.staticfiles', 28 | 29 | 'simple_elasticsearch' 30 | ) 31 | 32 | if IS_TEST: 33 | ELASTICSEARCH_TYPE_CLASSES = [ 34 | 'simple_elasticsearch.models.BlogPost' 35 | ] 36 | 37 | MIDDLEWARE_CLASSES = ( 38 | 'django.contrib.sessions.middleware.SessionMiddleware', 39 | 'django.middleware.common.CommonMiddleware', 40 | 'django.middleware.csrf.CsrfViewMiddleware', 41 | 'django.contrib.auth.middleware.AuthenticationMiddleware', 42 | 'django.contrib.auth.middleware.SessionAuthenticationMiddleware', 43 | 'django.contrib.messages.middleware.MessageMiddleware', 44 | 'django.middleware.clickjacking.XFrameOptionsMiddleware', 45 | ) 46 | 47 | DATABASES = { 48 | 'default': { 49 | 'ENGINE': 'django.db.backends.sqlite3', 50 | 'NAME': ':memory:', 51 | } 52 | } 53 | 54 | LANGUAGE_CODE = 'en-us' 55 | TIME_ZONE = 'UTC' 56 | USE_I18N = True 57 | USE_L10N = True 58 | USE_TZ = True 59 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | .PHONY: clean-pyc clean-build docs clean 2 | 3 | help: 4 | @echo "clean - remove all build, test, coverage and Python artifacts" 5 | @echo "clean-build - remove build artifacts" 6 | @echo "clean-pyc - remove Python file artifacts" 7 | @echo "clean-test - remove test and coverage artifacts" 8 | @echo "lint - check style with flake8" 9 | @echo "test - run tests quickly with the default Python" 10 | @echo "test-all - run tests on every Python version with tox" 11 | @echo "coverage - check code coverage quickly with the default Python" 12 | @echo "docs - generate Sphinx HTML documentation, including API docs" 13 | @echo "release - package and upload a release" 14 | @echo "dist - package" 15 | 16 | clean: clean-build clean-pyc clean-test 17 | 18 | clean-build: 19 | rm -fr build/ 20 | rm -fr dist/ 21 | rm -fr *.egg-info 22 | 23 | clean-pyc: 24 | find . -name '*.pyc' -exec rm -f {} + 25 | find . -name '*.pyo' -exec rm -f {} + 26 | find . -name '*~' -exec rm -f {} + 27 | find . -name '__pycache__' -exec rm -fr {} + 28 | 29 | clean-test: 30 | rm -fr .tox/ 31 | rm -f .coverage 32 | rm -fr htmlcov/ 33 | 34 | lint: 35 | flake8 django-simple-elasticsearch tests 36 | 37 | test: 38 | python setup.py test 39 | 40 | test-all: 41 | tox 42 | 43 | coverage: 44 | coverage run --source django-simple-elasticsearch setup.py test 45 | coverage report -m 46 | coverage html 47 | open htmlcov/index.html 48 | 49 | docs: 50 | rm -f docs/django-simple-elasticsearch.rst 51 | rm -f docs/modules.rst 52 | sphinx-apidoc -o docs/ django-simple-elasticsearch 53 | $(MAKE) -C docs clean 54 | $(MAKE) -C docs html 55 | open docs/_build/html/index.html 56 | 57 | release: clean 58 | python setup.py sdist upload 59 | python setup.py bdist_wheel upload 60 | 61 | dist: clean 62 | python setup.py sdist 63 | python setup.py bdist_wheel 64 | ls -l dist 65 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | 5 | try: 6 | from setuptools import setup 7 | except ImportError: 8 | from distutils.core import setup 9 | 10 | import simple_elasticsearch 11 | 12 | readme = open('README.rst').read() 13 | history = open('HISTORY.rst').read().replace('.. :changelog:', '') 14 | 15 | requirements = [ 16 | 'elasticsearch>=2.0.0', 17 | ] 18 | 19 | test_requirements = [ 20 | 'datadiff>=1.1.6', 21 | 'mock>=1.0.1' 22 | ] 23 | 24 | setup( 25 | name='django-simple-elasticsearch', 26 | version=simple_elasticsearch.__version__, 27 | description='Simple ElasticSearch indexing integration for Django.', 28 | long_description=readme + '\n\n' + history, 29 | author='James Addison', 30 | author_email='addi00+github.com@gmail.com', 31 | url='https://github.com/jaddison/django-simple-elasticsearch', 32 | packages=[ 33 | 'simple_elasticsearch', 34 | 'simple_elasticsearch.management', 35 | 'simple_elasticsearch.management.commands', 36 | ], 37 | package_dir={'django-simple-elasticsearch': 38 | 'django-simple-elasticsearch'}, 39 | include_package_data=True, 40 | install_requires=requirements, 41 | license="BSD", 42 | zip_safe=False, 43 | keywords='django simple elasticsearch search indexing', 44 | classifiers=[ 45 | 'Development Status :: 4 - Beta', 46 | 'License :: OSI Approved :: BSD License', 47 | 'Intended Audience :: Developers', 48 | 'Environment :: Web Environment', 49 | 'Programming Language :: Python', 50 | 'Framework :: Django', 51 | 'Topic :: Internet :: WWW/HTTP :: WSGI', 52 | 'Natural Language :: English', 53 | "Programming Language :: Python :: 2", 54 | 'Programming Language :: Python :: 2.7', 55 | 'Programming Language :: Python :: 3', 56 | 'Programming Language :: Python :: 3.3', 57 | 'Programming Language :: Python :: 3.4', 58 | 'Programming Language :: Python :: 3.5', 59 | 'Programming Language :: Python :: 3.6', 60 | ], 61 | test_suite='runtests.runtests', 62 | tests_require=test_requirements 63 | ) 64 | -------------------------------------------------------------------------------- /simple_elasticsearch/settings.py: -------------------------------------------------------------------------------- 1 | from django.conf import settings 2 | 3 | 4 | ELASTICSEARCH_SERVER = getattr(settings, 'ELASTICSEARCH_SERVER', ['127.0.0.1:9200', ]) 5 | ELASTICSEARCH_CONNECTION_PARAMS = getattr(settings, 'ELASTICSEARCH_CONNECTION_PARAMS', {'hosts': ELASTICSEARCH_SERVER}) 6 | 7 | # Override this if you want to have a base set of settings for all your indexes. This dictionary 8 | # gets cloned and then updated with custom index-specific from your ELASTICSEARCH_CUSTOM_INDEX_SETTINGS 9 | # Eg. to ensure that all of your indexes have 1 shard and have an edgengram tokenizer/analyzer 10 | # configured 11 | # ELASTICSEARCH_DEFAULT_INDEX_SETTINGS = { 12 | # "settings" : { 13 | # "index" : { 14 | # "number_of_replicas" : 1 15 | # }, 16 | # "analysis" : { 17 | # "analyzer" : { 18 | # "left" : { 19 | # "filter" : [ 20 | # "standard", 21 | # "lowercase", 22 | # "stop" 23 | # ], 24 | # "type" : "custom", 25 | # "tokenizer" : "left_tokenizer" 26 | # } 27 | # }, 28 | # "tokenizer" : { 29 | # "left_tokenizer" : { 30 | # "side" : "front", 31 | # "max_gram" : 12, 32 | # "type" : "edgeNGram" 33 | # } 34 | # } 35 | # } 36 | # } 37 | # } 38 | ELASTICSEARCH_DEFAULT_INDEX_SETTINGS = getattr(settings, 'ELASTICSEARCH_DEFAULT_INDEX_SETTINGS', {}) 39 | 40 | # Override this in your project settings to define any Elasticsearch-specific index settings. 41 | # Eg. 42 | # ELASTICSEARCH_CUSTOM_INDEX_SETTINGS = { 43 | # "twitter": { 44 | # "settings" : { 45 | # "index" : { 46 | # "number_of_shards" : 3, 47 | # } 48 | # } 49 | # }, 50 | # "": { 51 | # "settings" : { 52 | # "index" : { 53 | # "number_of_shards" : 50, 54 | # "number_of_replicas" : 2 55 | # } 56 | # } 57 | # } 58 | # } 59 | ELASTICSEARCH_CUSTOM_INDEX_SETTINGS = getattr(settings, 'ELASTICSEARCH_CUSTOM_INDEX_SETTINGS', {}) 60 | 61 | # Override this in your project settings, setting it to True, to have 62 | # old indexes deleted on a full rebuild. Currently a new index is 63 | # created, and the alias is switched to the new one from the old, leaving 64 | # old ones on the ES cluster. 65 | ELASTICSEARCH_DELETE_OLD_INDEXES = getattr(settings, 'ELASTICSEARCH_DELETE_OLD_INDEXES', False) 66 | -------------------------------------------------------------------------------- /simple_elasticsearch/models.py: -------------------------------------------------------------------------------- 1 | from django.conf import settings 2 | 3 | 4 | if getattr(settings, 'IS_TEST', False): 5 | from django.db import models 6 | from django.db.models.signals import post_save, pre_delete 7 | 8 | from .mixins import ElasticsearchTypeMixin 9 | 10 | class Blog(models.Model): 11 | name = models.CharField(max_length=50) 12 | description = models.TextField() 13 | 14 | class BlogPost(models.Model, ElasticsearchTypeMixin): 15 | blog = models.ForeignKey(Blog) 16 | slug = models.SlugField() 17 | title = models.CharField(max_length=50) 18 | body = models.TextField() 19 | created_at = models.DateTimeField(auto_now_add=True) 20 | bulk_ordering = 'pk' 21 | 22 | @classmethod 23 | def get_bulk_ordering(cls): 24 | return cls.bulk_ordering 25 | 26 | @classmethod 27 | def get_bulk_index_limit(cls): 28 | return 2 29 | 30 | @classmethod 31 | def get_queryset(cls): 32 | return BlogPost.objects.all().select_related('blog') 33 | 34 | @classmethod 35 | def get_index_name(cls): 36 | return 'blog' 37 | 38 | @classmethod 39 | def get_type_name(cls): 40 | return 'posts' 41 | 42 | @classmethod 43 | def get_request_params(cls, obj): 44 | return {'routing': obj.blog_id} 45 | 46 | @classmethod 47 | def get_type_mapping(cls): 48 | return { 49 | "properties": { 50 | "created_at": { 51 | "type": "date", 52 | "format": "dateOptionalTime" 53 | }, 54 | "title": { 55 | "type": "string" 56 | }, 57 | "body": { 58 | "type": "string" 59 | }, 60 | "slug": { 61 | "type": "string" 62 | }, 63 | "blog": { 64 | "properties": { 65 | "id": { 66 | "type": "long" 67 | }, 68 | "name": { 69 | "type": "string" 70 | }, 71 | "description": { 72 | "type": "string" 73 | } 74 | } 75 | } 76 | } 77 | } 78 | 79 | @classmethod 80 | def get_document(cls, obj): 81 | return { 82 | 'created_at': obj.created_at, 83 | 'title': obj.title, 84 | 'body': obj.body, 85 | 'slug': obj.slug, 86 | 'blog': { 87 | 'id': obj.blog.id, 88 | 'name': obj.blog.name, 89 | 'description': obj.blog.description, 90 | } 91 | } 92 | 93 | @classmethod 94 | def should_index(cls, obj): 95 | return obj.slug != 'DO-NOT-INDEX' 96 | 97 | post_save.connect(BlogPost.save_handler, sender=BlogPost) 98 | pre_delete.connect(BlogPost.delete_handler, sender=BlogPost) 99 | -------------------------------------------------------------------------------- /README.rst: -------------------------------------------------------------------------------- 1 | =========================== 2 | Django Simple Elasticsearch 3 | =========================== 4 | 5 | .. image:: https://badge.fury.io/py/django-simple-elasticsearch.png 6 | :target: http://badge.fury.io/py/django-simple-elasticsearch 7 | 8 | .. image:: https://travis-ci.org/jaddison/django-simple-elasticsearch.png 9 | :target: https://travis-ci.org/jaddison/django-simple-elasticsearch 10 | 11 | .. image:: https://coveralls.io/repos/jaddison/django-simple-elasticsearch/badge.png 12 | :target: https://coveralls.io/r/jaddison/django-simple-elasticsearch 13 | 14 | 15 | This package provides a simple method of creating Elasticsearch indexes for 16 | Django models. 17 | 18 | ----- 19 | 20 | Versions 21 | -------- 22 | 23 | Branch :code:`master` targets both Elasticsearch 2.x and 5.x and will receive new 24 | features. Both `elasticsearch-py` 2.x and 5.x Python modules are currently 25 | supported. `Documentation `_ 26 | 27 | Branch :code:`1.x` is the maintenance branch for the legacy 0.9.x versioned releases, 28 | which targeted Elasticsearch versions less than 2.0. This branch is unlikely to 29 | receive new features, but will receive required fixes. 30 | `Documentation `_ 31 | 32 | **Using a version older than 0.9.0? Please be aware that as of v0.9.0, this package 33 | has changed in a backwards-incompatible manner. Version 0.5 is deprecated and no 34 | longer maintained.** 35 | 36 | ----- 37 | 38 | Documentation 39 | ------------- 40 | 41 | Visit the `django-simple-elasticsearch documentation on ReadTheDocs `_. 42 | 43 | Features 44 | -------- 45 | 46 | * class mixin with a set of :code:`@classmethods` used to handle: 47 | * type mapping definition 48 | * individual object indexing and deletion 49 | * bulk object indexing 50 | * model signal handlers for pre/post_save and pre/post_delete (optional) 51 | * management command to handle index/type mapping initialization and bulk indexing 52 | * uses Elasticsearch aliases to ease the burden of re-indexing 53 | * small set of Django classes and functions to help deal with Elasticsearch querying 54 | * base search form class to handle input validation, query preparation and response handling 55 | * multi-search processor class to batch multiple Elasticsearch queries via :code:`_msearch` 56 | * 'get' shortcut functions 57 | * post index create/rebuild signals available to perform actions after certain stages (ie. add your own percolators) 58 | 59 | Installation 60 | ------------ 61 | 62 | At the command line:: 63 | 64 | $ easy_install django-simple-elasticsearch 65 | 66 | Or:: 67 | 68 | $ pip install django-simple-elasticsearch 69 | 70 | Configuring 71 | ----------- 72 | 73 | Add the simple_elasticsearch application to your INSTALLED_APPS list:: 74 | 75 | INSTALLED_APPS = ( 76 | ... 77 | 'simple_elasticsearch', 78 | ) 79 | 80 | Add any models to `ELASTICSEARCH_TYPE_CLASSES` setting for indexing using **es_manage** management command:: 81 | 82 | ELASTICSEARCH_TYPE_CLASSES = [ 83 | 'blog.models.BlogPost' 84 | ] 85 | 86 | License 87 | ------- 88 | 89 | **django-simple-elasticsearch** is licensed as free software under the BSD license. 90 | 91 | Todo 92 | ---- 93 | 94 | * Review search classes - simplify functionality where possible. This may cause breaking changes. 95 | * Tests. Write them. 96 | * Documentation. Write it. 97 | -------------------------------------------------------------------------------- /CONTRIBUTING.rst: -------------------------------------------------------------------------------- 1 | ============ 2 | Contributing 3 | ============ 4 | 5 | Contributions are welcome, and they are greatly appreciated! Every 6 | little bit helps, and credit will always be given. 7 | 8 | You can contribute in many ways: 9 | 10 | Types of Contributions 11 | ---------------------- 12 | 13 | Report Bugs 14 | ~~~~~~~~~~~ 15 | 16 | Report bugs at https://github.com/jaddison/django-simple-elasticsearch/issues. 17 | 18 | If you are reporting a bug, please include: 19 | 20 | * Your operating system name and version. 21 | * Any details about your local setup that might be helpful in troubleshooting. 22 | * Detailed steps to reproduce the bug. 23 | 24 | Fix Bugs 25 | ~~~~~~~~ 26 | 27 | Look through the GitHub issues for bugs. Anything tagged with "bug" 28 | is open to whoever wants to implement it. 29 | 30 | Implement Features 31 | ~~~~~~~~~~~~~~~~~~ 32 | 33 | Look through the GitHub issues for features. Anything tagged with "feature" 34 | is open to whoever wants to implement it. 35 | 36 | Write Documentation 37 | ~~~~~~~~~~~~~~~~~~~ 38 | 39 | Django Simple Elasticsearch could always use more documentation, whether as part of the 40 | official Django Simple Elasticsearch docs, in docstrings, or even on the web in blog posts, 41 | articles, and such. 42 | 43 | Submit Feedback 44 | ~~~~~~~~~~~~~~~ 45 | 46 | The best way to send feedback is to file an issue at https://github.com/jaddison/django-simple-elasticsearch/issues. 47 | 48 | If you are proposing a feature: 49 | 50 | * Explain in detail how it would work. 51 | * Keep the scope as narrow as possible, to make it easier to implement. 52 | * Remember that this is a volunteer-driven project, and that contributions 53 | are welcome :) 54 | 55 | Get Started! 56 | ------------ 57 | 58 | Ready to contribute? Here's how to set up `django-simple-elasticsearch` for local development. 59 | 60 | 1. Fork the `django-simple-elasticsearch` repo on GitHub. 61 | 2. Clone your fork locally:: 62 | 63 | $ git clone git@github.com:your_name_here/django-simple-elasticsearch.git 64 | 65 | 3. Install your local copy into a virtualenv. Assuming you have virtualenvwrapper installed, this is how you set up your fork for local development:: 66 | 67 | $ mkvirtualenv django-simple-elasticsearch 68 | $ cd django-simple-elasticsearch/ 69 | $ python setup.py develop 70 | 71 | 4. Create a branch for local development:: 72 | 73 | $ git checkout -b name-of-your-bugfix-or-feature 74 | 75 | Now you can make your changes locally. 76 | 77 | 5. When you're done making changes, check that your changes pass flake8 and the tests, including testing other Python versions with tox:: 78 | 79 | $ flake8 simple_elasticsearch 80 | $ python setup.py test 81 | $ tox 82 | 83 | To get flake8 and tox, just pip install them into your virtualenv. 84 | 85 | 6. Commit your changes and push your branch to GitHub:: 86 | 87 | $ git add . 88 | $ git commit -m "Your detailed description of your changes." 89 | $ git push origin name-of-your-bugfix-or-feature 90 | 91 | 7. Submit a pull request through the GitHub website. 92 | 93 | Pull Request Guidelines 94 | ----------------------- 95 | 96 | Before you submit a pull request, check that it meets these guidelines: 97 | 98 | 1. The pull request should include tests. 99 | 2. If the pull request adds functionality, the docs should be updated. Put 100 | your new functionality into a function with a docstring, and add the 101 | feature to the list in README.rst. 102 | 3. The pull request should work for Python 2.6, 2.7, 3.3, and 3.4, and for PyPy. Check 103 | https://travis-ci.org/jaddison/django-simple-elasticsearch/pull_requests 104 | and make sure that the tests pass for all supported Python versions. 105 | -------------------------------------------------------------------------------- /simple_elasticsearch/management/commands/es_manage.py: -------------------------------------------------------------------------------- 1 | import sys 2 | from django.conf import settings 3 | from django.core.management.base import BaseCommand, CommandError 4 | 5 | from ...utils import get_indices, create_indices, rebuild_indices, delete_indices 6 | 7 | try: 8 | raw_input 9 | except NameError: 10 | raw_input = input 11 | 12 | 13 | class Unbuffered(object): 14 | def __init__(self, stream): 15 | self.stream = stream 16 | def write(self, data): 17 | self.stream.write(data) 18 | self.stream.flush() 19 | def __getattr__(self, attr): 20 | return getattr(self.stream, attr) 21 | sys.stdout = Unbuffered(sys.stdout) 22 | 23 | 24 | class ESCommandError(CommandError): 25 | pass 26 | 27 | 28 | class Command(BaseCommand): 29 | help = '' 30 | 31 | def add_arguments(self, parser): 32 | parser.add_argument('--list', action='store_true', dest='list', default=False) 33 | parser.add_argument('--initialize', action='store_true', dest='initialize', default=False) 34 | parser.add_argument('--rebuild', action='store_true', dest='rebuild', default=False) 35 | parser.add_argument('--cleanup', action='store_true', dest='cleanup', default=False) 36 | parser.add_argument('--no_input', '--noinput', action='store_true', dest='no_input', default=False) 37 | parser.add_argument('--indexes', action='store', dest='indexes', default='') 38 | 39 | def handle(self, *args, **options): 40 | no_input = options.get('no_input') 41 | 42 | requested_indexes = options.get('indexes', '') or [] 43 | if requested_indexes: 44 | requested_indexes = requested_indexes.split(',') 45 | 46 | if options.get('list'): 47 | self.subcommand_list() 48 | elif options.get('initialize'): 49 | self.subcommand_initialize(requested_indexes, no_input) 50 | elif options.get('rebuild'): 51 | self.subcommand_rebuild(requested_indexes, no_input) 52 | elif options.get('cleanup'): 53 | self.subcommand_cleanup(requested_indexes, no_input) 54 | 55 | def subcommand_list(self): 56 | print("Available ES indexes:") 57 | for index_name, type_classes in get_indices().items(): 58 | print(" - index '{0}':".format(index_name)) 59 | for type_class in type_classes: 60 | print(" - type '{0}'".format(type_class.get_type_name())) 61 | 62 | def subcommand_initialize(self, indexes=None, no_input=False): 63 | user_input = 'y' if no_input else '' 64 | while user_input != 'y': 65 | user_input = raw_input('Are you sure you want to initialize {0} index(es)? [y/N]: '.format('the ' + ', '.join(indexes) if indexes else '**ALL**')).lower() 66 | if user_input == 'n': 67 | break 68 | 69 | if user_input == 'y': 70 | sys.stdout.write("Creating ES indexes: ") 71 | results, aliases = create_indices(indices=indexes) 72 | sys.stdout.write("complete.\n") 73 | for alias, index in aliases: 74 | print("'{0}' aliased to '{1}'".format(alias, index)) 75 | 76 | def subcommand_cleanup(self, indexes=None, no_input=False): 77 | user_input = 'y' if no_input else '' 78 | while user_input != 'y': 79 | user_input = raw_input('Are you sure you want to clean up (ie DELETE) {0} index(es)? [y/N]: '.format('the ' + ', '.join(indexes) if indexes else '**ALL**')).lower() 80 | if user_input == 'n': 81 | break 82 | 83 | if user_input == 'y': 84 | sys.stdout.write("Deleting ES indexes: ") 85 | indices = delete_indices(indices=indexes) 86 | sys.stdout.write("complete.\n") 87 | for index in indices: 88 | print("'{0}' index deleted".format(index)) 89 | else: 90 | print("{0} removed.".format(len(indices))) 91 | 92 | def subcommand_rebuild(self, indexes, no_input=False): 93 | if getattr(settings, 'DEBUG', False): 94 | import warnings 95 | warnings.warn('Rebuilding with `settings.DEBUG = True` can result in out of memory crashes. See https://docs.djangoproject.com/en/stable/ref/settings/#debug', stacklevel=2) 96 | 97 | # make sure the user continues explicitly after seeing this warning 98 | no_input = False 99 | 100 | user_input = 'y' if no_input else '' 101 | while user_input != 'y': 102 | user_input = raw_input('Are you sure you want to rebuild {0} index(es)? [y/N]: '.format('the ' + ', '.join(indexes) if indexes else '**ALL**')).lower() 103 | if user_input in ['n', '']: 104 | break 105 | 106 | if user_input == 'y': 107 | sys.stdout.write("Rebuilding ES indexes: ") 108 | results, aliases = rebuild_indices(indices=indexes) 109 | sys.stdout.write("complete.\n") 110 | for alias, index in aliases: 111 | print("'{0}' rebuilt and aliased to '{1}'".format(alias, index)) 112 | else: 113 | print("You chose not to rebuild indices.") 114 | -------------------------------------------------------------------------------- /simple_elasticsearch/mixins.py: -------------------------------------------------------------------------------- 1 | from elasticsearch import Elasticsearch, TransportError 2 | 3 | from . import settings as es_settings 4 | from .exceptions import MissingObjectError 5 | from .utils import queryset_iterator 6 | 7 | 8 | class ElasticsearchTypeMixin(object): 9 | queryset_ordering = 'pk' 10 | queryset_limit = 100 11 | bulk_index_limit = 100 12 | 13 | @classmethod 14 | def get_es(cls): 15 | if not hasattr(cls, '_es'): 16 | cls._es = Elasticsearch(**cls.get_es_connection_settings()) 17 | return cls._es 18 | 19 | @classmethod 20 | def get_es_connection_settings(cls): 21 | return es_settings.ELASTICSEARCH_CONNECTION_PARAMS 22 | 23 | @classmethod 24 | def get_index_name(cls): 25 | raise NotImplementedError 26 | 27 | @classmethod 28 | def get_type_name(cls): 29 | raise NotImplementedError 30 | 31 | @classmethod 32 | def get_document(cls, obj): 33 | raise NotImplementedError 34 | 35 | @classmethod 36 | def get_document_id(cls, obj): 37 | if not obj: 38 | raise MissingObjectError 39 | return obj.pk 40 | 41 | @classmethod 42 | def get_request_params(cls, obj): 43 | return {} 44 | 45 | @classmethod 46 | def get_type_mapping(cls): 47 | return {} 48 | 49 | @classmethod 50 | def get_queryset(cls): 51 | raise NotImplementedError 52 | 53 | @classmethod 54 | def get_bulk_index_limit(cls): 55 | return cls.bulk_index_limit 56 | 57 | @classmethod 58 | def get_bulk_ordering(cls): 59 | return cls.queryset_ordering 60 | 61 | @classmethod 62 | def get_query_limit(cls): 63 | return cls.queryset_limit 64 | 65 | @classmethod 66 | def should_index(cls, obj): 67 | return True 68 | 69 | @classmethod 70 | def bulk_index(cls, es=None, index_name='', queryset=None): 71 | es = es or cls.get_es() 72 | 73 | tmp = [] 74 | 75 | if queryset is None: 76 | queryset = cls.get_queryset() 77 | 78 | bulk_limit = cls.get_bulk_index_limit() 79 | 80 | # this requires that `get_queryset` is implemented 81 | for i, obj in enumerate(queryset_iterator(queryset, cls.get_query_limit(), cls.get_bulk_ordering())): 82 | delete = not cls.should_index(obj) 83 | 84 | doc = {} 85 | if not delete: 86 | # allow for the case where a document cannot be indexed; 87 | # the implementation of `get_document()` should return a 88 | # falsy value. 89 | doc = cls.get_document(obj) 90 | if not doc: 91 | continue 92 | 93 | data = { 94 | '_index': index_name or cls.get_index_name(), 95 | '_type': cls.get_type_name(), 96 | '_id': cls.get_document_id(obj) 97 | } 98 | data.update(cls.get_request_params(obj)) 99 | data = {'delete' if delete else 'index': data} 100 | 101 | # bulk operation instructions/details 102 | tmp.append(data) 103 | 104 | # only append bulk operation data if it's not a delete operation 105 | if not delete: 106 | tmp.append(doc) 107 | 108 | if not i % bulk_limit: 109 | es.bulk(tmp) 110 | tmp = [] 111 | 112 | if tmp: 113 | es.bulk(tmp) 114 | 115 | @classmethod 116 | def index_add(cls, obj, index_name=''): 117 | if obj and cls.should_index(obj): 118 | doc = cls.get_document(obj) 119 | if not doc: 120 | return False 121 | 122 | cls.get_es().index( 123 | index_name or cls.get_index_name(), 124 | cls.get_type_name(), 125 | doc, 126 | cls.get_document_id(obj), 127 | **cls.get_request_params(obj) 128 | ) 129 | return True 130 | return False 131 | 132 | @classmethod 133 | def index_delete(cls, obj, index_name=''): 134 | if obj: 135 | try: 136 | cls.get_es().delete( 137 | index_name or cls.get_index_name(), 138 | cls.get_type_name(), 139 | cls.get_document_id(obj), 140 | **cls.get_request_params(obj) 141 | ) 142 | except TransportError as e: 143 | if e.status_code != 404: 144 | raise 145 | return True 146 | return False 147 | 148 | @classmethod 149 | def index_add_or_delete(cls, obj, index_name=''): 150 | if obj: 151 | if cls.should_index(obj): 152 | return cls.index_add(obj, index_name) 153 | else: 154 | return cls.index_delete(obj, index_name) 155 | return False 156 | 157 | @classmethod 158 | def save_handler(cls, sender, instance, **kwargs): 159 | cls.index_add_or_delete(instance) 160 | 161 | @classmethod 162 | def delete_handler(cls, sender, instance, **kwargs): 163 | cls.index_delete(instance) 164 | -------------------------------------------------------------------------------- /simple_elasticsearch/search.py: -------------------------------------------------------------------------------- 1 | import warnings 2 | 3 | try: 4 | from collections.abc import MutableMapping, MutableSequence 5 | except: 6 | from collections import MutableMapping, MutableSequence 7 | 8 | from django.core.paginator import Paginator as DjangoPaginator 9 | from django.utils.functional import cached_property 10 | from elasticsearch import Elasticsearch 11 | 12 | from . import settings as es_settings 13 | 14 | 15 | class Paginator(DjangoPaginator): 16 | def __init__(self, response, *args, **kwargs): 17 | # `response` is a generator (MutableSequence), however `Paginator` was changed in 1.10 18 | # to require an object with either a `.count()` method (ie. QuerySet) or able 19 | # to call `len()` on the object - forcing the generator to resolve to a list 20 | # for this reason. 21 | super(Paginator, self).__init__(list(response), *args, **kwargs) 22 | 23 | # Override to set the count/total number of items; Elasticsearch provides the total 24 | # as a part of the query results, so we can minimize hits. 25 | self._count = response.total 26 | 27 | def page(self, number): 28 | # this is overridden to prevent any slicing of the object_list - Elasticsearch has 29 | # returned the sliced data already. 30 | number = self.validate_number(number) 31 | return self._get_page(self.object_list, number, self) 32 | 33 | @property 34 | def count(self): 35 | return self._count 36 | 37 | 38 | class Response(MutableSequence, object): 39 | def __init__(self, d, page_num, page_size): 40 | super(Response, self).__init__() 41 | self._page_num = page_num 42 | self._page_size = page_size 43 | 44 | self.results_meta = d.pop('hits', {}) 45 | self.__results = self.results_meta.pop('hits', []) 46 | 47 | self.aggregations = d.pop('aggregations', {}) 48 | self.response_meta = d 49 | 50 | @property 51 | def results_raw(self): 52 | return self.__results 53 | 54 | @property 55 | def results(self): 56 | return iter(self) 57 | 58 | @property 59 | def total(self): 60 | return self.results_meta.get('total', 0) 61 | 62 | @property 63 | def max_score(self): 64 | return self.results_meta.get('max_score', 0) 65 | 66 | @cached_property 67 | def page(self): 68 | paginator = Paginator(self, self._page_size) 69 | return paginator.page(self._page_num) 70 | 71 | def __len__(self): 72 | return len(self.__results) 73 | 74 | def __getitem__(self, index): 75 | return Result(self.__results[index]) 76 | 77 | def __setitem__(self, index, value): 78 | raise KeyError("Modifying results is not permitted.") 79 | 80 | def __delitem__(self, index): 81 | raise KeyError("Deleting results is not permitted.") 82 | 83 | def __iter__(self): 84 | for item in self.__results: 85 | yield Result(item) 86 | 87 | def insert(self, index, value): 88 | raise KeyError("Modifying results is not permitted.") 89 | 90 | 91 | class Result(MutableMapping, object): 92 | def __init__(self, data): 93 | super(Result, self).__init__() 94 | self.__rdata = data.pop('_source', {}) 95 | self.meta = data 96 | 97 | def __getattribute__(self, item): 98 | if item == 'result_meta': 99 | warnings.warn( 100 | "The `result_meta` attribute will be removed in future " 101 | "versions. It can now be referenced as `meta`.", 102 | DeprecationWarning) 103 | return self.meta 104 | return super(Result, self).__getattribute__(item) 105 | 106 | @property 107 | def data(self): 108 | return self.__rdata 109 | 110 | def __setitem__(self, key, value): 111 | raise KeyError("Modifying results is not permitted.") 112 | 113 | def __getitem__(self, key): 114 | return self.__rdata[key] 115 | 116 | def __delitem__(self, key): 117 | raise KeyError("Modifying results is not permitted.") 118 | 119 | def __iter__(self): 120 | return iter(self.__rdata) 121 | 122 | def __len__(self): 123 | return len(self.__rdata) 124 | 125 | 126 | class SimpleSearch(object): 127 | def __init__(self, es=None): 128 | self.es = es or Elasticsearch(es_settings.ELASTICSEARCH_SERVER) 129 | self.bulk_search_data = [] 130 | self.page_ranges = [] 131 | 132 | def reset(self): 133 | self.bulk_search_data = [] 134 | self.page_ranges = [] 135 | 136 | def add_search(self, query, page=1, page_size=20, index='', doc_type='', query_params=None): 137 | query_params = query_params if query_params is not None else {} 138 | 139 | page = int(page) 140 | page_size = int(page_size) 141 | 142 | query['from'] = (page - 1) * page_size 143 | query['size'] = page_size 144 | 145 | # save these here so we can attach the info the the responses below 146 | self.page_ranges.append((page, page_size)) 147 | 148 | data = query_params.copy() 149 | if index: 150 | data['index'] = index 151 | if doc_type: 152 | data['type'] = doc_type 153 | 154 | self.bulk_search_data.append(data) 155 | self.bulk_search_data.append(query) 156 | 157 | def search(self): 158 | responses = [] 159 | 160 | if self.bulk_search_data: 161 | # TODO: ES 5.x adds `max_concurrent_searches` as an option to `msearch()` 162 | data = self.es.msearch(self.bulk_search_data) 163 | for i, tmp in enumerate((data or {}).get('responses', [])): 164 | responses.append(Response(tmp, *self.page_ranges[i])) 165 | 166 | self.reset() 167 | 168 | return responses 169 | -------------------------------------------------------------------------------- /HISTORY.rst: -------------------------------------------------------------------------------- 1 | .. :changelog: 2 | 3 | History 4 | ------- 5 | 6 | 2.2.1 (2017-11-15) 7 | --------------------- 8 | 9 | * Fixing an issue with the recently released elasticsearch-py 6.0.0 10 | 11 | 2.2.0 (2017-07-17) 12 | --------------------- 13 | 14 | * Addressing inability to index models with a non-integer PK field (ie. `UUIDField`) - added ability to order bulk queryset on an arbitrary model field. 15 | 16 | 2.1.7 (2017-03-21) 17 | --------------------- 18 | 19 | * Allowing direct access (again) to underlying dict/list in `Result` and `Response` classes for serialization and other purposes. 20 | 21 | 2.1.5 (2017-03-20) 22 | --------------------- 23 | 24 | * Response class is now MutableSequence based, giving it the properties of a `list`. Its `results` attribute is deprecated, as you can now iterate over the results with the response instance itself. 25 | * Result class `results_meta` is deprecated. Use `meta` instead. 26 | * `get_from_es_or_None` now returns a `Result` object instead of the raw Elasticsearch result, for consistency. 27 | * `get_from_es_or_None` now catches only the Elasticsearch `NotFoundError` exception; previously it caught the more expansive `ElasticsearchException`, which could hide unrelated errors/issues. 28 | 29 | 2.1.4 (2017-03-12) 30 | --------------------- 31 | 32 | * Result class is now MutableMapping based, giving it the properties of a `dict`. Its `data` attribute is deprecated. 33 | 34 | 2.1.3 (2017-03-11) 35 | --------------------- 36 | 37 | * Minor changes to enable support for elasticsearch-py 5.x. 38 | 39 | 2.1.0 (2017-03-10) 40 | --------------------- 41 | 42 | * Addressing a packaging problem which erroneously included pyc/__pycache__ files. 43 | 44 | 2.0.0 (2016-12-20) 45 | --------------------- 46 | 47 | * **ALERT: this is a backwards incompatible release**; the old `1.x` (formerly `0.9.x`+) code is maintained on a separate branch for now. 48 | * Added support for Django 1.10. 49 | * Ported delete/cleanup functionality from `1.x`. 50 | * Removed support for Django versions older than 1.8. The goal going forward will be to only support Django versions that the Django core team lists as supported. 51 | * Removed elasticsearch-dsl support: responses and results are now represented by simpler internal representations; queries can ONLY be done via a `dict` form. 52 | * Removed `ElasticsearchForm` - it is easy enough to build a form to validate search input and then form a query `dict` manually. 53 | * Renamed `ElasticsearchIndexMixin` to `ElasticsearchTypeMixin`, seeing as the mixin represented an ES type mapping, not an actual index. 54 | * Renamed `ElasticsearchProcessor` to `SimpleSearch`. 55 | * Overall, this module has been greatly simplified. 56 | 57 | 1.0.0 (2016-12-20) 58 | --------------------- 59 | 60 | * Updated 0.9.x codebase version to 1.0.0. 61 | * Reversed decision on the deprecation of the 0.9.x codebase - it will be maintained in this new 1.x branch, although new functionality will mostly occur on newer releases. 62 | * Adding cleanup command to remove unaliased indices. 63 | * Added ELASTICSEARCH_DELETE_OLD_INDEXES setting to auto-remove after a rebuild. 64 | * Thanks to Github user @jimjkelly for the index removal inspiration. 65 | 66 | 0.9.16 (2015-04-24) 67 | --------------------- 68 | 69 | * Addressing Django 1.8 warnings. 70 | 71 | 0.9.15 (2015-01-31) 72 | --------------------- 73 | 74 | * BUGFIX: Merging pull request from @key that addresses Python 3 support (management command now works). 75 | 76 | 0.9.14 (2015-01-31) 77 | --------------------- 78 | 79 | * BUGFIX: Adding in missing `signals.py` file. 80 | 81 | 0.9.13 (2015-01-30) 82 | --------------------- 83 | 84 | * Added in new `post_indices_create` and `post_indices_rebuild` signals to allow users to run actions at various points during the index creation and bulk indexing processes. 85 | 86 | 0.9.12 (2014-12-17) 87 | --------------------- 88 | 89 | * fixed an issue where per-item request parameters were being added to the bulk data request JSON incorrectly. Tests updated. 90 | 91 | 0.9.11 (2014-12-08) 92 | --------------------- 93 | 94 | * added warning if Django's DEBUG=True (causes out of memory errors on constrained 95 | systems due to Django query caching) 96 | * added index setting modification on rebuilding indices to remove replicas, lucene 97 | segment merging and disabling the refresh interval - restoring the original 98 | settings afterwards. 99 | 100 | 0.9.10 (2014-12-04) 101 | --------------------- 102 | 103 | * added `page` and `page_size` validation in `add_search()` 104 | 105 | 0.9.9 (2014-11-24) 106 | --------------------- 107 | 108 | * Renamed search form related classes - more breaking changes. Added in support 109 | for Django's pagination classes (internal hack). 110 | 111 | 0.9.8 (2014-11-23) 112 | --------------------- 113 | 114 | * Revamped search form related classes - includes breaking changes. 115 | 116 | 0.9.7 (2014-11-16) 117 | --------------------- 118 | 119 | * Python3 supported mentioned in PyPi categorization; new testcases added. Minor 120 | interface change (added `@classmethod`). 121 | 122 | 0.9.6 (2014-11-16) 123 | --------------------- 124 | 125 | * Python 3.3+ support, modified (no new) testcases. 126 | 127 | 0.9.5 (2014-11-15) 128 | --------------------- 129 | 130 | * Added in tox support, initial set of test cases and verified travis-ci is working. 131 | 132 | 0.9.2 (2014-11-12) 133 | --------------------- 134 | 135 | * Fixed broken management command. 136 | 137 | 0.9.1 (2014-11-10) 138 | --------------------- 139 | 140 | * Added missing management command module. 141 | 142 | 0.9.0 (2014-11-10) 143 | --------------------- 144 | 145 | * In what will become version 1.0, this 0.9.x codebase is a revamp of the 146 | original codebase (v0.5.x). Completely breaking over previous versions. 147 | 148 | 0.5.0 (2014-03-05) 149 | --------------------- 150 | 151 | Final release in 0.x codebase - this old codebase is now unmaintained. 152 | -------------------------------------------------------------------------------- /docs/usage.rst: -------------------------------------------------------------------------------- 1 | Usage 2 | ===== 3 | 4 | For a minimal investment of time, Django Simple Elasticsearch offers a number of perks. Implementing a class 5 | with the :code:`ElasticsearchTypeMixin` lets you: 6 | 7 | * initialize your Elasticsearch indices and mappings via the included :code:`es_manage` management command 8 | * perform Elasticsearch bulk indexing via the same :code:`es_manage` management command 9 | * perform Elasticsearch bulk indexing as well as individual index/delete requests on demand in your code 10 | * connect the available :code:`ElasticsearchTypeMixin` save and delete handlers to Django's available 11 | model signals (ie :code:`post_save`, :code:`post_delete`) 12 | 13 | Let's look at an example implementation of :code:`ElasticsearchTypeMixin`. Here's a couple of blog-related Models 14 | in a :code:`models.py` file: 15 | 16 | .. code-block:: python 17 | 18 | class Blog(models.Model): 19 | name = models.CharField(max_length=50) 20 | description = models.TextField() 21 | 22 | class BlogPost(models.Model): 23 | blog = models.ForeignKey(Blog) 24 | slug = models.SlugField() 25 | title = models.CharField(max_length=50) 26 | body = models.TextField() 27 | created_at = models.DateTimeField(auto_now_add=True) 28 | 29 | To start with :code:`simple_elasticsearch`, you'll need to tell it that the :code:`BlogPost` class implements the 30 | :code:`ElasticsearchTypeMixin` mixin, so in your :code:`settings.py` set the :code:`ELASTICSEARCH_TYPE_CLASSES` setting: 31 | 32 | .. code-block:: python 33 | 34 | ELASTICSEARCH_TYPE_CLASSES = [ 35 | 'blog.models.BlogPost' 36 | ] 37 | 38 | If you do not add this setting, everything will still work except for the :code:`es_manage` command - it won't know 39 | what indices to create, type mappings to set or what objects to index. As you add additional 40 | :code:`ElasticsearchTypeMixin`-based index handlers, add them to this list. 41 | 42 | All right, let's add in :code:`ElasticsearchTypeMixin` to the :code:`BlogPost` model. Only pertinent changes from the 43 | above :code:`models.py` are shown: 44 | 45 | .. code-block:: python 46 | 47 | from simple_elasticsearch.mixins import ElasticsearchTypeMixin 48 | 49 | ... 50 | 51 | class BlogPost(models.Model, ElasticsearchTypeMixin): 52 | blog = models.ForeignKey(Blog) 53 | slug = models.SlugField() 54 | title = models.CharField(max_length=50) 55 | body = models.TextField() 56 | created_at = models.DateTimeField(auto_now_add=True) 57 | 58 | @classmethod 59 | def get_queryset(cls): 60 | return BlogPost.objects.all().select_related('blog') 61 | 62 | @classmethod 63 | def get_index_name(cls): 64 | return 'blog' 65 | 66 | @classmethod 67 | def get_type_name(cls): 68 | return 'posts' 69 | 70 | @classmethod 71 | def get_type_mapping(cls): 72 | return { 73 | "properties": { 74 | "created_at": { 75 | "type": "date", 76 | "format": "dateOptionalTime" 77 | }, 78 | "title": { 79 | "type": "string" 80 | }, 81 | "body": { 82 | "type": "string" 83 | }, 84 | "slug": { 85 | "type": "string" 86 | }, 87 | "blog": { 88 | "properties": { 89 | "id": { 90 | "type": "long" 91 | }, 92 | "name": { 93 | "type": "string" 94 | }, 95 | "description": { 96 | "type": "string" 97 | } 98 | } 99 | } 100 | } 101 | } 102 | 103 | @classmethod 104 | def get_document(cls, obj): 105 | return { 106 | 'created_at': obj.created_at, 107 | 'title': obj.title, 108 | 'body': obj.body, 109 | 'slug': obj.slug, 110 | 'blog': { 111 | 'id': obj.blog.id, 112 | 'name': obj.blog.name, 113 | 'description': obj.blog.description, 114 | } 115 | } 116 | 117 | With this mixin implementation, you can now use the :code:`es_manage` management command to bulk reindex all :code:`BlogPost` 118 | items. Note that there are additional :code:`@classmethods` you can override to customize functionality. Sane defaults 119 | have been provided for these - see the source for details. 120 | 121 | Of course, our :code:`BlogPost` implementation doesn't ensure that your Elasticsearch index is updated every time you 122 | save or delete - for this, you can use the :code:`ElasticsearchTypeMixin` built-in save and delete handlers. 123 | 124 | .. code-block:: python 125 | 126 | from django.db.models.signals import post_save, pre_delete 127 | 128 | ... 129 | 130 | post_save.connect(BlogPost.save_handler, sender=BlogPost) 131 | pre_delete.connect(BlogPost.delete_handler, sender=BlogPost) 132 | 133 | Awesome - Django's magic is applied. 134 | 135 | Notes 136 | ===== 137 | 138 | * Prior to version 2.2.0 of this package, only models with numerical primary keys could be indexed properly due to the 139 | way the :code:`queryset_iterator()` utility function was implemented. This has been changed and the primary key no longer 140 | matters. 141 | 142 | Ordering the bulk queryset is important due to the fact that records may have been added during the indexing process 143 | (indexing data can take a long time); if the results are ordered properly, the indexing process will catch the most 144 | recent records. For most cases, the default bulk ordering of :code:`pk` will suffice (Django's default primary key field is 145 | an auto-incrementing integer). 146 | 147 | If a model has PK using a :code:`UUIDField` however, things change: UUIDs are randomly generated, so ordering by a 148 | :code:`UUIDField` PK will most likely result in newly created items being missed in the indexin process. Overriding the 149 | :code:`ElasticsearchTypeMixin` class method :code:`get_bulk_ordering()` addresses this issue - set it to order by a 150 | :code:`DateTimeField` on the model. 151 | 152 | TODO: 153 | 154 | * add examples for more complex data situations 155 | * add examples of using :code:`es_manage` management command options 156 | * add examples/scenarios when to use :code:`post_indices_create` and :code:`post_indices_rebuild` signals (ie. adding percolators to new indices) 157 | -------------------------------------------------------------------------------- /docs/make.bat: -------------------------------------------------------------------------------- 1 | @ECHO OFF 2 | 3 | REM Command file for Sphinx documentation 4 | 5 | if "%SPHINXBUILD%" == "" ( 6 | set SPHINXBUILD=sphinx-build 7 | ) 8 | set BUILDDIR=_build 9 | set ALLSPHINXOPTS=-d %BUILDDIR%/doctrees %SPHINXOPTS% . 10 | set I18NSPHINXOPTS=%SPHINXOPTS% . 11 | if NOT "%PAPER%" == "" ( 12 | set ALLSPHINXOPTS=-D latex_paper_size=%PAPER% %ALLSPHINXOPTS% 13 | set I18NSPHINXOPTS=-D latex_paper_size=%PAPER% %I18NSPHINXOPTS% 14 | ) 15 | 16 | if "%1" == "" goto help 17 | 18 | if "%1" == "help" ( 19 | :help 20 | echo.Please use `make ^` where ^ is one of 21 | echo. html to make standalone HTML files 22 | echo. dirhtml to make HTML files named index.html in directories 23 | echo. singlehtml to make a single large HTML file 24 | echo. pickle to make pickle files 25 | echo. json to make JSON files 26 | echo. htmlhelp to make HTML files and a HTML help project 27 | echo. qthelp to make HTML files and a qthelp project 28 | echo. devhelp to make HTML files and a Devhelp project 29 | echo. epub to make an epub 30 | echo. latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter 31 | echo. text to make text files 32 | echo. man to make manual pages 33 | echo. texinfo to make Texinfo files 34 | echo. gettext to make PO message catalogs 35 | echo. changes to make an overview over all changed/added/deprecated items 36 | echo. xml to make Docutils-native XML files 37 | echo. pseudoxml to make pseudoxml-XML files for display purposes 38 | echo. linkcheck to check all external links for integrity 39 | echo. doctest to run all doctests embedded in the documentation if enabled 40 | goto end 41 | ) 42 | 43 | if "%1" == "clean" ( 44 | for /d %%i in (%BUILDDIR%\*) do rmdir /q /s %%i 45 | del /q /s %BUILDDIR%\* 46 | goto end 47 | ) 48 | 49 | 50 | %SPHINXBUILD% 2> nul 51 | if errorlevel 9009 ( 52 | echo. 53 | echo.The 'sphinx-build' command was not found. Make sure you have Sphinx 54 | echo.installed, then set the SPHINXBUILD environment variable to point 55 | echo.to the full path of the 'sphinx-build' executable. Alternatively you 56 | echo.may add the Sphinx directory to PATH. 57 | echo. 58 | echo.If you don't have Sphinx installed, grab it from 59 | echo.http://sphinx-doc.org/ 60 | exit /b 1 61 | ) 62 | 63 | if "%1" == "html" ( 64 | %SPHINXBUILD% -b html %ALLSPHINXOPTS% %BUILDDIR%/html 65 | if errorlevel 1 exit /b 1 66 | echo. 67 | echo.Build finished. The HTML pages are in %BUILDDIR%/html. 68 | goto end 69 | ) 70 | 71 | if "%1" == "dirhtml" ( 72 | %SPHINXBUILD% -b dirhtml %ALLSPHINXOPTS% %BUILDDIR%/dirhtml 73 | if errorlevel 1 exit /b 1 74 | echo. 75 | echo.Build finished. The HTML pages are in %BUILDDIR%/dirhtml. 76 | goto end 77 | ) 78 | 79 | if "%1" == "singlehtml" ( 80 | %SPHINXBUILD% -b singlehtml %ALLSPHINXOPTS% %BUILDDIR%/singlehtml 81 | if errorlevel 1 exit /b 1 82 | echo. 83 | echo.Build finished. The HTML pages are in %BUILDDIR%/singlehtml. 84 | goto end 85 | ) 86 | 87 | if "%1" == "pickle" ( 88 | %SPHINXBUILD% -b pickle %ALLSPHINXOPTS% %BUILDDIR%/pickle 89 | if errorlevel 1 exit /b 1 90 | echo. 91 | echo.Build finished; now you can process the pickle files. 92 | goto end 93 | ) 94 | 95 | if "%1" == "json" ( 96 | %SPHINXBUILD% -b json %ALLSPHINXOPTS% %BUILDDIR%/json 97 | if errorlevel 1 exit /b 1 98 | echo. 99 | echo.Build finished; now you can process the JSON files. 100 | goto end 101 | ) 102 | 103 | if "%1" == "htmlhelp" ( 104 | %SPHINXBUILD% -b htmlhelp %ALLSPHINXOPTS% %BUILDDIR%/htmlhelp 105 | if errorlevel 1 exit /b 1 106 | echo. 107 | echo.Build finished; now you can run HTML Help Workshop with the ^ 108 | .hhp project file in %BUILDDIR%/htmlhelp. 109 | goto end 110 | ) 111 | 112 | if "%1" == "qthelp" ( 113 | %SPHINXBUILD% -b qthelp %ALLSPHINXOPTS% %BUILDDIR%/qthelp 114 | if errorlevel 1 exit /b 1 115 | echo. 116 | echo.Build finished; now you can run "qcollectiongenerator" with the ^ 117 | .qhcp project file in %BUILDDIR%/qthelp, like this: 118 | echo.^> qcollectiongenerator %BUILDDIR%\qthelp\django-simple-elasticsearch.qhcp 119 | echo.To view the help file: 120 | echo.^> assistant -collectionFile %BUILDDIR%\qthelp\django-simple-elasticsearch.ghc 121 | goto end 122 | ) 123 | 124 | if "%1" == "devhelp" ( 125 | %SPHINXBUILD% -b devhelp %ALLSPHINXOPTS% %BUILDDIR%/devhelp 126 | if errorlevel 1 exit /b 1 127 | echo. 128 | echo.Build finished. 129 | goto end 130 | ) 131 | 132 | if "%1" == "epub" ( 133 | %SPHINXBUILD% -b epub %ALLSPHINXOPTS% %BUILDDIR%/epub 134 | if errorlevel 1 exit /b 1 135 | echo. 136 | echo.Build finished. The epub file is in %BUILDDIR%/epub. 137 | goto end 138 | ) 139 | 140 | if "%1" == "latex" ( 141 | %SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex 142 | if errorlevel 1 exit /b 1 143 | echo. 144 | echo.Build finished; the LaTeX files are in %BUILDDIR%/latex. 145 | goto end 146 | ) 147 | 148 | if "%1" == "latexpdf" ( 149 | %SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex 150 | cd %BUILDDIR%/latex 151 | make all-pdf 152 | cd %BUILDDIR%/.. 153 | echo. 154 | echo.Build finished; the PDF files are in %BUILDDIR%/latex. 155 | goto end 156 | ) 157 | 158 | if "%1" == "latexpdfja" ( 159 | %SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex 160 | cd %BUILDDIR%/latex 161 | make all-pdf-ja 162 | cd %BUILDDIR%/.. 163 | echo. 164 | echo.Build finished; the PDF files are in %BUILDDIR%/latex. 165 | goto end 166 | ) 167 | 168 | if "%1" == "text" ( 169 | %SPHINXBUILD% -b text %ALLSPHINXOPTS% %BUILDDIR%/text 170 | if errorlevel 1 exit /b 1 171 | echo. 172 | echo.Build finished. The text files are in %BUILDDIR%/text. 173 | goto end 174 | ) 175 | 176 | if "%1" == "man" ( 177 | %SPHINXBUILD% -b man %ALLSPHINXOPTS% %BUILDDIR%/man 178 | if errorlevel 1 exit /b 1 179 | echo. 180 | echo.Build finished. The manual pages are in %BUILDDIR%/man. 181 | goto end 182 | ) 183 | 184 | if "%1" == "texinfo" ( 185 | %SPHINXBUILD% -b texinfo %ALLSPHINXOPTS% %BUILDDIR%/texinfo 186 | if errorlevel 1 exit /b 1 187 | echo. 188 | echo.Build finished. The Texinfo files are in %BUILDDIR%/texinfo. 189 | goto end 190 | ) 191 | 192 | if "%1" == "gettext" ( 193 | %SPHINXBUILD% -b gettext %I18NSPHINXOPTS% %BUILDDIR%/locale 194 | if errorlevel 1 exit /b 1 195 | echo. 196 | echo.Build finished. The message catalogs are in %BUILDDIR%/locale. 197 | goto end 198 | ) 199 | 200 | if "%1" == "changes" ( 201 | %SPHINXBUILD% -b changes %ALLSPHINXOPTS% %BUILDDIR%/changes 202 | if errorlevel 1 exit /b 1 203 | echo. 204 | echo.The overview file is in %BUILDDIR%/changes. 205 | goto end 206 | ) 207 | 208 | if "%1" == "linkcheck" ( 209 | %SPHINXBUILD% -b linkcheck %ALLSPHINXOPTS% %BUILDDIR%/linkcheck 210 | if errorlevel 1 exit /b 1 211 | echo. 212 | echo.Link check complete; look for any errors in the above output ^ 213 | or in %BUILDDIR%/linkcheck/output.txt. 214 | goto end 215 | ) 216 | 217 | if "%1" == "doctest" ( 218 | %SPHINXBUILD% -b doctest %ALLSPHINXOPTS% %BUILDDIR%/doctest 219 | if errorlevel 1 exit /b 1 220 | echo. 221 | echo.Testing of doctests in the sources finished, look at the ^ 222 | results in %BUILDDIR%/doctest/output.txt. 223 | goto end 224 | ) 225 | 226 | if "%1" == "xml" ( 227 | %SPHINXBUILD% -b xml %ALLSPHINXOPTS% %BUILDDIR%/xml 228 | if errorlevel 1 exit /b 1 229 | echo. 230 | echo.Build finished. The XML files are in %BUILDDIR%/xml. 231 | goto end 232 | ) 233 | 234 | if "%1" == "pseudoxml" ( 235 | %SPHINXBUILD% -b pseudoxml %ALLSPHINXOPTS% %BUILDDIR%/pseudoxml 236 | if errorlevel 1 exit /b 1 237 | echo. 238 | echo.Build finished. The pseudo-XML files are in %BUILDDIR%/pseudoxml. 239 | goto end 240 | ) 241 | 242 | :end 243 | -------------------------------------------------------------------------------- /docs/Makefile: -------------------------------------------------------------------------------- 1 | # Makefile for Sphinx documentation 2 | # 3 | 4 | # You can set these variables from the command line. 5 | SPHINXOPTS = 6 | SPHINXBUILD = sphinx-build 7 | PAPER = 8 | BUILDDIR = _build 9 | 10 | # User-friendly check for sphinx-build 11 | ifeq ($(shell which $(SPHINXBUILD) >/dev/null 2>&1; echo $$?), 1) 12 | $(error The '$(SPHINXBUILD)' command was not found. Make sure you have Sphinx installed, then set the SPHINXBUILD environment variable to point to the full path of the '$(SPHINXBUILD)' executable. Alternatively you can add the directory with the executable to your PATH. If you don't have Sphinx installed, grab it from http://sphinx-doc.org/) 13 | endif 14 | 15 | # Internal variables. 16 | PAPEROPT_a4 = -D latex_paper_size=a4 17 | PAPEROPT_letter = -D latex_paper_size=letter 18 | ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . 19 | # the i18n builder cannot share the environment and doctrees with the others 20 | I18NSPHINXOPTS = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . 21 | 22 | .PHONY: help clean html dirhtml singlehtml pickle json htmlhelp qthelp devhelp epub latex latexpdf text man changes linkcheck doctest gettext 23 | 24 | help: 25 | @echo "Please use \`make ' where is one of" 26 | @echo " html to make standalone HTML files" 27 | @echo " dirhtml to make HTML files named index.html in directories" 28 | @echo " singlehtml to make a single large HTML file" 29 | @echo " pickle to make pickle files" 30 | @echo " json to make JSON files" 31 | @echo " htmlhelp to make HTML files and a HTML help project" 32 | @echo " qthelp to make HTML files and a qthelp project" 33 | @echo " devhelp to make HTML files and a Devhelp project" 34 | @echo " epub to make an epub" 35 | @echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter" 36 | @echo " latexpdf to make LaTeX files and run them through pdflatex" 37 | @echo " latexpdfja to make LaTeX files and run them through platex/dvipdfmx" 38 | @echo " text to make text files" 39 | @echo " man to make manual pages" 40 | @echo " texinfo to make Texinfo files" 41 | @echo " info to make Texinfo files and run them through makeinfo" 42 | @echo " gettext to make PO message catalogs" 43 | @echo " changes to make an overview of all changed/added/deprecated items" 44 | @echo " xml to make Docutils-native XML files" 45 | @echo " pseudoxml to make pseudoxml-XML files for display purposes" 46 | @echo " linkcheck to check all external links for integrity" 47 | @echo " doctest to run all doctests embedded in the documentation (if enabled)" 48 | 49 | clean: 50 | rm -rf $(BUILDDIR)/* 51 | 52 | html: 53 | $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html 54 | @echo 55 | @echo "Build finished. The HTML pages are in $(BUILDDIR)/html." 56 | 57 | dirhtml: 58 | $(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml 59 | @echo 60 | @echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml." 61 | 62 | singlehtml: 63 | $(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) $(BUILDDIR)/singlehtml 64 | @echo 65 | @echo "Build finished. The HTML page is in $(BUILDDIR)/singlehtml." 66 | 67 | pickle: 68 | $(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle 69 | @echo 70 | @echo "Build finished; now you can process the pickle files." 71 | 72 | json: 73 | $(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json 74 | @echo 75 | @echo "Build finished; now you can process the JSON files." 76 | 77 | htmlhelp: 78 | $(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp 79 | @echo 80 | @echo "Build finished; now you can run HTML Help Workshop with the" \ 81 | ".hhp project file in $(BUILDDIR)/htmlhelp." 82 | 83 | qthelp: 84 | $(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp 85 | @echo 86 | @echo "Build finished; now you can run "qcollectiongenerator" with the" \ 87 | ".qhcp project file in $(BUILDDIR)/qthelp, like this:" 88 | @echo "# qcollectiongenerator $(BUILDDIR)/qthelp/django-simple-elasticsearch.qhcp" 89 | @echo "To view the help file:" 90 | @echo "# assistant -collectionFile $(BUILDDIR)/qthelp/django-simple-elasticsearch.qhc" 91 | 92 | devhelp: 93 | $(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILDDIR)/devhelp 94 | @echo 95 | @echo "Build finished." 96 | @echo "To view the help file:" 97 | @echo "# mkdir -p $$HOME/.local/share/devhelp/django-simple-elasticsearch" 98 | @echo "# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/django-simple-elasticsearch" 99 | @echo "# devhelp" 100 | 101 | epub: 102 | $(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILDDIR)/epub 103 | @echo 104 | @echo "Build finished. The epub file is in $(BUILDDIR)/epub." 105 | 106 | latex: 107 | $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex 108 | @echo 109 | @echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex." 110 | @echo "Run \`make' in that directory to run these through (pdf)latex" \ 111 | "(use \`make latexpdf' here to do that automatically)." 112 | 113 | latexpdf: 114 | $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex 115 | @echo "Running LaTeX files through pdflatex..." 116 | $(MAKE) -C $(BUILDDIR)/latex all-pdf 117 | @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." 118 | 119 | latexpdfja: 120 | $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex 121 | @echo "Running LaTeX files through platex and dvipdfmx..." 122 | $(MAKE) -C $(BUILDDIR)/latex all-pdf-ja 123 | @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." 124 | 125 | text: 126 | $(SPHINXBUILD) -b text $(ALLSPHINXOPTS) $(BUILDDIR)/text 127 | @echo 128 | @echo "Build finished. The text files are in $(BUILDDIR)/text." 129 | 130 | man: 131 | $(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man 132 | @echo 133 | @echo "Build finished. The manual pages are in $(BUILDDIR)/man." 134 | 135 | texinfo: 136 | $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo 137 | @echo 138 | @echo "Build finished. The Texinfo files are in $(BUILDDIR)/texinfo." 139 | @echo "Run \`make' in that directory to run these through makeinfo" \ 140 | "(use \`make info' here to do that automatically)." 141 | 142 | info: 143 | $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo 144 | @echo "Running Texinfo files through makeinfo..." 145 | make -C $(BUILDDIR)/texinfo info 146 | @echo "makeinfo finished; the Info files are in $(BUILDDIR)/texinfo." 147 | 148 | gettext: 149 | $(SPHINXBUILD) -b gettext $(I18NSPHINXOPTS) $(BUILDDIR)/locale 150 | @echo 151 | @echo "Build finished. The message catalogs are in $(BUILDDIR)/locale." 152 | 153 | changes: 154 | $(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes 155 | @echo 156 | @echo "The overview file is in $(BUILDDIR)/changes." 157 | 158 | linkcheck: 159 | $(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck 160 | @echo 161 | @echo "Link check complete; look for any errors in the above output " \ 162 | "or in $(BUILDDIR)/linkcheck/output.txt." 163 | 164 | doctest: 165 | $(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest 166 | @echo "Testing of doctests in the sources finished, look at the " \ 167 | "results in $(BUILDDIR)/doctest/output.txt." 168 | 169 | xml: 170 | $(SPHINXBUILD) -b xml $(ALLSPHINXOPTS) $(BUILDDIR)/xml 171 | @echo 172 | @echo "Build finished. The XML files are in $(BUILDDIR)/xml." 173 | 174 | pseudoxml: 175 | $(SPHINXBUILD) -b pseudoxml $(ALLSPHINXOPTS) $(BUILDDIR)/pseudoxml 176 | @echo 177 | @echo "Build finished. The pseudo-XML files are in $(BUILDDIR)/pseudoxml." 178 | 179 | livehtml: 180 | sphinx-autobuild -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html -------------------------------------------------------------------------------- /simple_elasticsearch/utils.py: -------------------------------------------------------------------------------- 1 | import collections 2 | import datetime 3 | import gc 4 | import sys 5 | from django.conf import settings 6 | from django.http import Http404 7 | from elasticsearch import Elasticsearch, NotFoundError 8 | 9 | from simple_elasticsearch.search import Result 10 | from . import settings as es_settings 11 | from .signals import post_indices_create, post_indices_rebuild 12 | 13 | try: 14 | from importlib import import_module 15 | except ImportError: 16 | from django.utils.importlib import import_module 17 | 18 | _elasticsearch_indices = collections.defaultdict(lambda: []) 19 | 20 | 21 | def get_indices(indices=[]): 22 | if not _elasticsearch_indices: 23 | type_classes = getattr(settings, 'ELASTICSEARCH_TYPE_CLASSES', ()) 24 | if not type_classes: 25 | raise Exception('Missing `ELASTICSEARCH_TYPE_CLASSES` in project `settings`.') 26 | 27 | for type_class in type_classes: 28 | package_name, klass_name = type_class.rsplit('.', 1) 29 | try: 30 | package = import_module(package_name) 31 | klass = getattr(package, klass_name) 32 | except ImportError: 33 | sys.stderr.write('Unable to import `{}`.\n'.format(type_class)) 34 | continue 35 | _elasticsearch_indices[klass.get_index_name()].append(klass) 36 | 37 | if not indices: 38 | return _elasticsearch_indices 39 | else: 40 | result = {} 41 | for k, v in _elasticsearch_indices.items(): 42 | if k in indices: 43 | result[k] = v 44 | return result 45 | 46 | 47 | def create_aliases(es=None, indices=[]): 48 | es = es or Elasticsearch(**es_settings.ELASTICSEARCH_CONNECTION_PARAMS) 49 | 50 | current_aliases = es.indices.get_alias() 51 | aliases_for_removal = collections.defaultdict(lambda: []) 52 | for item, tmp in current_aliases.items(): 53 | for iname in list(tmp.get('aliases', {}).keys()): 54 | aliases_for_removal[iname].append(item) 55 | 56 | actions = [] 57 | for index_alias, index_name in indices: 58 | for item in aliases_for_removal[index_alias]: 59 | actions.append({ 60 | 'remove': { 61 | 'index': item, 62 | 'alias': index_alias 63 | } 64 | }) 65 | actions.append({ 66 | 'add': { 67 | 'index': index_name, 68 | 'alias': index_alias 69 | } 70 | }) 71 | 72 | es.indices.update_aliases({'actions': actions}) 73 | 74 | 75 | def create_indices(es=None, indices=[], set_aliases=True): 76 | es = es or Elasticsearch(**es_settings.ELASTICSEARCH_CONNECTION_PARAMS) 77 | 78 | result = [] 79 | aliases = [] 80 | 81 | now = datetime.datetime.now().strftime("%Y%m%d-%H%M%S") 82 | for index_alias, type_classes in get_indices(indices).items(): 83 | index_settings = es_settings.ELASTICSEARCH_DEFAULT_INDEX_SETTINGS 84 | index_settings = recursive_dict_update( 85 | index_settings, 86 | es_settings.ELASTICSEARCH_CUSTOM_INDEX_SETTINGS.get(index_alias, {}) 87 | ) 88 | 89 | index_name = "{0}-{1}".format(index_alias, now) 90 | 91 | aliases.append((index_alias, index_name)) 92 | 93 | type_mappings = {} 94 | for type_class in type_classes: 95 | tmp = type_class.get_type_mapping() 96 | if tmp: 97 | type_mappings[type_class.get_type_name()] = tmp 98 | 99 | result.append(( 100 | type_class, 101 | index_alias, 102 | index_name 103 | )) 104 | 105 | # if we got any type mappings, put them in the index settings 106 | if type_mappings: 107 | index_settings['mappings'] = type_mappings 108 | 109 | es.indices.create(index_name, index_settings) 110 | 111 | if set_aliases: 112 | create_aliases(es, aliases) 113 | 114 | # `aliases` is a list of (index alias, index timestamped-name) tuples 115 | post_indices_create.send(None, indices=aliases, aliases_set=set_aliases) 116 | 117 | return result, aliases 118 | 119 | 120 | def rebuild_indices(es=None, indices=[], set_aliases=True): 121 | es = es or Elasticsearch(**es_settings.ELASTICSEARCH_CONNECTION_PARAMS) 122 | 123 | created_indices, aliases = create_indices(es, indices, False) 124 | 125 | # kludge to avoid OOM due to Django's query logging 126 | # db_logger = logging.getLogger('django.db.backends') 127 | # oldlevel = db_logger.level 128 | # db_logger.setLevel(logging.ERROR) 129 | 130 | current_index_name = None 131 | current_index_settings = {} 132 | 133 | def change_index(): 134 | if current_index_name: 135 | # restore the original (or their ES defaults) settings back into 136 | # the index to restore desired elasticsearch functionality 137 | settings = { 138 | 'number_of_replicas': current_index_settings.get('index', {}).get('number_of_replicas', 1), 139 | 'refresh_interval': current_index_settings.get('index', {}).get('refresh_interval', '1s'), 140 | } 141 | es.indices.put_settings({'index': settings}, current_index_name) 142 | es.indices.refresh(current_index_name) 143 | 144 | for type_class, index_alias, index_name in created_indices: 145 | if index_name != current_index_name: 146 | change_index() 147 | 148 | # save the current index's settings locally so that we can restore them after 149 | current_index_settings = es.indices.get_settings(index_name).get(index_name, {}).get('settings', {}) 150 | current_index_name = index_name 151 | 152 | # modify index settings to speed up bulk indexing and then restore them after 153 | es.indices.put_settings({'index': { 154 | 'number_of_replicas': 0, 155 | 'refresh_interval': '-1', 156 | }}, index=index_name) 157 | 158 | try: 159 | type_class.bulk_index(es, index_name) 160 | except NotImplementedError: 161 | sys.stderr.write('`bulk_index` not implemented on `{}`.\n'.format(type_class.get_index_name())) 162 | continue 163 | else: 164 | change_index() 165 | 166 | # return to the norm for db query logging 167 | # db_logger.setLevel(oldlevel) 168 | 169 | if set_aliases: 170 | create_aliases(es, aliases) 171 | if es_settings.ELASTICSEARCH_DELETE_OLD_INDEXES: 172 | delete_indices(es, [a for a, i in aliases]) 173 | 174 | # `aliases` is a list of (index alias, index timestamped-name) tuples 175 | post_indices_rebuild.send(None, indices=aliases, aliases_set=set_aliases) 176 | 177 | return created_indices, aliases 178 | 179 | 180 | def delete_indices(es=None, indices=[], only_unaliased=True): 181 | es = es or Elasticsearch(**es_settings.ELASTICSEARCH_CONNECTION_PARAMS) 182 | indices = indices or get_indices(indices=[]).keys() 183 | indices_to_remove = [] 184 | for index, aliases in es.indices.get_alias().items(): 185 | # Make sure it isn't currently aliased, which would mean it's active (UNLESS 186 | # we want to force-delete all `simple_elasticsearch`-managed indices). 187 | # AND 188 | # Make sure it isn't an arbitrary non-`simple_elasticsearch` managed index. 189 | if (not only_unaliased or not aliases.get('aliases')) and index.split('-', 1)[0] in indices: 190 | indices_to_remove.append(index) 191 | 192 | if indices_to_remove: 193 | es.indices.delete(','.join(indices_to_remove)) 194 | return indices_to_remove 195 | 196 | 197 | def recursive_dict_update(d, u): 198 | for k, v in u.items(): 199 | if isinstance(v, collections.Mapping): 200 | r = recursive_dict_update(d.get(k, {}), v) 201 | d[k] = r 202 | else: 203 | d[k] = u[k] 204 | return d 205 | 206 | 207 | def queryset_iterator(queryset, chunksize=1000, order_by='pk'): 208 | if order_by: 209 | queryset = queryset.order_by(order_by) 210 | 211 | chunk = 0 212 | while True: 213 | n = 0 214 | for n, row in enumerate(queryset[chunk * chunksize:(chunk + 1) * chunksize]): 215 | yield row 216 | 217 | if not n: 218 | break 219 | chunk += 1 220 | gc.collect() 221 | 222 | 223 | def get_from_es_or_None(index, type, id, **kwargs): 224 | es = kwargs.pop('es', Elasticsearch(es_settings.ELASTICSEARCH_SERVER)) 225 | try: 226 | return Result(es.get(index=index, doc_type=type, id=id, **kwargs)) 227 | except NotFoundError: 228 | return None 229 | 230 | 231 | def get_from_es_or_404(index, type, id, **kwargs): 232 | item = get_from_es_or_None(index, type, id, **kwargs) 233 | if not item: 234 | raise Http404('No {0} matches the parameters.'.format(type)) 235 | return item 236 | -------------------------------------------------------------------------------- /docs/conf.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # 4 | # django-simple-elasticsearch documentation build configuration file, created by 5 | # sphinx-quickstart on Tue Jul 9 22:26:36 2013. 6 | # 7 | # This file is execfile()d with the current directory set to its 8 | # containing dir. 9 | # 10 | # Note that not all possible configuration values are present in this 11 | # autogenerated file. 12 | # 13 | # All configuration values have a default; values that are commented out 14 | # serve to show the default. 15 | 16 | import sys 17 | import os 18 | 19 | # If extensions (or modules to document with autodoc) are in another 20 | # directory, add these directories to sys.path here. If the directory is 21 | # relative to the documentation root, use os.path.abspath to make it 22 | # absolute, like shown here. 23 | #sys.path.insert(0, os.path.abspath('.')) 24 | 25 | # Get the project root dir, which is the parent dir of this 26 | cwd = os.getcwd() 27 | project_root = os.path.dirname(cwd) 28 | 29 | # Insert the project root dir as the first element in the PYTHONPATH. 30 | # This lets us ensure that the source package is imported, and that its 31 | # version is used. 32 | sys.path.insert(0, project_root) 33 | 34 | import simple_elasticsearch 35 | import alabaster 36 | 37 | html_theme_path = [alabaster.get_path()] 38 | # extensions = ['alabaster'] 39 | html_theme = 'alabaster' 40 | html_sidebars = { 41 | '**': [ 42 | 'about.html', 'navigation.html', 'searchbox.html', 'donate.html', 43 | ] 44 | } 45 | html_theme_options = { 46 | # 'logo': 'logo.png', 47 | 'github_user': 'jaddison', 48 | 'github_repo': 'django-simple-elasticsearch', 49 | 'github_banner': True 50 | } 51 | # -- General configuration --------------------------------------------- 52 | 53 | # If your documentation needs a minimal Sphinx version, state it here. 54 | #needs_sphinx = '1.0' 55 | 56 | # Add any Sphinx extension module names here, as strings. They can be 57 | # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom ones. 58 | extensions = ['sphinx.ext.autodoc', 'sphinx.ext.viewcode', 'alabaster'] 59 | 60 | # Add any paths that contain templates here, relative to this directory. 61 | templates_path = ['_templates'] 62 | 63 | # The suffix of source filenames. 64 | source_suffix = '.rst' 65 | 66 | # The encoding of source files. 67 | #source_encoding = 'utf-8-sig' 68 | 69 | # The master toctree document. 70 | master_doc = 'index' 71 | 72 | # General information about the project. 73 | project = 'Django Simple Elasticsearch' 74 | copyright = '2014, James Addison' 75 | 76 | # The version info for the project you're documenting, acts as replacement 77 | # for |version| and |release|, also used in various other places throughout 78 | # the built documents. 79 | # 80 | # The short X.Y version. 81 | version = simple_elasticsearch.__version__ 82 | # The full version, including alpha/beta/rc tags. 83 | release = simple_elasticsearch.__version__ 84 | 85 | # The language for content autogenerated by Sphinx. Refer to documentation 86 | # for a list of supported languages. 87 | #language = None 88 | 89 | # There are two options for replacing |today|: either, you set today to 90 | # some non-false value, then it is used: 91 | #today = '' 92 | # Else, today_fmt is used as the format for a strftime call. 93 | #today_fmt = '%B %d, %Y' 94 | 95 | # List of patterns, relative to source directory, that match files and 96 | # directories to ignore when looking for source files. 97 | exclude_patterns = ['_build'] 98 | 99 | # The reST default role (used for this markup: `text`) to use for all 100 | # documents. 101 | #default_role = None 102 | 103 | # If true, '()' will be appended to :func: etc. cross-reference text. 104 | #add_function_parentheses = True 105 | 106 | # If true, the current module name will be prepended to all description 107 | # unit titles (such as .. function::). 108 | #add_module_names = True 109 | 110 | # If true, sectionauthor and moduleauthor directives will be shown in the 111 | # output. They are ignored by default. 112 | #show_authors = False 113 | 114 | # The name of the Pygments (syntax highlighting) style to use. 115 | pygments_style = 'sphinx' 116 | 117 | # A list of ignored prefixes for module index sorting. 118 | #modindex_common_prefix = [] 119 | 120 | # If true, keep warnings as "system message" paragraphs in the built 121 | # documents. 122 | #keep_warnings = False 123 | 124 | 125 | # -- Options for HTML output ------------------------------------------- 126 | 127 | # The theme to use for HTML and HTML Help pages. See the documentation for 128 | # a list of builtin themes. 129 | # html_theme = 'default' 130 | 131 | # Theme options are theme-specific and customize the look and feel of a 132 | # theme further. For a list of options available for each theme, see the 133 | # documentation. 134 | #html_theme_options = {} 135 | 136 | # Add any paths that contain custom themes here, relative to this directory. 137 | #html_theme_path = [] 138 | 139 | # The name for this set of Sphinx documents. If None, it defaults to 140 | # " v documentation". 141 | #html_title = None 142 | 143 | # A shorter title for the navigation bar. Default is the same as 144 | # html_title. 145 | #html_short_title = None 146 | 147 | # The name of an image file (relative to this directory) to place at the 148 | # top of the sidebar. 149 | #html_logo = None 150 | 151 | # The name of an image file (within the static path) to use as favicon 152 | # of the docs. This file should be a Windows icon file (.ico) being 153 | # 16x16 or 32x32 pixels large. 154 | #html_favicon = None 155 | 156 | # Add any paths that contain custom static files (such as style sheets) 157 | # here, relative to this directory. They are copied after the builtin 158 | # static files, so a file named "default.css" will overwrite the builtin 159 | # "default.css". 160 | html_static_path = ['_static'] 161 | 162 | # If not '', a 'Last updated on:' timestamp is inserted at every page 163 | # bottom, using the given strftime format. 164 | #html_last_updated_fmt = '%b %d, %Y' 165 | 166 | # If true, SmartyPants will be used to convert quotes and dashes to 167 | # typographically correct entities. 168 | #html_use_smartypants = True 169 | 170 | # Custom sidebar templates, maps document names to template names. 171 | #html_sidebars = {} 172 | 173 | # Additional templates that should be rendered to pages, maps page names 174 | # to template names. 175 | #html_additional_pages = {} 176 | 177 | # If false, no module index is generated. 178 | #html_domain_indices = True 179 | 180 | # If false, no index is generated. 181 | #html_use_index = True 182 | 183 | # If true, the index is split into individual pages for each letter. 184 | #html_split_index = False 185 | 186 | # If true, links to the reST sources are added to the pages. 187 | #html_show_sourcelink = True 188 | 189 | # If true, "Created using Sphinx" is shown in the HTML footer. 190 | # Default is True. 191 | #html_show_sphinx = True 192 | 193 | # If true, "(C) Copyright ..." is shown in the HTML footer. 194 | # Default is True. 195 | #html_show_copyright = True 196 | 197 | # If true, an OpenSearch description file will be output, and all pages 198 | # will contain a tag referring to it. The value of this option 199 | # must be the base URL from which the finished HTML is served. 200 | #html_use_opensearch = '' 201 | 202 | # This is the file name suffix for HTML files (e.g. ".xhtml"). 203 | #html_file_suffix = None 204 | 205 | # Output file base name for HTML help builder. 206 | htmlhelp_basename = 'simple_elasticsearchdoc' 207 | 208 | 209 | # -- Options for LaTeX output ------------------------------------------ 210 | 211 | latex_elements = { 212 | # The paper size ('letterpaper' or 'a4paper'). 213 | #'papersize': 'letterpaper', 214 | 215 | # The font size ('10pt', '11pt' or '12pt'). 216 | #'pointsize': '10pt', 217 | 218 | # Additional stuff for the LaTeX preamble. 219 | #'preamble': '', 220 | } 221 | 222 | # Grouping the document tree into LaTeX files. List of tuples 223 | # (source start file, target name, title, author, documentclass 224 | # [howto/manual]). 225 | latex_documents = [ 226 | ('index', 'simple_elasticsearch.tex', 227 | 'Django Simple Elasticsearch Documentation', 228 | 'James Addison', 'manual'), 229 | ] 230 | 231 | # The name of an image file (relative to this directory) to place at 232 | # the top of the title page. 233 | #latex_logo = None 234 | 235 | # For "manual" documents, if this is true, then toplevel headings 236 | # are parts, not chapters. 237 | #latex_use_parts = False 238 | 239 | # If true, show page references after internal links. 240 | #latex_show_pagerefs = False 241 | 242 | # If true, show URL addresses after external links. 243 | #latex_show_urls = False 244 | 245 | # Documents to append as an appendix to all manuals. 246 | #latex_appendices = [] 247 | 248 | # If false, no module index is generated. 249 | #latex_domain_indices = True 250 | 251 | 252 | # -- Options for manual page output ------------------------------------ 253 | 254 | # One entry per manual page. List of tuples 255 | # (source start file, name, description, authors, manual section). 256 | man_pages = [ 257 | ('index', 'simple_elasticsearch', 258 | 'Django Simple Elasticsearch Documentation', 259 | ['James Addison'], 1) 260 | ] 261 | 262 | # If true, show URL addresses after external links. 263 | #man_show_urls = False 264 | 265 | 266 | # -- Options for Texinfo output ---------------------------------------- 267 | 268 | # Grouping the document tree into Texinfo files. List of tuples 269 | # (source start file, target name, title, author, 270 | # dir menu entry, description, category) 271 | texinfo_documents = [ 272 | ('index', 'simple_elasticsearch', 273 | 'Django Simple Elasticsearch Documentation', 274 | 'James Addison', 275 | 'simple_elasticsearch', 276 | 'One line description of project.', 277 | 'Miscellaneous'), 278 | ] 279 | 280 | # Documents to append as an appendix to all manuals. 281 | #texinfo_appendices = [] 282 | 283 | # If false, no module index is generated. 284 | #texinfo_domain_indices = True 285 | 286 | # How to display URL addresses: 'footnote', 'no', or 'inline'. 287 | #texinfo_show_urls = 'footnote' 288 | 289 | # If true, do not generate a @detailmenu in the "Top" node's menu. 290 | #texinfo_no_detailmenu = False 291 | -------------------------------------------------------------------------------- /simple_elasticsearch/tests.py: -------------------------------------------------------------------------------- 1 | import copy 2 | from datadiff import tools as ddtools 3 | from django.core.paginator import Page 4 | from django.test import TestCase 5 | from elasticsearch import Elasticsearch 6 | import mock 7 | 8 | try: 9 | # `reload` is not a python3 builtin like python2 10 | reload 11 | except NameError: 12 | from imp import reload 13 | 14 | from . import settings as es_settings 15 | from .search import SimpleSearch 16 | from .mixins import ElasticsearchTypeMixin 17 | from .models import Blog, BlogPost 18 | 19 | 20 | class ElasticsearchTypeMixinClass(ElasticsearchTypeMixin): 21 | pass 22 | 23 | 24 | class ElasticsearchTypeMixinTestCase(TestCase): 25 | 26 | @property 27 | def latest_post(self): 28 | return BlogPost.objects.select_related('blog').latest('id') 29 | 30 | @mock.patch('simple_elasticsearch.mixins.Elasticsearch.delete') 31 | @mock.patch('simple_elasticsearch.mixins.Elasticsearch.index') 32 | def setUp(self, mock_index, mock_delete): 33 | self.blog = Blog.objects.create( 34 | name='test blog name', 35 | description='test blog description' 36 | ) 37 | 38 | # hack the return value to ensure we save some BlogPosts here; 39 | # without this mock, the post_save handler indexing blows up 40 | # as there is no real ES instance running 41 | mock_index.return_value = mock_delete.return_value = {} 42 | 43 | post = BlogPost.objects.create( 44 | blog=self.blog, 45 | title="DO-NOT-INDEX title", 46 | slug="DO-NOT-INDEX", 47 | body="DO-NOT-INDEX body" 48 | ) 49 | 50 | for x in range(1, 10): 51 | BlogPost.objects.create( 52 | blog=self.blog, 53 | title="blog post title {0}".format(x), 54 | slug="blog-post-title-{0}".format(x), 55 | body="blog post body {0}".format(x) 56 | ) 57 | 58 | def test__get_es__with_default_settings(self): 59 | result = BlogPost.get_es() 60 | self.assertIsInstance(result, Elasticsearch) 61 | self.assertEqual(result.transport.hosts[0]['host'], '127.0.0.1') 62 | self.assertEqual(result.transport.hosts[0]['port'], 9200) 63 | 64 | def test__get_es__with_custom_server(self): 65 | # include a custom class here as the internal `_es` is cached, so can't reuse the 66 | # `ElasticsearchIndexClassDefaults` global class (see above). 67 | class ElasticsearchIndexClassCustomSettings(ElasticsearchTypeMixin): 68 | pass 69 | 70 | with self.settings(ELASTICSEARCH_SERVER=['search.example.com:9201']): 71 | reload(es_settings) 72 | result = ElasticsearchIndexClassCustomSettings.get_es() 73 | self.assertIsInstance(result, Elasticsearch) 74 | self.assertEqual(result.transport.hosts[0]['host'], 'search.example.com') 75 | self.assertEqual(result.transport.hosts[0]['port'], 9201) 76 | 77 | reload(es_settings) 78 | 79 | def test__get_es__with_custom_connection_settings(self): 80 | # include a custom class here as the internal `_es` is cached, so can't reuse the 81 | # `ElasticsearchIndexClassDefaults` global class (see above). 82 | class ElasticsearchIndexClassCustomSettings(ElasticsearchTypeMixin): 83 | pass 84 | 85 | with self.settings(ELASTICSEARCH_CONNECTION_PARAMS={'hosts': ['search2.example.com:9202'], 'sniffer_timeout': 15}): 86 | reload(es_settings) 87 | result = ElasticsearchIndexClassCustomSettings.get_es() 88 | self.assertIsInstance(result, Elasticsearch) 89 | self.assertEqual(result.transport.hosts[0]['host'], 'search2.example.com') 90 | self.assertEqual(result.transport.hosts[0]['port'], 9202) 91 | self.assertEqual(result.transport.sniffer_timeout, 15) 92 | reload(es_settings) 93 | 94 | @mock.patch('simple_elasticsearch.mixins.ElasticsearchTypeMixin.index_add_or_delete') 95 | def test__save_handler(self, mock_index_add_or_delete): 96 | # with a create call 97 | post = BlogPost.objects.create( 98 | blog=self.blog, 99 | title="blog post title foo", 100 | slug="blog-post-title-foo", 101 | body="blog post body foo" 102 | ) 103 | mock_index_add_or_delete.assert_called_with(post) 104 | mock_index_add_or_delete.reset_mock() 105 | 106 | # with a plain save call 107 | post.save() 108 | mock_index_add_or_delete.assert_called_with(post) 109 | 110 | @mock.patch('simple_elasticsearch.mixins.ElasticsearchTypeMixin.index_delete') 111 | def test__delete_handler(self, mock_index_delete): 112 | post = self.latest_post 113 | post.delete() 114 | mock_index_delete.assert_called_with(post) 115 | 116 | @mock.patch('simple_elasticsearch.mixins.Elasticsearch.index') 117 | def test__index_add(self, mock_index): 118 | post = self.latest_post 119 | mock_index.return_value = {} 120 | 121 | # make sure an invalid object passed in returns False 122 | result = BlogPost.index_add(None) 123 | self.assertFalse(result) 124 | 125 | # make sure indexing an item calls Elasticsearch.index() with 126 | # the correct variables, with normal index name 127 | result = BlogPost.index_add(post) 128 | self.assertTrue(result) 129 | mock_index.assert_called_with('blog', 'posts', BlogPost.get_document(post), post.pk, routing=1) 130 | 131 | # make sure indexing an item calls Elasticsearch.index() with 132 | # the correct variables, with non-standard index name 133 | result = BlogPost.index_add(post, 'foo') 134 | self.assertTrue(result) 135 | mock_index.assert_called_with('foo', 'posts', BlogPost.get_document(post), post.pk, routing=1) 136 | 137 | # this one should not index (return false) because the 138 | # 'should_index' for this post should make it skip it 139 | post = BlogPost.objects.get(slug="DO-NOT-INDEX") 140 | result = BlogPost.index_add(post) 141 | self.assertFalse(result) 142 | 143 | @mock.patch('simple_elasticsearch.mixins.Elasticsearch.delete') 144 | def test__index_delete(self, mock_delete): 145 | post = self.latest_post 146 | mock_delete.return_value = { 147 | "acknowledged": True 148 | } 149 | 150 | # make sure an invalid object passed in returns False 151 | result = BlogPost.index_delete(None) 152 | self.assertFalse(result) 153 | 154 | # make sure deleting an item calls Elasticsearch.delete() with 155 | # the correct variables, with normal index name 156 | result = BlogPost.index_delete(post) 157 | self.assertTrue(result) 158 | mock_delete.assert_called_with('blog', 'posts', post.pk, routing=1) 159 | 160 | # make sure deleting an item calls Elasticsearch.delete() with 161 | # the correct variables, with non-standard index name 162 | result = BlogPost.index_delete(post, 'foo') 163 | self.assertTrue(result) 164 | mock_delete.assert_called_with('foo', 'posts', post.pk, routing=1) 165 | 166 | @mock.patch('simple_elasticsearch.mixins.ElasticsearchTypeMixin.index_add') 167 | @mock.patch('simple_elasticsearch.mixins.ElasticsearchTypeMixin.index_delete') 168 | def test__index_add_or_delete(self, mock_index_delete, mock_index_add): 169 | # invalid object passed in, should return False 170 | result = BlogPost.index_add_or_delete(None) 171 | self.assertFalse(result) 172 | 173 | # this one should not index (return false) because the 174 | # `should_index` for this post should make it skip it; 175 | # `index_delete` should get called 176 | mock_index_delete.return_value = True 177 | post = BlogPost.objects.get(slug="DO-NOT-INDEX") 178 | 179 | result = BlogPost.index_add_or_delete(post) 180 | self.assertTrue(result) 181 | mock_index_delete.assert_called_with(post, '') 182 | 183 | result = BlogPost.index_add_or_delete(post, 'foo') 184 | self.assertTrue(result) 185 | mock_index_delete.assert_called_with(post, 'foo') 186 | 187 | # `index_add` call results below 188 | mock_index_add.return_value = True 189 | post = self.latest_post 190 | 191 | result = BlogPost.index_add_or_delete(post) 192 | self.assertTrue(result) 193 | mock_index_add.assert_called_with(post, '') 194 | 195 | result = BlogPost.index_add_or_delete(post, 'foo') 196 | self.assertTrue(result) 197 | mock_index_add.assert_called_with(post, 'foo') 198 | 199 | def test__get_index_name(self): 200 | self.assertEqual(BlogPost.get_index_name(), 'blog') 201 | 202 | def test__get_type_name(self): 203 | self.assertEqual(BlogPost.get_type_name(), 'posts') 204 | 205 | def test__get_queryset(self): 206 | queryset = BlogPost.objects.all().select_related('blog').order_by('pk') 207 | self.assertEqual(list(BlogPost.get_queryset().order_by('pk')), list(queryset)) 208 | 209 | def test__get_index_name_notimplemented(self): 210 | with self.assertRaises(NotImplementedError): 211 | ElasticsearchTypeMixinClass.get_index_name() 212 | 213 | def test__get_type_name_notimplemented(self): 214 | with self.assertRaises(NotImplementedError): 215 | ElasticsearchTypeMixinClass.get_type_name() 216 | 217 | def test__get_queryset_notimplemented(self): 218 | with self.assertRaises(NotImplementedError): 219 | ElasticsearchTypeMixinClass.get_queryset() 220 | 221 | def test__get_type_mapping(self): 222 | mapping = { 223 | "properties": { 224 | "created_at": { 225 | "type": "date", 226 | "format": "dateOptionalTime" 227 | }, 228 | "title": { 229 | "type": "string" 230 | }, 231 | "body": { 232 | "type": "string" 233 | }, 234 | "slug": { 235 | "type": "string" 236 | }, 237 | "blog": { 238 | "properties": { 239 | "id": { 240 | "type": "long" 241 | }, 242 | "name": { 243 | "type": "string" 244 | }, 245 | "description": { 246 | "type": "string" 247 | } 248 | } 249 | } 250 | } 251 | } 252 | self.assertEqual(BlogPost.get_type_mapping(), mapping) 253 | 254 | def test__get_type_mapping_notimplemented(self): 255 | self.assertEqual(ElasticsearchTypeMixinClass.get_type_mapping(), {}) 256 | 257 | def test__get_request_params(self): 258 | post = self.latest_post 259 | # TODO: implement the method to test it works properly 260 | self.assertEqual(BlogPost.get_request_params(post), {'routing':1}) 261 | 262 | def test__get_request_params_notimplemented(self): 263 | self.assertEqual(ElasticsearchTypeMixinClass.get_request_params(1), {}) 264 | 265 | def test__get_bulk_index_limit(self): 266 | self.assertTrue(str(BlogPost.get_bulk_index_limit()).isdigit()) 267 | 268 | def test__get_query_limit(self): 269 | self.assertTrue(str(BlogPost.get_query_limit()).isdigit()) 270 | 271 | def test__get_document_id(self): 272 | post = self.latest_post 273 | result = BlogPost.get_document_id(post) 274 | self.assertEqual(result, post.pk) 275 | 276 | def test__get_document(self): 277 | post = self.latest_post 278 | result = BlogPost.get_document(post) 279 | self.assertEqual(result, { 280 | 'title': post.title, 281 | 'slug': post.slug, 282 | 'blog': { 283 | 'id': post.blog.pk, 284 | 'description': post.blog.description, 285 | 'name': post.blog.name 286 | }, 287 | 'created_at': post.created_at, 288 | 'body': post.body 289 | }) 290 | 291 | def test__get_document_notimplemented(self): 292 | with self.assertRaises(NotImplementedError): 293 | ElasticsearchTypeMixinClass.get_document(1) 294 | 295 | @mock.patch('simple_elasticsearch.mixins.Elasticsearch.index') 296 | def test__should_index(self, mock_index): 297 | post = self.latest_post 298 | self.assertTrue(BlogPost.should_index(post)) 299 | 300 | mock_index.return_value = {} 301 | post = BlogPost.objects.get(slug="DO-NOT-INDEX") 302 | self.assertFalse(BlogPost.should_index(post)) 303 | 304 | def test__should_index_notimplemented(self): 305 | self.assertTrue(ElasticsearchTypeMixinClass.should_index(1)) 306 | 307 | @mock.patch('simple_elasticsearch.mixins.queryset_iterator') 308 | def test__bulk_index_queryset(self, mock_queryset_iterator): 309 | queryset = BlogPost.get_queryset().exclude(slug='DO-NOT-INDEX') 310 | BlogPost.bulk_index(queryset=queryset) 311 | mock_queryset_iterator.assert_called_with(queryset, BlogPost.get_query_limit(), 'pk') 312 | 313 | mock_queryset_iterator.reset_mock() 314 | 315 | queryset = BlogPost.get_queryset() 316 | BlogPost.bulk_index() 317 | # to compare QuerySets, they must first be converted to lists. 318 | self.assertEqual(list(mock_queryset_iterator.call_args[0][0]), list(queryset)) 319 | 320 | mock_queryset_iterator.reset_mock() 321 | 322 | # hack in a test for ensuring the proper bulk ordering is used 323 | BlogPost.bulk_ordering = 'created_at' 324 | BlogPost.bulk_index(queryset=queryset) 325 | mock_queryset_iterator.assert_called_with(queryset, BlogPost.get_query_limit(), 'created_at') 326 | BlogPost.bulk_ordering = 'pk' 327 | 328 | 329 | @mock.patch('simple_elasticsearch.models.BlogPost.get_document') 330 | @mock.patch('simple_elasticsearch.models.BlogPost.should_index') 331 | @mock.patch('simple_elasticsearch.mixins.Elasticsearch.bulk') 332 | def test__bulk_index_should_index(self, mock_bulk, mock_should_index, mock_get_document): 333 | # hack the return value to ensure we save some BlogPosts here; 334 | # without this mock, the post_save handler indexing blows up 335 | # as there is no real ES instance running 336 | mock_bulk.return_value = {} 337 | 338 | queryset_count = BlogPost.get_queryset().count() 339 | BlogPost.bulk_index() 340 | self.assertTrue(mock_should_index.call_count == queryset_count) 341 | 342 | @mock.patch('simple_elasticsearch.models.BlogPost.get_document') 343 | @mock.patch('simple_elasticsearch.mixins.Elasticsearch.bulk') 344 | def test__bulk_index_get_document(self, mock_bulk, mock_get_document): 345 | mock_bulk.return_value = mock_get_document.return_value = {} 346 | 347 | queryset_count = BlogPost.get_queryset().count() 348 | BlogPost.bulk_index() 349 | 350 | # One of the items is not meant to be indexed (slug='DO-NOT-INDEX'), so the 351 | # get_document function will get called one less time due to this. 352 | self.assertTrue(mock_get_document.call_count == (queryset_count - 1)) 353 | 354 | @mock.patch('simple_elasticsearch.mixins.Elasticsearch.bulk') 355 | def test__bulk_index_bulk(self, mock_bulk): 356 | mock_bulk.return_value = {} 357 | 358 | queryset_count = BlogPost.get_queryset().count() 359 | BlogPost.bulk_index() 360 | 361 | # figure out how many times es.bulk() should get called in the 362 | # .bulk_index() method and verify it's the same 363 | bulk_times = int(queryset_count / BlogPost.get_bulk_index_limit()) + 1 364 | self.assertTrue(mock_bulk.call_count == bulk_times) 365 | 366 | 367 | class SimpleSearchTestCase(TestCase): 368 | 369 | def setUp(self): 370 | self.query = {'q': 'python'} 371 | 372 | def test__esp_reset(self): 373 | esp = SimpleSearch() 374 | 375 | self.assertTrue(len(esp.bulk_search_data) == 0) 376 | self.assertTrue(len(esp.page_ranges) == 0) 377 | 378 | esp.add_search({ 379 | "query": { 380 | "match": { 381 | "_all": "foobar" 382 | } 383 | } 384 | }) 385 | 386 | self.assertFalse(len(esp.bulk_search_data) == 0) 387 | self.assertFalse(len(esp.page_ranges) == 0) 388 | 389 | esp.reset() 390 | 391 | self.assertTrue(len(esp.bulk_search_data) == 0) 392 | self.assertTrue(len(esp.page_ranges) == 0) 393 | 394 | def test__esp_add_query_dict(self): 395 | esp = SimpleSearch() 396 | 397 | page = 1 398 | page_size = 20 399 | 400 | query = { 401 | "query": { 402 | "match": { 403 | "_all": "foobar" 404 | } 405 | } 406 | } 407 | 408 | # SimpleSearch internally sets the from/size parameters 409 | # on the query; we need to compare with those values included 410 | query_with_size = query.copy() 411 | query_with_size.update({ 412 | 'from': (page - 1) * page_size, 413 | 'size': page_size 414 | }) 415 | 416 | esp.add_search(query.copy()) 417 | ddtools.assert_equal(esp.bulk_search_data[0], {}) 418 | ddtools.assert_equal(esp.bulk_search_data[1], query_with_size) 419 | 420 | esp.reset() 421 | esp.add_search(query.copy(), index='blog') 422 | ddtools.assert_equal(esp.bulk_search_data[0], {'index': 'blog'}) 423 | ddtools.assert_equal(esp.bulk_search_data[1], query_with_size) 424 | 425 | esp.reset() 426 | esp.add_search(query.copy(), index='blog', doc_type='posts') 427 | ddtools.assert_equal(esp.bulk_search_data[0], {'index': 'blog', 'type': 'posts'}) 428 | ddtools.assert_equal(esp.bulk_search_data[1], query_with_size) 429 | 430 | @mock.patch('simple_elasticsearch.search.Elasticsearch.msearch') 431 | def test__esp_search(self, mock_msearch): 432 | mock_msearch.return_value = { 433 | "responses": [ 434 | { 435 | "hits": { 436 | "total": 20, 437 | "hits": [ 438 | { 439 | "_index": "blog", 440 | "_type": "posts", 441 | "_id": "1", 442 | "_score": 1.0, 443 | "_source": {"account_number": 1,} 444 | }, { 445 | "_index": "blog", 446 | "_type": "posts", 447 | "_id": "6", 448 | "_score": 1.0, 449 | "_source": {"account_number": 6,} 450 | } 451 | ] 452 | } 453 | } 454 | ] 455 | } 456 | 457 | esp = SimpleSearch() 458 | esp.add_search({}, 3, 2, index='blog', doc_type='posts') 459 | 460 | bulk_data = copy.deepcopy(esp.bulk_search_data) 461 | ddtools.assert_equal(bulk_data, [{'index': 'blog', 'type': 'posts'}, {'from': 4, 'size': 2}]) 462 | 463 | responses = esp.search() 464 | mock_msearch.assert_called_with(bulk_data) 465 | 466 | # ensure that our hack to get size and from into the hit 467 | # data works 468 | self.assertEqual(responses[0]._page_num, 3) 469 | self.assertEqual(responses[0]._page_size, 2) 470 | 471 | # ensure that the bulk data gets reset 472 | self.assertEqual(len(esp.bulk_search_data), 0) 473 | 474 | page = responses[0].page 475 | self.assertIsInstance(page, Page) 476 | self.assertEqual(page.number, 3) 477 | self.assertTrue(page.has_next()) 478 | self.assertTrue(page.has_previous()) 479 | self.assertEqual(len(list(page)), 2) # 2 items on the page 480 | 481 | @mock.patch('simple_elasticsearch.search.Elasticsearch.msearch') 482 | def test__esp_search2(self, mock_msearch): 483 | mock_msearch.return_value = { 484 | "responses": [ 485 | { 486 | "hits": { 487 | "total": 20, 488 | "hits": [ 489 | { 490 | "_index": "blog", 491 | "_type": "posts", 492 | "_id": "1", 493 | "_score": 1.0, 494 | "_source": {"account_number": 1,} 495 | }, { 496 | "_index": "blog", 497 | "_type": "posts", 498 | "_id": "6", 499 | "_score": 1.0, 500 | "_source": {"account_number": 6,} 501 | } 502 | ] 503 | } 504 | } 505 | ] 506 | } 507 | 508 | esp = SimpleSearch() 509 | esp.add_search({}, 1, 2, index='blog', doc_type='posts') 510 | responses = esp.search() 511 | page = responses[0].page 512 | self.assertTrue(page.has_next()) 513 | self.assertFalse(page.has_previous()) 514 | --------------------------------------------------------------------------------