├── .editorconfig ├── .github └── workflows │ └── ci.yml ├── .gitignore ├── .readthedocs.yml ├── AUTHORS.rst ├── CONTRIBUTING.rst ├── HISTORY.rst ├── LICENSE ├── MANIFEST.in ├── README.rst ├── django_elasticsearch_dsl ├── __init__.py ├── apps.py ├── documents.py ├── exceptions.py ├── fields.py ├── indices.py ├── management │ ├── __init__.py │ └── commands │ │ ├── __init__.py │ │ └── search_index.py ├── models.py ├── registries.py ├── search.py ├── signals.py └── test │ ├── __init__.py │ └── testcases.py ├── docs ├── Makefile ├── _static │ └── .empty ├── make.bat └── source │ ├── about.rst │ ├── conf.py │ ├── develop.rst │ ├── es_index.rst │ ├── fields.rst │ ├── index.rst │ ├── management.rst │ ├── quickstart.rst │ └── settings.rst ├── example ├── README.rst ├── example │ ├── __init__.py │ ├── settings.py │ ├── urls.py │ └── wsgi.py ├── manage.py ├── requirements.txt └── test_app │ ├── __init__.py │ ├── admin.py │ ├── documents.py │ ├── migrations │ ├── 0001_initial.py │ ├── 0002_alter_ad_car_alter_ad_id_alter_car_id_and_more.py │ └── __init__.py │ └── models.py ├── requirements.txt ├── requirements_dev.txt ├── requirements_test.txt ├── runtests.py ├── setup.cfg ├── setup.py ├── tests ├── __init__.py ├── documents.py ├── fixtures.py ├── models.py ├── test_commands.py ├── test_documents.py ├── test_fields.py ├── test_indices.py ├── test_integration.py ├── test_registries.py └── test_signals.py └── tox.ini /.editorconfig: -------------------------------------------------------------------------------- 1 | # http://editorconfig.org 2 | 3 | root = true 4 | 5 | [*] 6 | charset = utf-8 7 | end_of_line = lf 8 | insert_final_newline = true 9 | trim_trailing_whitespace = true 10 | 11 | [*.{py,rst,ini}] 12 | indent_style = space 13 | indent_size = 4 14 | 15 | [*.{html,css,scss,json,yml}] 16 | indent_style = space 17 | indent_size = 2 18 | 19 | [*.md] 20 | trim_trailing_whitespace = false 21 | 22 | [Makefile] 23 | indent_style = tab 24 | -------------------------------------------------------------------------------- /.github/workflows/ci.yml: -------------------------------------------------------------------------------- 1 | name: CI 2 | 3 | on: 4 | push: 5 | pull_request: 6 | workflow_dispatch: 7 | 8 | jobs: 9 | test: 10 | runs-on: ubuntu-latest 11 | 12 | strategy: 13 | fail-fast: false 14 | 15 | matrix: 16 | python-version: ["3.8", "3.9", "3.10", "3.11"] 17 | django-version: ["3.2", "4.1", "4.2"] 18 | es-dsl-version: ["6.4", "7.4"] 19 | es-version: ["8.10.2"] 20 | 21 | exclude: 22 | - python-version: "3.11" 23 | django-version: "3.2" 24 | 25 | steps: 26 | - name: Install and Run Elasticsearch 27 | uses: elastic/elastic-github-actions/elasticsearch@master 28 | with: 29 | stack-version: ${{ matrix.es-version }} 30 | 31 | - uses: actions/checkout@v4 32 | 33 | - name: Install Python ${{ matrix.python-version }} 34 | uses: actions/setup-python@v4 35 | with: 36 | python-version: ${{ matrix.python-version }} 37 | 38 | - name: Cache Pip Dependencies 39 | uses: actions/cache@v3 40 | with: 41 | path: ~/.cache/pip 42 | key: ${{ runner.os }}-pip-${{ hashFiles('requirements_test.txt') }} 43 | restore-keys: | 44 | ${{ runner.os }}-pip- 45 | 46 | - name: Install Dependencies 47 | run: | 48 | python -m pip install --upgrade pip 49 | python -m pip install "Django==${{ matrix.django-version }}" 50 | python -m pip install "elasticsearch-dsl==${{ matrix.es-dsl-version }}" 51 | python -m pip install -r requirements_test.txt 52 | 53 | - name: Run tests with Python ${{ matrix.python-version }} and Django ${{ matrix.django-version }} and elasticsearch-dsl-py ${{ matrix.es-dsl-version }} 54 | run: | 55 | TOX_ENV=$(echo "py${{ matrix.python-version }}-django-${{ matrix.django-version }}-es${{ matrix.es-dsl-version }}" | tr -d .) 56 | python -m tox -e $TOX_ENV -- --elasticsearch 57 | python -m tox -e $TOX_ENV -- --elasticsearch --signal-processor celery 58 | 59 | - name: Publish Coverage Report 60 | uses: codecov/codecov-action@v3 61 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.py[cod] 2 | __pycache__ 3 | 4 | # C extensions 5 | *.so 6 | 7 | # Packages 8 | *.egg 9 | *.egg-info 10 | dist 11 | build 12 | eggs 13 | parts 14 | bin 15 | var 16 | sdist 17 | develop-eggs 18 | .installed.cfg 19 | lib 20 | lib64 21 | 22 | # Installer logs 23 | pip-log.txt 24 | 25 | # Unit test / coverage reports 26 | .coverage 27 | .tox 28 | nosetests.xml 29 | htmlcov 30 | 31 | # Translations 32 | *.mo 33 | 34 | # Mr Developer 35 | .mr.developer.cfg 36 | .project 37 | .pydevproject 38 | 39 | # Pycharm/Intellij 40 | .idea 41 | 42 | # Complexity 43 | output/*.html 44 | output/*/index.html 45 | 46 | # Sphinx 47 | docs/_buildenv* 48 | 49 | *.sqlite3 50 | _autofixture* 51 | -------------------------------------------------------------------------------- /.readthedocs.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | 3 | sphinx: 4 | configuration: docs/source/conf.py 5 | 6 | formats: all 7 | 8 | python: 9 | version: 3.8 10 | -------------------------------------------------------------------------------- /AUTHORS.rst: -------------------------------------------------------------------------------- 1 | ======= 2 | Credits 3 | ======= 4 | 5 | Development Lead 6 | ---------------- 7 | 8 | sabricot 9 | safwanrahman - Safwan Rahman 10 | 11 | Contributors 12 | ------------ 13 | 14 | markotibold 15 | HansAdema - Devhouse Spindle 16 | barseghyanartur 17 | -------------------------------------------------------------------------------- /CONTRIBUTING.rst: -------------------------------------------------------------------------------- 1 | ============ 2 | Contributing 3 | ============ 4 | 5 | Contributions are welcome, and they are greatly appreciated! Every 6 | little bit helps, and credit will always be given. 7 | 8 | You can contribute in many ways: 9 | 10 | Types of Contributions 11 | ---------------------- 12 | 13 | Report Bugs 14 | ~~~~~~~~~~~ 15 | 16 | Report bugs at https://github.com/sabricot/django-elasticsearch-dsl/issues. 17 | 18 | If you are reporting a bug, please include: 19 | 20 | * Your operating system name and version. 21 | * Any details about your local setup that might be helpful in troubleshooting. 22 | * Detailed steps to reproduce the bug. 23 | 24 | Fix Bugs 25 | ~~~~~~~~ 26 | 27 | Look through the GitHub issues for bugs. Anything tagged with "bug" 28 | is open to whoever wants to implement it. 29 | 30 | Implement Features 31 | ~~~~~~~~~~~~~~~~~~ 32 | 33 | Look through the GitHub issues for features. Anything tagged with "feature" 34 | is open to whoever wants to implement it. 35 | 36 | Write Documentation 37 | ~~~~~~~~~~~~~~~~~~~ 38 | 39 | Django Elasticsearch DSL could always use more documentation, whether as part of the 40 | official Django Elasticsearch DSL docs, in docstrings, or even on the web in blog posts, 41 | articles, and such. 42 | 43 | Submit Feedback 44 | ~~~~~~~~~~~~~~~ 45 | 46 | The best way to send feedback is to file an issue at https://github.com/sabricot/django-elasticsearch-dsl/issues. 47 | 48 | If you are proposing a feature: 49 | 50 | * Explain in detail how it would work. 51 | * Keep the scope as narrow as possible, to make it easier to implement. 52 | * Remember that this is a volunteer-driven project, and that contributions 53 | are welcome :) 54 | 55 | Get Started! 56 | ------------ 57 | 58 | Ready to contribute? Here's how to set up `django-elasticsearch-dsl` for local development. 59 | 60 | 1. Fork the `django-elasticsearch-dsl` repo on GitHub. 61 | 2. Clone your fork locally:: 62 | 63 | $ git clone git@github.com:your_name_here/django-elasticsearch-dsl.git 64 | 65 | 3. Install your local copy into a virtualenv. Assuming you have virtualenvwrapper installed, this is how you set up your fork for local development:: 66 | 67 | $ mkvirtualenv django-elasticsearch-dsl 68 | $ cd django-elasticsearch-dsl/ 69 | $ python setup.py develop 70 | 71 | 4. Create a branch for local development:: 72 | 73 | $ git checkout -b name-of-your-bugfix-or-feature 74 | 75 | Now you can make your changes locally. 76 | 77 | 5. When you're done making changes, check that your changes pass flake8 and the 78 | tests, including testing other Python versions with tox:: 79 | 80 | $ flake8 django_elasticsearch_dsl tests 81 | $ python setup.py test 82 | $ tox 83 | 84 | To get flake8 and tox, just pip install them into your virtualenv. 85 | 86 | 6. Commit your changes and push your branch to GitHub:: 87 | 88 | $ git add . 89 | $ git commit -m "Your detailed description of your changes." 90 | $ git push origin name-of-your-bugfix-or-feature 91 | 92 | 7. Submit a pull request through the GitHub website. 93 | 94 | Pull Request Guidelines 95 | ----------------------- 96 | 97 | Before you submit a pull request, check that it meets these guidelines: 98 | 99 | 1. The pull request should include tests. 100 | 2. If the pull request adds functionality, the docs should be updated. Put 101 | your new functionality into a function with a docstring, and add the 102 | feature to the list in README.rst. 103 | 3. The pull request should work for Python 2.7, and 3.4, and for PyPy. Check 104 | https://github.com/django-es/django-elasticsearch-dsl/actions 105 | and make sure that the tests pass for all supported Python versions. 106 | 107 | -------------------------------------------------------------------------------- /HISTORY.rst: -------------------------------------------------------------------------------- 1 | .. :changelog: 2 | 3 | History 4 | ------- 5 | 7.1.4 (2020-07-05) 6 | ~~~~~~~~~~~~~~~~~~ 7 | * Configure Elasticsearch _id dynamically from document (#272) 8 | * Use chain.from_iterable in for performance improvement (#278) 9 | * Handle case where SimpleLazyObject being treated as an Iterable (#255) 10 | * Camelcase default value in management command (#254) 11 | * Various updates and fixup in docs (#250, #276) 12 | * Start testing against Python 3.8 (#266) 13 | 14 | 15 | 7.1.1 (2019-12-26) 16 | ~~~~~~~~~~~~~~~~~~ 17 | * Adding detailed documentation and published to Read The Docs #222 18 | * Resolve name resolution while delete, create index (#228) 19 | * Added support for Django 3.0. (#230) 20 | * Removing old Elasticsearc compatibility (#219) 21 | * Drop StringField in favor of TextField. 22 | 23 | 24 | 7.1.0 (2019-10-29) 25 | ~~~~~~~~~~~~~~~~~~ 26 | * Support for Django `DecimalField` #141 27 | * Indexing speedup by using `parallel` indexing. #213. 28 | Now you can pass `--parallel` or set `ELASTICSEARCH_DSL_PARALLEL` 29 | in your settings to get indexing speed boost while indexing 30 | through management command. 31 | * Fixing name resolution in management command #206 32 | * Small documentation fixes. #196 33 | 34 | 35 | 7.0.0 (2019-08-11) 36 | ~~~~~~~~~~~~~~~~~~ 37 | * Support Elasticsearch 7.0 (See PR #176) 38 | * Added order by to paginate queryset properly (See PR #153) 39 | * Remove `standard` token filter from `README.md` and test files 40 | * Various documentation fixes 41 | 42 | 43 | 6.4.2 (2019-07-26) 44 | ~~~~~~~~~~~~~~~~~~ 45 | * Fix document importing path 46 | * Update readme 47 | 48 | 49 | 50 | 6.4.1 (2019-06-14) 51 | ~~~~~~~~~~~~~~~~~~ 52 | * The `DocType` import has changed to `Document` 53 | 54 | 55 | 56 | 6.4.0 (2019-06-01) 57 | ~~~~~~~~~~~~~~~~~~ 58 | * Support elasticsearch-dsl>6.3.0 59 | * Class `Meta` has changed to class `Django` (See PR #136) 60 | * Add `register_document` decorator to register a document (See PR #136) 61 | * Additional Bug fixing and others 62 | 63 | 64 | 0.5.1 (2018-11-07) 65 | ~~~~~~~~~~~~~~~~~~ 66 | * Limit elastsearch-dsl to supported versions 67 | 68 | 0.5.0 (2018-04-22) 69 | ~~~~~~~~~~~~~~~~~~ 70 | * Add Support for Elasticsearch 6 thanks to HansAdema 71 | 72 | Breaking Change: 73 | ~~~~~~~~~~~~~~~~ 74 | Django string fields now point to ES text field by default. 75 | Nothing should change for ES 2.X but if you are using ES 5.X, 76 | you may need to rebuild and/or update some of your documents. 77 | 78 | 79 | 0.4.5 (2018-04-22) 80 | ~~~~~~~~~~~~~~~~~~ 81 | * Fix prepare with related models when deleted (See PR #99) 82 | * Fix unwanted calls to get_instances_from_related 83 | * Fix for empty ArrayField (CBinyenya) 84 | * Fix nested OneToOneField when related object doesn't exist (CBinyenya) 85 | * Update elasticsearch-dsl minimal version 86 | 87 | 0.4.4 (2017-12-13) 88 | ~~~~~~~~~~~~~~~~~~ 89 | * Fix to_queryset with es 5.0/5.1 90 | 91 | 0.4.3 (2017-12-12) 92 | ~~~~~~~~~~~~~~~~~~ 93 | * Fix syncing of related objects when deleted 94 | * Add django 2.0 support 95 | 96 | 0.4.2 (2017-11-27) 97 | ~~~~~~~~~~~~~~~~~~ 98 | * Convert lazy string to string before serialization 99 | * Readme update (arielpontes) 100 | 101 | 0.4.1 (2017-10-17) 102 | ~~~~~~~~~~~~~~~~~~ 103 | * Update example app with get_instances_from_related 104 | * Typo/grammar fixes 105 | 106 | 0.4.0 (2017-10-07) 107 | ~~~~~~~~~~~~~~~~~~ 108 | * Add a method on the Search class to return a django queryset from an es result 109 | * Add a queryset_pagination option to DocType.Meta for allow the pagination of 110 | big django querysets during the index populating 111 | * Remove the call to iterator method for the django queryset 112 | * Fix DocType inheritance. The DocType is store in the registry as a class and not anymore as an instance 113 | 114 | 115 | 0.3.0 (2017-10-01) 116 | ~~~~~~~~~~~~~~~~~~ 117 | * Add support for resynching ES documents if related models are updated (HansAdema) 118 | * Better management for django FileField and ImageField 119 | * Fix some errors in the doc (barseghyanartur, diwu1989) 120 | 121 | 0.2.0 (2017-07-02) 122 | ~~~~~~~~~~~~~~~~~~ 123 | * Replace simple model signals with easier to customise signal processors (barseghyanartur) 124 | * Add options to disable automatic index refreshes (HansAdema) 125 | * Support defining DocType indexes through Meta class (HansAdema) 126 | * Add option to set default Index settings through Django config (HansAdema) 127 | 128 | 0.1.0 (2017-05-26) 129 | ~~~~~~~~~~~~~~~~~~ 130 | * First release on PyPI. 131 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | 2 | Apache Software License 2.0 3 | 4 | Copyright (c) 2016, Sabre 5 | 6 | Licensed under the Apache License, Version 2.0 (the "License"); 7 | you may not use this file except in compliance with the License. 8 | You may obtain a copy of the License at 9 | 10 | http://www.apache.org/licenses/LICENSE-2.0 11 | 12 | Unless required by applicable law or agreed to in writing, software 13 | distributed under the License is distributed on an "AS IS" BASIS, 14 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | See the License for the specific language governing permissions and 16 | limitations under the License. 17 | 18 | 19 | 20 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include AUTHORS.rst 2 | include CONTRIBUTING.rst 3 | include HISTORY.rst 4 | include LICENSE 5 | include README.rst 6 | recursive-include django_elasticsearch_dsl *.html *.png *.gif *js *.css *jpg *jpeg *svg *py 7 | -------------------------------------------------------------------------------- /README.rst: -------------------------------------------------------------------------------- 1 | ======================== 2 | Django Elasticsearch DSL 3 | ======================== 4 | 5 | .. image:: https://github.com/django-es/django-elasticsearch-dsl/actions/workflows/ci.yml/badge.svg 6 | :target: https://github.com/django-es/django-elasticsearch-dsl/actions/workflows/ci.yml 7 | .. image:: https://codecov.io/gh/django-es/django-elasticsearch-dsl/coverage.svg?branch=master 8 | :target: https://codecov.io/gh/django-es/django-elasticsearch-dsl 9 | .. image:: https://badge.fury.io/py/django-elasticsearch-dsl.svg 10 | :target: https://pypi.python.org/pypi/django-elasticsearch-dsl 11 | .. image:: https://readthedocs.org/projects/django-elasticsearch-dsl/badge/?version=latest&style=flat 12 | :target: https://django-elasticsearch-dsl.readthedocs.io/en/latest/ 13 | 14 | Django Elasticsearch DSL is a package that allows indexing of django models in elasticsearch. 15 | It is built as a thin wrapper around elasticsearch-dsl-py_ 16 | so you can use all the features developed by the elasticsearch-dsl-py team. 17 | 18 | You can view the full documentation at https://django-elasticsearch-dsl.readthedocs.io 19 | 20 | .. _elasticsearch-dsl-py: https://github.com/elastic/elasticsearch-dsl-py 21 | 22 | Features 23 | -------- 24 | 25 | - Based on elasticsearch-dsl-py_ so you can make queries with the Search_ class. 26 | - Django signal receivers on save and delete for keeping Elasticsearch in sync. 27 | - Management commands for creating, deleting, rebuilding and populating indices. 28 | - Elasticsearch auto mapping from django models fields. 29 | - Complex field type support (ObjectField, NestedField). 30 | - Index fast using `parallel` indexing. 31 | - Requirements 32 | 33 | - Django >= 3.2 34 | - Python 3.8, 3.9, 3.10, 3.11 35 | 36 | **Elasticsearch Compatibility:** 37 | The library is compatible with all Elasticsearch versions since 5.x 38 | **but you have to use a matching major version:** 39 | 40 | - For Elasticsearch 8.0 and later, use the major version 8 (8.x.y) of the library. 41 | 42 | - For Elasticsearch 7.0 and later, use the major version 7 (7.x.y) of the library. 43 | 44 | - For Elasticsearch 6.0 and later, use the major version 6 (6.x.y) of the library. 45 | 46 | .. code-block:: python 47 | 48 | # Elasticsearch 8.x 49 | elasticsearch-dsl>=8.0.0,<9.0.0 50 | 51 | # Elasticsearch 7.x 52 | elasticsearch-dsl>=7.0.0,<8.0.0 53 | 54 | # Elasticsearch 6.x 55 | elasticsearch-dsl>=6.0.0,<7.0.0 56 | 57 | .. _Search: http://elasticsearch-dsl.readthedocs.io/en/stable/search_dsl.html 58 | -------------------------------------------------------------------------------- /django_elasticsearch_dsl/__init__.py: -------------------------------------------------------------------------------- 1 | import django 2 | 3 | from django.utils.module_loading import autodiscover_modules 4 | 5 | from .documents import Document # noqa 6 | from .indices import Index # noqa 7 | from .fields import * # noqa 8 | 9 | __version__ = '7.1.1' 10 | 11 | 12 | def autodiscover(): 13 | autodiscover_modules('documents') 14 | 15 | 16 | -------------------------------------------------------------------------------- /django_elasticsearch_dsl/apps.py: -------------------------------------------------------------------------------- 1 | from django.apps import AppConfig 2 | from django.conf import settings 3 | from django.utils.module_loading import import_string 4 | 5 | from elasticsearch_dsl.connections import connections 6 | 7 | 8 | class DEDConfig(AppConfig): 9 | name = 'django_elasticsearch_dsl' 10 | verbose_name = "Django elasticsearch-dsl" 11 | signal_processor = None 12 | 13 | def ready(self): 14 | self.module.autodiscover() 15 | connections.configure(**settings.ELASTICSEARCH_DSL) 16 | # Setup the signal processor. 17 | if not self.signal_processor: 18 | signal_processor_path = getattr( 19 | settings, 20 | 'ELASTICSEARCH_DSL_SIGNAL_PROCESSOR', 21 | 'django_elasticsearch_dsl.signals.RealTimeSignalProcessor' 22 | ) 23 | signal_processor_class = import_string(signal_processor_path) 24 | self.signal_processor = signal_processor_class(connections) 25 | 26 | @classmethod 27 | def autosync_enabled(cls): 28 | return getattr(settings, 'ELASTICSEARCH_DSL_AUTOSYNC', True) 29 | 30 | @classmethod 31 | def default_index_settings(cls): 32 | return getattr(settings, 'ELASTICSEARCH_DSL_INDEX_SETTINGS', {}) 33 | 34 | @classmethod 35 | def auto_refresh_enabled(cls): 36 | return getattr(settings, 'ELASTICSEARCH_DSL_AUTO_REFRESH', True) 37 | -------------------------------------------------------------------------------- /django_elasticsearch_dsl/documents.py: -------------------------------------------------------------------------------- 1 | from __future__ import unicode_literals 2 | 3 | from collections import deque 4 | from fnmatch import fnmatch 5 | from functools import partial 6 | 7 | from django import VERSION as DJANGO_VERSION 8 | from django.db import models 9 | from elasticsearch.helpers import bulk, parallel_bulk 10 | from elasticsearch_dsl import Document as DSLDocument 11 | from six import iteritems 12 | 13 | from .exceptions import ModelFieldNotMappedError 14 | from .fields import ( 15 | BooleanField, 16 | DateField, 17 | DEDField, 18 | DoubleField, 19 | FileField, 20 | IntegerField, 21 | KeywordField, 22 | LongField, 23 | ShortField, 24 | TextField, TimeField, 25 | ) 26 | from .search import Search 27 | from .signals import post_index 28 | 29 | model_field_class_to_field_class = { 30 | models.AutoField: IntegerField, 31 | models.BigAutoField: LongField, 32 | models.BigIntegerField: LongField, 33 | models.BooleanField: BooleanField, 34 | models.CharField: TextField, 35 | models.DateField: DateField, 36 | models.DateTimeField: DateField, 37 | models.DecimalField: DoubleField, 38 | models.EmailField: TextField, 39 | models.FileField: FileField, 40 | models.FilePathField: KeywordField, 41 | models.FloatField: DoubleField, 42 | models.ImageField: FileField, 43 | models.IntegerField: IntegerField, 44 | models.NullBooleanField: BooleanField, 45 | models.PositiveIntegerField: IntegerField, 46 | models.PositiveSmallIntegerField: ShortField, 47 | models.SlugField: KeywordField, 48 | models.SmallIntegerField: ShortField, 49 | models.TextField: TextField, 50 | models.TimeField: TimeField, 51 | models.URLField: TextField, 52 | models.UUIDField: KeywordField, 53 | } 54 | 55 | if DJANGO_VERSION >= (3.1,): 56 | model_field_class_to_field_class[models.PositiveBigIntegerField] = LongField 57 | 58 | 59 | class DocType(DSLDocument): 60 | _prepared_fields = [] 61 | 62 | def __init__(self, related_instance_to_ignore=None, **kwargs): 63 | super(DocType, self).__init__(**kwargs) 64 | self._related_instance_to_ignore = related_instance_to_ignore 65 | self._prepared_fields = self.init_prepare() 66 | 67 | def __eq__(self, other): 68 | return id(self) == id(other) 69 | 70 | def __hash__(self): 71 | return id(self) 72 | 73 | @classmethod 74 | def _matches(cls, hit): 75 | """ 76 | Determine which index or indices in a pattern to be used in a hit. 77 | Overrides DSLDocument _matches function to match indices in a pattern, 78 | which is needed in case of using aliases. This is needed as the 79 | document class will not be used to deserialize the documents. The 80 | documents will have the index set to the concrete index, whereas the 81 | class refers to the alias. 82 | """ 83 | return fnmatch(hit.get("_index", ""), cls._index._name + "*") 84 | 85 | @classmethod 86 | def search(cls, using=None, index=None): 87 | return Search( 88 | using=cls._get_using(using), 89 | index=cls._default_index(index), 90 | doc_type=[cls], 91 | model=cls.django.model 92 | ) 93 | 94 | def get_queryset(self): 95 | """ 96 | Return the queryset that should be indexed by this doc type. 97 | """ 98 | return self.django.model._default_manager.all() 99 | 100 | def get_indexing_queryset(self): 101 | """ 102 | Build queryset (iterator) for use by indexing. 103 | """ 104 | qs = self.get_queryset() 105 | kwargs = {} 106 | if DJANGO_VERSION >= (2,) and self.django.queryset_pagination: 107 | kwargs = {'chunk_size': self.django.queryset_pagination} 108 | return qs.iterator(**kwargs) 109 | 110 | def init_prepare(self): 111 | """ 112 | Initialise the data model preparers once here. Extracts the preparers 113 | from the model and generate a list of callables to avoid doing that 114 | work on every object instance over. 115 | """ 116 | index_fields = getattr(self, '_fields', {}) 117 | fields = [] 118 | for name, field in iteritems(index_fields): 119 | if not isinstance(field, DEDField): 120 | continue 121 | 122 | if not field._path: 123 | field._path = [name] 124 | 125 | prep_func = getattr(self, 'prepare_%s_with_related' % name, None) 126 | if prep_func: 127 | fn = partial(prep_func, related_to_ignore=self._related_instance_to_ignore) 128 | else: 129 | prep_func = getattr(self, 'prepare_%s' % name, None) 130 | if prep_func: 131 | fn = prep_func 132 | else: 133 | fn = partial(field.get_value_from_instance, field_value_to_ignore=self._related_instance_to_ignore) 134 | 135 | fields.append((name, field, fn)) 136 | 137 | return fields 138 | 139 | def prepare(self, instance): 140 | """ 141 | Take a model instance, and turn it into a dict that can be serialized 142 | based on the fields defined on this DocType subclass 143 | """ 144 | data = { 145 | name: prep_func(instance) 146 | for name, field, prep_func in self._prepared_fields 147 | } 148 | return data 149 | 150 | @classmethod 151 | def get_model_field_class_to_field_class(cls): 152 | """ 153 | Returns dict of relationship from model field class to elasticsearch 154 | field class 155 | 156 | You may want to override this if you have model field class not included 157 | in model_field_class_to_field_class. 158 | """ 159 | return model_field_class_to_field_class 160 | 161 | @classmethod 162 | def to_field(cls, field_name, model_field): 163 | """ 164 | Returns the elasticsearch field instance appropriate for the model 165 | field class. This is a good place to hook into if you have more complex 166 | model field to ES field logic 167 | """ 168 | try: 169 | return cls.get_model_field_class_to_field_class()[ 170 | model_field.__class__](attr=field_name) 171 | except KeyError: 172 | raise ModelFieldNotMappedError( 173 | "Cannot convert model field {} " 174 | "to an Elasticsearch field!".format(field_name) 175 | ) 176 | 177 | def bulk(self, actions, **kwargs): 178 | response = bulk(client=self._get_connection(), actions=actions, **kwargs) 179 | # send post index signal 180 | post_index.send( 181 | sender=self.__class__, 182 | instance=self, 183 | actions=actions, 184 | response=response 185 | ) 186 | return response 187 | 188 | def parallel_bulk(self, actions, **kwargs): 189 | if self.django.queryset_pagination and 'chunk_size' not in kwargs: 190 | kwargs['chunk_size'] = self.django.queryset_pagination 191 | bulk_actions = parallel_bulk(client=self._get_connection(), actions=actions, **kwargs) 192 | # As the `parallel_bulk` is lazy, we need to get it into `deque` to run it instantly 193 | # See https://discuss.elastic.co/t/helpers-parallel-bulk-in-python-not-working/39498/2 194 | deque(bulk_actions, maxlen=0) 195 | # Fake return value to emulate bulk() since we don't have a result yet, 196 | # the result is currently not used upstream anyway. 197 | return (1, []) 198 | 199 | @classmethod 200 | def generate_id(cls, object_instance): 201 | """ 202 | The default behavior is to use the Django object's pk (id) as the 203 | elasticseach index id (_id). If needed, this method can be overloaded 204 | to change this default behavior. 205 | """ 206 | return object_instance.pk 207 | 208 | def _prepare_action(self, object_instance, action): 209 | return { 210 | '_op_type': action, 211 | '_index': self._index._name, 212 | '_id': self.generate_id(object_instance), 213 | '_source': ( 214 | self.prepare(object_instance) if action != 'delete' else None 215 | ), 216 | } 217 | 218 | def _get_actions(self, object_list, action): 219 | for object_instance in object_list: 220 | if action == 'delete' or self.should_index_object(object_instance): 221 | yield self._prepare_action(object_instance, action) 222 | 223 | def get_actions(self, object_list, action): 224 | """ 225 | Generate the elasticsearch payload. 226 | """ 227 | return self._get_actions(object_list, action) 228 | 229 | 230 | def _bulk(self, *args, **kwargs): 231 | """Helper for switching between normal and parallel bulk operation""" 232 | parallel = kwargs.pop('parallel', False) 233 | if parallel: 234 | return self.parallel_bulk(*args, **kwargs) 235 | else: 236 | return self.bulk(*args, **kwargs) 237 | 238 | def should_index_object(self, obj): 239 | """ 240 | Overwriting this method and returning a boolean value 241 | should determine whether the object should be indexed. 242 | """ 243 | return True 244 | 245 | def update(self, thing, refresh=None, action='index', parallel=False, **kwargs): 246 | """ 247 | Update each document in ES for a model, iterable of models or queryset 248 | """ 249 | if refresh is not None: 250 | kwargs['refresh'] = refresh 251 | elif self.django.auto_refresh: 252 | kwargs['refresh'] = self.django.auto_refresh 253 | 254 | if isinstance(thing, models.Model): 255 | object_list = [thing] 256 | else: 257 | object_list = thing 258 | 259 | return self._bulk( 260 | self._get_actions(object_list, action), 261 | parallel=parallel, 262 | **kwargs 263 | ) 264 | 265 | 266 | # Alias of DocType. Need to remove DocType in 7.x 267 | Document = DocType 268 | -------------------------------------------------------------------------------- /django_elasticsearch_dsl/exceptions.py: -------------------------------------------------------------------------------- 1 | class DjangoElasticsearchDslError(Exception): 2 | pass 3 | 4 | 5 | class VariableLookupError(DjangoElasticsearchDslError): 6 | pass 7 | 8 | 9 | class RedeclaredFieldError(DjangoElasticsearchDslError): 10 | pass 11 | 12 | 13 | class ModelFieldNotMappedError(DjangoElasticsearchDslError): 14 | pass 15 | -------------------------------------------------------------------------------- /django_elasticsearch_dsl/fields.py: -------------------------------------------------------------------------------- 1 | from types import MethodType 2 | 3 | import django 4 | from django.core.exceptions import ObjectDoesNotExist 5 | from django.db import models 6 | from django.db.models.fields.files import FieldFile 7 | 8 | if django.VERSION < (4, 0): 9 | from django.utils.encoding import force_text as force_str 10 | else: 11 | from django.utils.encoding import force_str 12 | from django.utils.functional import Promise 13 | from elasticsearch_dsl.field import ( 14 | Boolean, 15 | Byte, 16 | Completion, 17 | Date, 18 | Double, 19 | Field, 20 | Float, 21 | GeoPoint, 22 | GeoShape, 23 | Integer, 24 | Ip, 25 | Long, 26 | Nested, 27 | Object, 28 | ScaledFloat, 29 | Short, 30 | Keyword, 31 | Text, 32 | SearchAsYouType, 33 | ) 34 | 35 | from .exceptions import VariableLookupError 36 | 37 | 38 | class DEDField(Field): 39 | def __init__(self, attr=None, **kwargs): 40 | super(DEDField, self).__init__(**kwargs) 41 | self._path = attr.split('.') if attr else [] 42 | 43 | def __setattr__(self, key, value): 44 | if key == 'get_value_from_instance': 45 | self.__dict__[key] = value 46 | else: 47 | super(DEDField, self).__setattr__(key, value) 48 | 49 | def get_value_from_instance(self, instance, field_value_to_ignore=None): 50 | """ 51 | Given an model instance to index with ES, return the value that 52 | should be put into ES for this field. 53 | """ 54 | if not instance: 55 | return None 56 | 57 | for attr in self._path: 58 | try: 59 | instance = instance[attr] 60 | except ( 61 | TypeError, AttributeError, 62 | KeyError, ValueError, IndexError 63 | ): 64 | try: 65 | instance = getattr(instance, attr) 66 | except ObjectDoesNotExist: 67 | return None 68 | except (TypeError, AttributeError): 69 | try: 70 | instance = instance[int(attr)] 71 | except ( 72 | IndexError, ValueError, 73 | KeyError, TypeError 74 | ): 75 | if self._required: 76 | raise VariableLookupError( 77 | "Failed lookup for key [{}] in " 78 | "{!r}".format(attr, instance) 79 | ) 80 | return None 81 | 82 | if isinstance(instance, models.manager.Manager): 83 | instance = instance.all() 84 | elif callable(instance): 85 | instance = instance() 86 | elif instance is None: 87 | return None 88 | 89 | if instance == field_value_to_ignore: 90 | return None 91 | 92 | # convert lazy object like lazy translations to string 93 | if isinstance(instance, Promise): 94 | return force_str(instance) 95 | 96 | return instance 97 | 98 | 99 | class ObjectField(DEDField, Object): 100 | def _get_inner_field_data(self, obj, field_value_to_ignore=None): 101 | data = {} 102 | 103 | if hasattr(self, 'properties'): 104 | for name, field in self.properties.to_dict().items(): 105 | if not isinstance(field, DEDField): 106 | continue 107 | 108 | if field._path == []: 109 | field._path = [name] 110 | 111 | data[name] = field.get_value_from_instance( 112 | obj, field_value_to_ignore 113 | ) 114 | else: 115 | doc_instance = self._doc_class() 116 | for name, field in self._doc_class._doc_type.mapping.properties._params.get( 117 | 'properties', {}).items(): # noqa 118 | if not isinstance(field, DEDField): 119 | continue 120 | 121 | if field._path == []: 122 | field._path = [name] 123 | 124 | # This allows for retrieving data from an InnerDoc with prepare_field_name functions. 125 | prep_func = getattr(doc_instance, 'prepare_%s' % name, None) 126 | 127 | if prep_func: 128 | data[name] = prep_func(obj) 129 | else: 130 | data[name] = field.get_value_from_instance( 131 | obj, field_value_to_ignore 132 | ) 133 | 134 | # This allows for ObjectFields to be indexed from dicts with 135 | # dynamic keys (i.e. keys/fields not defined in 'properties') 136 | if not data and obj and isinstance(obj, dict): 137 | data = obj 138 | 139 | return data 140 | 141 | def get_value_from_instance(self, instance, field_value_to_ignore=None): 142 | objs = super(ObjectField, self).get_value_from_instance( 143 | instance, field_value_to_ignore 144 | ) 145 | 146 | if objs is None: 147 | return {} 148 | try: 149 | is_iterable = bool(iter(objs)) 150 | except TypeError: 151 | is_iterable = False 152 | 153 | # While dicts are iterable, they need to be excluded here so 154 | # their full data is indexed 155 | if is_iterable and not isinstance(objs, dict): 156 | return [ 157 | self._get_inner_field_data(obj, field_value_to_ignore) 158 | for obj in objs if obj != field_value_to_ignore 159 | ] 160 | 161 | return self._get_inner_field_data(objs, field_value_to_ignore) 162 | 163 | 164 | def ListField(field): 165 | """ 166 | This wraps a field so that when get_value_from_instance 167 | is called, the field's values are iterated over 168 | """ 169 | original_get_value_from_instance = field.get_value_from_instance 170 | 171 | def get_value_from_instance(self, instance, field_value_to_ignore=None): 172 | if not original_get_value_from_instance(instance): 173 | return [] 174 | return [value for value in original_get_value_from_instance(instance)] 175 | 176 | field.get_value_from_instance = MethodType(get_value_from_instance, field) 177 | return field 178 | 179 | 180 | class BooleanField(DEDField, Boolean): 181 | pass 182 | 183 | 184 | class ByteField(DEDField, Byte): 185 | pass 186 | 187 | 188 | class CompletionField(DEDField, Completion): 189 | pass 190 | 191 | 192 | class DateField(DEDField, Date): 193 | pass 194 | 195 | 196 | class DoubleField(DEDField, Double): 197 | pass 198 | 199 | 200 | class FloatField(DEDField, Float): 201 | pass 202 | 203 | 204 | class ScaledFloatField(DEDField, ScaledFloat): 205 | pass 206 | 207 | 208 | class GeoPointField(DEDField, GeoPoint): 209 | pass 210 | 211 | 212 | class GeoShapeField(DEDField, GeoShape): 213 | pass 214 | 215 | 216 | class IntegerField(DEDField, Integer): 217 | pass 218 | 219 | 220 | class IpField(DEDField, Ip): 221 | pass 222 | 223 | 224 | class LongField(DEDField, Long): 225 | pass 226 | 227 | 228 | class NestedField(Nested, ObjectField): 229 | pass 230 | 231 | 232 | class ShortField(DEDField, Short): 233 | pass 234 | 235 | 236 | class KeywordField(DEDField, Keyword): 237 | pass 238 | 239 | 240 | class TextField(DEDField, Text): 241 | pass 242 | 243 | 244 | class SearchAsYouTypeField(DEDField, SearchAsYouType): 245 | pass 246 | 247 | 248 | class FileFieldMixin(object): 249 | def get_value_from_instance(self, instance, field_value_to_ignore=None): 250 | _file = super(FileFieldMixin, self).get_value_from_instance( 251 | instance, field_value_to_ignore) 252 | 253 | if isinstance(_file, FieldFile): 254 | return _file.url if _file else '' 255 | return _file if _file else '' 256 | 257 | 258 | class FileField(FileFieldMixin, DEDField, Text): 259 | pass 260 | 261 | 262 | class TimeField(KeywordField): 263 | def get_value_from_instance(self, instance, field_value_to_ignore=None): 264 | time = super(TimeField, self).get_value_from_instance(instance, 265 | field_value_to_ignore) 266 | 267 | if time: 268 | return time.isoformat() 269 | -------------------------------------------------------------------------------- /django_elasticsearch_dsl/indices.py: -------------------------------------------------------------------------------- 1 | from copy import deepcopy 2 | 3 | from elasticsearch_dsl import Index as DSLIndex 4 | from six import python_2_unicode_compatible 5 | 6 | from .apps import DEDConfig 7 | from .registries import registry 8 | 9 | 10 | @python_2_unicode_compatible 11 | class Index(DSLIndex): 12 | def __init__(self, *args, **kwargs): 13 | super(Index, self).__init__(*args, **kwargs) 14 | default_index_settings = deepcopy(DEDConfig.default_index_settings()) 15 | self.settings(**default_index_settings) 16 | 17 | def document(self, document): 18 | """ 19 | Extend to register the document in the global document registry 20 | """ 21 | document = super(Index, self).document(document) 22 | registry.register_document(document) 23 | return document 24 | 25 | doc_type = document 26 | 27 | def __str__(self): 28 | return self._name 29 | -------------------------------------------------------------------------------- /django_elasticsearch_dsl/management/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/django-es/django-elasticsearch-dsl/e453afffe024725be27e457cdf67b1e8229bccfe/django_elasticsearch_dsl/management/__init__.py -------------------------------------------------------------------------------- /django_elasticsearch_dsl/management/commands/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/django-es/django-elasticsearch-dsl/e453afffe024725be27e457cdf67b1e8229bccfe/django_elasticsearch_dsl/management/commands/__init__.py -------------------------------------------------------------------------------- /django_elasticsearch_dsl/management/commands/search_index.py: -------------------------------------------------------------------------------- 1 | from __future__ import unicode_literals, absolute_import 2 | from datetime import datetime 3 | 4 | from elasticsearch_dsl import connections 5 | from django.conf import settings 6 | from django.core.management.base import BaseCommand, CommandError 7 | from six.moves import input 8 | from ...registries import registry 9 | 10 | 11 | class Command(BaseCommand): 12 | help = 'Manage elasticsearch index.' 13 | 14 | def __init__(self, *args, **kwargs): 15 | super(Command, self).__init__(*args, **kwargs) 16 | self.es_conn = connections.get_connection() 17 | 18 | def add_arguments(self, parser): 19 | parser.add_argument( 20 | '--models', 21 | metavar='app[.model]', 22 | type=str, 23 | nargs='*', 24 | help="Specify the model or app to be updated in elasticsearch" 25 | ) 26 | parser.add_argument( 27 | '--create', 28 | action='store_const', 29 | dest='action', 30 | const='create', 31 | help="Create the indices in elasticsearch" 32 | ) 33 | parser.add_argument( 34 | '--populate', 35 | action='store_const', 36 | dest='action', 37 | const='populate', 38 | help="Populate elasticsearch indices with models data" 39 | ) 40 | parser.add_argument( 41 | '--delete', 42 | action='store_const', 43 | dest='action', 44 | const='delete', 45 | help="Delete the indices in elasticsearch" 46 | ) 47 | parser.add_argument( 48 | '--rebuild', 49 | action='store_const', 50 | dest='action', 51 | const='rebuild', 52 | help="Delete the indices and then recreate and populate them" 53 | ) 54 | parser.add_argument( 55 | '-f', 56 | action='store_true', 57 | dest='force', 58 | help="Force operations without asking" 59 | ) 60 | parser.add_argument( 61 | '--parallel', 62 | action='store_true', 63 | dest='parallel', 64 | help='Run populate/rebuild update multi threaded' 65 | ) 66 | parser.add_argument( 67 | '--no-parallel', 68 | action='store_false', 69 | dest='parallel', 70 | help='Run populate/rebuild update single threaded' 71 | ) 72 | parser.add_argument( 73 | '--use-alias', 74 | action='store_true', 75 | dest='use_alias', 76 | help='Use alias with indices' 77 | ) 78 | parser.add_argument( 79 | '--use-alias-keep-index', 80 | action='store_true', 81 | dest='use_alias_keep_index', 82 | help=""" 83 | Do not delete replaced indices when used with '--rebuild' and 84 | '--use-alias' args 85 | """ 86 | ) 87 | parser.set_defaults(parallel=getattr(settings, 'ELASTICSEARCH_DSL_PARALLEL', False)) 88 | parser.add_argument( 89 | '--refresh', 90 | action='store_true', 91 | dest='refresh', 92 | default=None, 93 | help='Refresh indices after populate/rebuild' 94 | ) 95 | parser.add_argument( 96 | '--no-count', 97 | action='store_false', 98 | default=True, 99 | dest='count', 100 | help='Do not include a total count in the summary log line' 101 | ) 102 | 103 | def _get_models(self, args): 104 | """ 105 | Get Models from registry that match the --models args 106 | """ 107 | if args: 108 | models = [] 109 | for arg in args: 110 | arg = arg.lower() 111 | match_found = False 112 | 113 | for model in registry.get_models(): 114 | if model._meta.app_label == arg: 115 | models.append(model) 116 | match_found = True 117 | elif '{}.{}'.format( 118 | model._meta.app_label.lower(), 119 | model._meta.model_name.lower() 120 | ) == arg: 121 | models.append(model) 122 | match_found = True 123 | 124 | if not match_found: 125 | raise CommandError("No model or app named {}".format(arg)) 126 | else: 127 | models = registry.get_models() 128 | 129 | return set(models) 130 | 131 | def _create(self, models, aliases, options): 132 | for index in registry.get_indices(models): 133 | alias_exists = index._name in aliases 134 | if not alias_exists: 135 | self.stdout.write("Creating index '{}'".format(index._name)) 136 | index.create() 137 | elif options['action'] == 'create': 138 | self.stdout.write( 139 | "'{}' already exists as an alias. Run '--delete' with" 140 | " '--use-alias' arg to delete indices pointed at the " 141 | "alias to make index name available.".format(index._name) 142 | ) 143 | 144 | def _populate(self, models, options): 145 | parallel = options['parallel'] 146 | for doc in registry.get_documents(models): 147 | self.stdout.write("Indexing {} '{}' objects {}".format( 148 | doc().get_queryset().count() if options['count'] else "all", 149 | doc.django.model.__name__, 150 | "(parallel)" if parallel else "") 151 | ) 152 | qs = doc().get_indexing_queryset() 153 | doc().update(qs, parallel=parallel, refresh=options['refresh']) 154 | 155 | def _get_alias_indices(self, alias): 156 | alias_indices = self.es_conn.indices.get_alias(name=alias) 157 | return list(alias_indices.keys()) 158 | 159 | def _delete_alias_indices(self, alias): 160 | alias_indices = self._get_alias_indices(alias) 161 | alias_delete_actions = [ 162 | {"remove_index": {"index": index}} for index in alias_indices 163 | ] 164 | self.es_conn.indices.update_aliases(actions=alias_delete_actions) 165 | for index in alias_indices: 166 | self.stdout.write("Deleted index '{}'".format(index)) 167 | 168 | def _delete(self, models, aliases, options): 169 | index_names = [index._name for index in registry.get_indices(models)] 170 | 171 | if not options['force']: 172 | response = input( 173 | "Are you sure you want to delete " 174 | "the '{}' indices? [y/N]: ".format(", ".join(index_names))) 175 | if response.lower() != 'y': 176 | self.stdout.write('Aborted') 177 | return False 178 | 179 | if options['use_alias']: 180 | for index in index_names: 181 | alias_exists = index in aliases 182 | if alias_exists: 183 | self._delete_alias_indices(index) 184 | elif self.es_conn.indices.exists(index=index): 185 | self.stdout.write( 186 | "'{}' refers to an index, not an alias. Run " 187 | "'--delete' without '--use-alias' arg to delete " 188 | "index.".format(index) 189 | ) 190 | return False 191 | else: 192 | for index in registry.get_indices(models): 193 | alias_exists = index._name in aliases 194 | if not alias_exists: 195 | self.stdout.write("Deleting index '{}'".format(index._name)) 196 | index.delete(ignore=404) 197 | elif options['action'] == 'rebuild': 198 | self._delete_alias_indices(index._name) 199 | elif options['action'] == 'delete': 200 | self.stdout.write( 201 | "'{}' refers to an alias, not an index. Run " 202 | "'--delete' with '--use-alias' arg to delete indices " 203 | "pointed at the alias.".format(index._name) 204 | ) 205 | return False 206 | 207 | return True 208 | 209 | def _update_alias(self, alias, new_index, alias_exists, options): 210 | alias_actions = [{"add": {"alias": alias, "index": new_index}}] 211 | 212 | delete_existing_index = False 213 | if not alias_exists and self.es_conn.indices.exists(index=alias): 214 | # Elasticsearch will return an error if an index already 215 | # exists with the desired alias name. Therefore, we need to 216 | # delete that index. 217 | delete_existing_index = True 218 | alias_actions.append({"remove_index": {"index": alias}}) 219 | 220 | old_indices = [] 221 | alias_delete_actions = [] 222 | if alias_exists: 223 | # Elasticsearch will return an error if we search for 224 | # indices by alias but the alias doesn't exist. Therefore, 225 | # we want to be sure the alias exists. 226 | old_indices = self._get_alias_indices(alias) 227 | alias_actions.append( 228 | {"remove": {"alias": alias, "indices": old_indices}} 229 | ) 230 | alias_delete_actions = [ 231 | {"remove_index": {"index": index}} for index in old_indices 232 | ] 233 | 234 | self.es_conn.indices.update_aliases(actions=alias_actions) 235 | if delete_existing_index: 236 | self.stdout.write("Deleted index '{}'".format(alias)) 237 | 238 | self.stdout.write( 239 | "Added alias '{}' to index '{}'".format(alias, new_index) 240 | ) 241 | 242 | if old_indices: 243 | for index in old_indices: 244 | self.stdout.write( 245 | "Removed alias '{}' from index '{}'".format(alias, index) 246 | ) 247 | 248 | if alias_delete_actions and not options['use_alias_keep_index']: 249 | self.es_conn.indices.update_aliases( 250 | actions=alias_delete_actions 251 | ) 252 | for index in old_indices: 253 | self.stdout.write("Deleted index '{}'".format(index)) 254 | 255 | def _rebuild(self, models, aliases, options): 256 | if (not options['use_alias'] 257 | and not self._delete(models, aliases, options)): 258 | return 259 | 260 | if options['use_alias']: 261 | alias_index_pairs = [] 262 | index_suffix = "-" + datetime.now().strftime("%Y%m%d%H%M%S%f") 263 | for index in registry.get_indices(models): 264 | # The alias takes the original index name value. The 265 | # index name sent to Elasticsearch will be the alias 266 | # plus the suffix from above. In addition, the index 267 | # name needs to be limited to 255 characters, of which 268 | # 21 will always be taken by the suffix, leaving 234 269 | # characters from the original index name value. 270 | new_index = index._name[:234] + index_suffix 271 | alias_index_pairs.append( 272 | {'alias': index._name, 'index': new_index} 273 | ) 274 | index._name = new_index 275 | 276 | self._create(models, aliases, options) 277 | self._populate(models, options) 278 | 279 | if options['use_alias']: 280 | for alias_index_pair in alias_index_pairs: 281 | alias = alias_index_pair['alias'] 282 | alias_exists = alias in aliases 283 | self._update_alias( 284 | alias, alias_index_pair['index'], alias_exists, options 285 | ) 286 | 287 | def handle(self, *args, **options): 288 | if not options['action']: 289 | raise CommandError( 290 | "No action specified. Must be one of" 291 | " '--create','--populate', '--delete' or '--rebuild' ." 292 | ) 293 | 294 | action = options['action'] 295 | models = self._get_models(options['models']) 296 | 297 | # We need to know if and which aliases exist to mitigate naming 298 | # conflicts with indices, therefore this is needed regardless 299 | # of using the '--use-alias' arg. 300 | aliases = [] 301 | for index in self.es_conn.indices.get_alias().values(): 302 | aliases += index['aliases'].keys() 303 | 304 | if action == 'create': 305 | self._create(models, aliases, options) 306 | elif action == 'populate': 307 | self._populate(models, options) 308 | elif action == 'delete': 309 | self._delete(models, aliases, options) 310 | elif action == 'rebuild': 311 | self._rebuild(models, aliases, options) 312 | else: 313 | raise CommandError( 314 | "Invalid action. Must be one of" 315 | " '--create','--populate', '--delete' or '--rebuild' ." 316 | ) 317 | -------------------------------------------------------------------------------- /django_elasticsearch_dsl/models.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/django-es/django-elasticsearch-dsl/e453afffe024725be27e457cdf67b1e8229bccfe/django_elasticsearch_dsl/models.py -------------------------------------------------------------------------------- /django_elasticsearch_dsl/registries.py: -------------------------------------------------------------------------------- 1 | from collections import defaultdict 2 | from copy import deepcopy 3 | 4 | from itertools import chain 5 | 6 | from django.core.exceptions import ObjectDoesNotExist 7 | from django.core.exceptions import ImproperlyConfigured 8 | from elasticsearch_dsl import AttrDict 9 | from six import itervalues, iterkeys, iteritems 10 | 11 | from django_elasticsearch_dsl.exceptions import RedeclaredFieldError 12 | from .apps import DEDConfig 13 | 14 | 15 | class DocumentRegistry(object): 16 | """ 17 | Registry of models classes to a set of Document classes. 18 | """ 19 | def __init__(self): 20 | self._indices = defaultdict(set) 21 | self._models = defaultdict(set) 22 | self._related_models = defaultdict(set) 23 | 24 | def register(self, index, doc_class): 25 | """Register the model with the registry""" 26 | self._models[doc_class.django.model].add(doc_class) 27 | 28 | for related in doc_class.django.related_models: 29 | self._related_models[related].add(doc_class.django.model) 30 | 31 | for idx, docs in iteritems(self._indices): 32 | if index._name == idx._name: 33 | docs.add(doc_class) 34 | return 35 | 36 | self._indices[index].add(doc_class) 37 | 38 | def register_document(self, document): 39 | django_meta = getattr(document, 'Django') 40 | # Raise error if Django class can not be found 41 | if not django_meta: 42 | message = "You must declare the Django class inside {}".format(document.__name__) 43 | raise ImproperlyConfigured(message) 44 | 45 | # Keep all django related attribute in a django_attr AttrDict 46 | data = {'model': getattr(document.Django, 'model')} 47 | django_attr = AttrDict(data) 48 | 49 | if not django_attr.model: 50 | raise ImproperlyConfigured("You must specify the django model") 51 | 52 | # Add The model fields into elasticsearch mapping field 53 | model_field_names = getattr(document.Django, "fields", []) 54 | mapping_fields = document._doc_type.mapping.properties.properties.to_dict().keys() 55 | 56 | for field_name in model_field_names: 57 | if field_name in mapping_fields: 58 | raise RedeclaredFieldError( 59 | "You cannot redeclare the field named '{}' on {}" 60 | .format(field_name, document.__name__) 61 | ) 62 | 63 | django_field = django_attr.model._meta.get_field(field_name) 64 | 65 | field_instance = document.to_field(field_name, django_field) 66 | document._doc_type.mapping.field(field_name, field_instance) 67 | 68 | django_attr.ignore_signals = getattr(django_meta, "ignore_signals", False) 69 | django_attr.auto_refresh = getattr(django_meta, 70 | "auto_refresh", DEDConfig.auto_refresh_enabled()) 71 | django_attr.related_models = getattr(django_meta, "related_models", []) 72 | django_attr.queryset_pagination = getattr(django_meta, "queryset_pagination", None) 73 | 74 | # Add django attribute in the document class with all the django attribute 75 | setattr(document, 'django', django_attr) 76 | 77 | # Set the fields of the mappings 78 | fields = document._doc_type.mapping.properties.properties.to_dict() 79 | setattr(document, '_fields', fields) 80 | 81 | # Update settings of the document index 82 | default_index_settings = deepcopy(DEDConfig.default_index_settings()) 83 | document._index.settings(**default_index_settings) 84 | 85 | # Register the document and index class to our registry 86 | self.register(index=document._index, doc_class=document) 87 | 88 | return document 89 | 90 | def _get_related_doc(self, instance): 91 | for model in self._related_models.get(instance.__class__, []): 92 | for doc in self._models[model]: 93 | if instance.__class__ in doc.django.related_models: 94 | yield doc 95 | 96 | def update_related(self, instance, **kwargs): 97 | """ 98 | Update docs that have related_models. 99 | """ 100 | if not DEDConfig.autosync_enabled(): 101 | return 102 | 103 | for doc in self._get_related_doc(instance): 104 | doc_instance = doc() 105 | try: 106 | related = doc_instance.get_instances_from_related(instance) 107 | except ObjectDoesNotExist: 108 | related = None 109 | 110 | if related is not None: 111 | doc_instance.update(related, **kwargs) 112 | 113 | def delete_related(self, instance, **kwargs): 114 | """ 115 | Remove `instance` from related models. 116 | """ 117 | if not DEDConfig.autosync_enabled(): 118 | return 119 | 120 | for doc in self._get_related_doc(instance): 121 | doc_instance = doc(related_instance_to_ignore=instance) 122 | try: 123 | related = doc_instance.get_instances_from_related(instance) 124 | except ObjectDoesNotExist: 125 | related = None 126 | 127 | if related is not None: 128 | doc_instance.update(related, **kwargs) 129 | 130 | def update(self, instance, **kwargs): 131 | """ 132 | Update all the elasticsearch documents attached to this model (if their 133 | ignore_signals flag allows it) 134 | """ 135 | if not DEDConfig.autosync_enabled(): 136 | return 137 | 138 | if instance.__class__ in self._models: 139 | for doc in self._models[instance.__class__]: 140 | if not doc.django.ignore_signals: 141 | doc().update(instance, **kwargs) 142 | 143 | def delete(self, instance, **kwargs): 144 | """ 145 | Delete all the elasticsearch documents attached to this model (if their 146 | ignore_signals flag allows it) 147 | """ 148 | self.update(instance, action="delete", **kwargs) 149 | 150 | def get_documents(self, models=None): 151 | """ 152 | Get all documents in the registry or the documents for a list of models 153 | """ 154 | if models is not None: 155 | return set(chain.from_iterable(self._models[model] for model in models 156 | if model in self._models)) 157 | return set(chain.from_iterable(itervalues(self._indices))) 158 | 159 | def get_models(self): 160 | """ 161 | Get all models in the registry 162 | """ 163 | return set(iterkeys(self._models)) 164 | 165 | def get_indices(self, models=None): 166 | """ 167 | Get all indices in the registry or the indices for a list of models 168 | """ 169 | if models is not None: 170 | return set( 171 | indice for indice, docs in iteritems(self._indices) 172 | for doc in docs if doc.django.model in models 173 | ) 174 | 175 | return set(iterkeys(self._indices)) 176 | 177 | def __contains__(self, model): 178 | """ 179 | Checks that model is in registry 180 | """ 181 | return model in self._models or model in self._related_models 182 | 183 | 184 | registry = DocumentRegistry() 185 | -------------------------------------------------------------------------------- /django_elasticsearch_dsl/search.py: -------------------------------------------------------------------------------- 1 | from django.db.models import Case, When 2 | from django.db.models.fields import IntegerField 3 | 4 | from elasticsearch_dsl import Search as DSLSearch 5 | 6 | 7 | class Search(DSLSearch): 8 | def __init__(self, **kwargs): 9 | self._model = kwargs.pop('model', None) 10 | super(Search, self).__init__(**kwargs) 11 | 12 | def _clone(self): 13 | s = super(Search, self)._clone() 14 | s._model = self._model 15 | return s 16 | 17 | def filter_queryset(self, queryset, keep_search_order=True): 18 | """ 19 | Filter an existing django queryset using the elasticsearch result. 20 | It costs a query to the sql db. 21 | """ 22 | s = self 23 | if s._model is not queryset.model: 24 | raise TypeError( 25 | 'Unexpected queryset model ' 26 | '(should be: %s, got: %s)' % (s._model, queryset.model) 27 | ) 28 | 29 | # Do not query again if the es result is already cached 30 | if not hasattr(self, '_response'): 31 | # We only need the meta fields with the models ids 32 | s = self.source(excludes=['*']) 33 | s = s.execute() 34 | 35 | pks = [result.meta.id for result in s] 36 | queryset = queryset.filter(pk__in=pks) 37 | 38 | if keep_search_order: 39 | preserved_order = Case( 40 | *[When(pk=pk, then=pos) for pos, pk in enumerate(pks)], 41 | output_field=IntegerField() 42 | ) 43 | queryset = queryset.order_by(preserved_order) 44 | 45 | return queryset 46 | 47 | def _get_queryset(self): 48 | """ 49 | Return a django queryset that will be filtered by to_queryset method. 50 | """ 51 | return self._model._default_manager.all() 52 | 53 | def to_queryset(self, keep_order=True): 54 | """ 55 | Return a django queryset from the elasticsearch result. 56 | It costs a query to the sql db. 57 | """ 58 | qs = self._get_queryset() 59 | return self.filter_queryset(qs, keep_order) 60 | -------------------------------------------------------------------------------- /django_elasticsearch_dsl/signals.py: -------------------------------------------------------------------------------- 1 | # encoding: utf-8 2 | """ 3 | A convenient way to attach django-elasticsearch-dsl to Django's signals and 4 | cause things to index. 5 | """ 6 | 7 | from __future__ import absolute_import 8 | 9 | from django.db import models 10 | from django.apps import apps 11 | from django.dispatch import Signal 12 | from .registries import registry 13 | from django.core.exceptions import ObjectDoesNotExist 14 | from importlib import import_module 15 | # Sent after document indexing is completed 16 | post_index = Signal() 17 | 18 | class BaseSignalProcessor(object): 19 | """Base signal processor. 20 | 21 | By default, does nothing with signals but provides underlying 22 | functionality. 23 | """ 24 | 25 | def __init__(self, connections): 26 | self.connections = connections 27 | self.setup() 28 | 29 | def setup(self): 30 | """Set up. 31 | 32 | A hook for setting up anything necessary for 33 | ``handle_save/handle_delete`` to be executed. 34 | 35 | Default behavior is to do nothing (``pass``). 36 | """ 37 | # Do nothing. 38 | 39 | def teardown(self): 40 | """Tear-down. 41 | 42 | A hook for tearing down anything necessary for 43 | ``handle_save/handle_delete`` to no longer be executed. 44 | 45 | Default behavior is to do nothing (``pass``). 46 | """ 47 | # Do nothing. 48 | 49 | def handle_m2m_changed(self, sender, instance, action, **kwargs): 50 | if action in ('post_add', 'post_remove', 'post_clear'): 51 | self.handle_save(sender, instance) 52 | elif action in ('pre_remove', 'pre_clear'): 53 | self.handle_pre_delete(sender, instance) 54 | 55 | def handle_save(self, sender, instance, **kwargs): 56 | """Handle save. 57 | 58 | Given an individual model instance, update the object in the index. 59 | Update the related objects either. 60 | """ 61 | registry.update(instance) 62 | registry.update_related(instance) 63 | 64 | def handle_pre_delete(self, sender, instance, **kwargs): 65 | """Handle removing of instance object from related models instance. 66 | We need to do this before the real delete otherwise the relation 67 | doesn't exists anymore and we can't get the related models instance. 68 | """ 69 | registry.delete_related(instance) 70 | 71 | def handle_delete(self, sender, instance, **kwargs): 72 | """Handle delete. 73 | 74 | Given an individual model instance, delete the object from index. 75 | """ 76 | registry.delete(instance, raise_on_error=False) 77 | 78 | 79 | class RealTimeSignalProcessor(BaseSignalProcessor): 80 | """Real-time signal processor. 81 | 82 | Allows for observing when saves/deletes fire and automatically updates the 83 | search engine appropriately. 84 | """ 85 | 86 | def setup(self): 87 | # Listen to all model saves. 88 | models.signals.post_save.connect(self.handle_save) 89 | models.signals.post_delete.connect(self.handle_delete) 90 | 91 | # Use to manage related objects update 92 | models.signals.m2m_changed.connect(self.handle_m2m_changed) 93 | models.signals.pre_delete.connect(self.handle_pre_delete) 94 | 95 | def teardown(self): 96 | # Listen to all model saves. 97 | models.signals.post_save.disconnect(self.handle_save) 98 | models.signals.post_delete.disconnect(self.handle_delete) 99 | models.signals.m2m_changed.disconnect(self.handle_m2m_changed) 100 | models.signals.pre_delete.disconnect(self.handle_pre_delete) 101 | 102 | try: 103 | from celery import shared_task 104 | except ImportError: 105 | pass 106 | else: 107 | class CelerySignalProcessor(RealTimeSignalProcessor): 108 | """Celery signal processor. 109 | 110 | Allows automatic updates on the index as delayed background tasks using 111 | Celery. 112 | 113 | NB: We cannot process deletes as background tasks. 114 | By the time the Celery worker would pick up the delete job, the 115 | model instance would already deleted. We can get around this by 116 | setting Celery to use `pickle` and sending the object to the worker, 117 | but using `pickle` opens the application up to security concerns. 118 | """ 119 | 120 | def handle_save(self, sender, instance, **kwargs): 121 | """Handle save with a Celery task. 122 | 123 | Given an individual model instance, update the object in the index. 124 | Update the related objects either. 125 | """ 126 | pk = instance.pk 127 | app_label = instance._meta.app_label 128 | model_name = instance.__class__.__name__ 129 | 130 | self.registry_update_task.delay(pk, app_label, model_name) 131 | self.registry_update_related_task.delay(pk, app_label, model_name) 132 | 133 | def handle_pre_delete(self, sender, instance, **kwargs): 134 | """Handle removing of instance object from related models instance. 135 | We need to do this before the real delete otherwise the relation 136 | doesn't exists anymore and we can't get the related models instance. 137 | """ 138 | self.prepare_registry_delete_related_task(instance) 139 | 140 | def handle_delete(self, sender, instance, **kwargs): 141 | """Handle delete. 142 | 143 | Given an individual model instance, delete the object from index. 144 | """ 145 | self.prepare_registry_delete_task(instance) 146 | 147 | def prepare_registry_delete_related_task(self, instance): 148 | """ 149 | Select its related instance before this instance was deleted. 150 | And pass that to celery. 151 | """ 152 | action = 'index' 153 | for doc in registry._get_related_doc(instance): 154 | doc_instance = doc(related_instance_to_ignore=instance) 155 | try: 156 | related = doc_instance.get_instances_from_related(instance) 157 | except ObjectDoesNotExist: 158 | related = None 159 | if related is not None: 160 | doc_instance.update(related) 161 | if isinstance(related, models.Model): 162 | object_list = [related] 163 | else: 164 | object_list = related 165 | bulk_data = list(doc_instance._get_actions(object_list, action)), 166 | self.registry_delete_task.delay(doc_instance.__class__.__name__, bulk_data) 167 | 168 | @shared_task() 169 | def registry_delete_task(doc_label, data): 170 | """ 171 | Handle the bulk delete data on the registry as a Celery task. 172 | The different implementations used are due to the difference between delete and update operations. 173 | The update operation can re-read the updated data from the database to ensure eventual consistency, 174 | but the delete needs to be processed before the database record is deleted to obtain the associated data. 175 | """ 176 | doc_instance = import_module(doc_label) 177 | parallel = True 178 | doc_instance._bulk(bulk_data, parallel=parallel) 179 | 180 | def prepare_registry_delete_task(self, instance): 181 | """ 182 | Get the prepare did before database record deleted. 183 | """ 184 | action = 'delete' 185 | for doc in registry._get_related_doc(instance): 186 | doc_instance = doc(related_instance_to_ignore=instance) 187 | try: 188 | related = doc_instance.get_instances_from_related(instance) 189 | except ObjectDoesNotExist: 190 | related = None 191 | if related is not None: 192 | doc_instance.update(related) 193 | if isinstance(related, models.Model): 194 | object_list = [related] 195 | else: 196 | object_list = related 197 | bulk_data = list(doc_instance.get_actions(object_list, action)), 198 | self.registry_delete_task.delay(doc_instance.__class__.__name__, bulk_data) 199 | 200 | @shared_task() 201 | def registry_update_task(pk, app_label, model_name): 202 | """Handle the update on the registry as a Celery task.""" 203 | try: 204 | model = apps.get_model(app_label, model_name) 205 | except LookupError: 206 | pass 207 | else: 208 | registry.update( 209 | model.objects.get(pk=pk) 210 | ) 211 | 212 | @shared_task() 213 | def registry_update_related_task(pk, app_label, model_name): 214 | """Handle the related update on the registry as a Celery task.""" 215 | try: 216 | model = apps.get_model(app_label, model_name) 217 | except LookupError: 218 | pass 219 | else: 220 | registry.update_related( 221 | model.objects.get(pk=pk) 222 | ) 223 | -------------------------------------------------------------------------------- /django_elasticsearch_dsl/test/__init__.py: -------------------------------------------------------------------------------- 1 | from .testcases import ESTestCase, is_es_online 2 | 3 | 4 | __all__ = ['ESTestCase', 'is_es_online'] 5 | -------------------------------------------------------------------------------- /django_elasticsearch_dsl/test/testcases.py: -------------------------------------------------------------------------------- 1 | import re 2 | 3 | from django.test.utils import captured_stderr 4 | from elasticsearch_dsl.connections import connections 5 | 6 | from ..registries import registry 7 | 8 | 9 | def is_es_online(connection_alias='default'): 10 | with captured_stderr(): 11 | es = connections.get_connection(connection_alias) 12 | return es.ping() 13 | 14 | 15 | class ESTestCase(object): 16 | _index_suffixe = '_ded_test' 17 | 18 | def setUp(self): 19 | for doc in registry.get_documents(): 20 | doc._index._name += self._index_suffixe 21 | 22 | for index in registry.get_indices(): 23 | index._name += self._index_suffixe 24 | index.delete(ignore=[404, 400]) 25 | index.create() 26 | 27 | super(ESTestCase, self).setUp() 28 | 29 | def tearDown(self): 30 | pattern = re.compile(self._index_suffixe + '$') 31 | 32 | for index in registry.get_indices(): 33 | index.delete(ignore=[404, 400]) 34 | index._name = pattern.sub('', index._name) 35 | 36 | for doc in registry.get_documents(): 37 | doc._index._name = pattern.sub('', doc._index._name) 38 | 39 | super(ESTestCase, self).tearDown() 40 | -------------------------------------------------------------------------------- /docs/Makefile: -------------------------------------------------------------------------------- 1 | # Makefile for Sphinx documentation 2 | # 3 | 4 | # You can set these variables from the command line. 5 | SPHINXOPTS = 6 | SPHINXBUILD = sphinx-build 7 | PAPER = 8 | BUILDDIR = build 9 | 10 | # User-friendly check for sphinx-build 11 | ifeq ($(shell which $(SPHINXBUILD) >/dev/null 2>&1; echo $$?), 1) 12 | $(error The '$(SPHINXBUILD)' command was not found. Make sure you have Sphinx installed, then set the SPHINXBUILD environment variable to point to the full path of the '$(SPHINXBUILD)' executable. Alternatively you can add the directory with the executable to your PATH. If you don\'t have Sphinx installed, grab it from http://sphinx-doc.org/) 13 | endif 14 | 15 | # Internal variables. 16 | PAPEROPT_a4 = -D latex_paper_size=a4 17 | PAPEROPT_letter = -D latex_paper_size=letter 18 | ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) source 19 | # the i18n builder cannot share the environment and doctrees with the others 20 | I18NSPHINXOPTS = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) source 21 | 22 | .PHONY: help 23 | help: 24 | @echo "Please use \`make ' where is one of" 25 | @echo " html to make standalone HTML files" 26 | @echo " dirhtml to make HTML files named index.html in directories" 27 | @echo " singlehtml to make a single large HTML file" 28 | @echo " pickle to make pickle files" 29 | @echo " json to make JSON files" 30 | @echo " htmlhelp to make HTML files and a HTML help project" 31 | @echo " qthelp to make HTML files and a qthelp project" 32 | @echo " applehelp to make an Apple Help Book" 33 | @echo " devhelp to make HTML files and a Devhelp project" 34 | @echo " epub to make an epub" 35 | @echo " epub3 to make an epub3" 36 | @echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter" 37 | @echo " latexpdf to make LaTeX files and run them through pdflatex" 38 | @echo " latexpdfja to make LaTeX files and run them through platex/dvipdfmx" 39 | @echo " text to make text files" 40 | @echo " man to make manual pages" 41 | @echo " texinfo to make Texinfo files" 42 | @echo " info to make Texinfo files and run them through makeinfo" 43 | @echo " gettext to make PO message catalogs" 44 | @echo " changes to make an overview of all changed/added/deprecated items" 45 | @echo " xml to make Docutils-native XML files" 46 | @echo " pseudoxml to make pseudoxml-XML files for display purposes" 47 | @echo " linkcheck to check all external links for integrity" 48 | @echo " doctest to run all doctests embedded in the documentation (if enabled)" 49 | @echo " coverage to run coverage check of the documentation (if enabled)" 50 | @echo " dummy to check syntax errors of document sources" 51 | 52 | .PHONY: clean 53 | clean: 54 | rm -rf $(BUILDDIR)/* 55 | 56 | .PHONY: html 57 | html: 58 | $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html 59 | @echo 60 | @echo "Build finished. The HTML pages are in $(BUILDDIR)/html." 61 | 62 | .PHONY: dirhtml 63 | dirhtml: 64 | $(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml 65 | @echo 66 | @echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml." 67 | 68 | .PHONY: singlehtml 69 | singlehtml: 70 | $(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) $(BUILDDIR)/singlehtml 71 | @echo 72 | @echo "Build finished. The HTML page is in $(BUILDDIR)/singlehtml." 73 | 74 | .PHONY: pickle 75 | pickle: 76 | $(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle 77 | @echo 78 | @echo "Build finished; now you can process the pickle files." 79 | 80 | .PHONY: json 81 | json: 82 | $(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json 83 | @echo 84 | @echo "Build finished; now you can process the JSON files." 85 | 86 | .PHONY: htmlhelp 87 | htmlhelp: 88 | $(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp 89 | @echo 90 | @echo "Build finished; now you can run HTML Help Workshop with the" \ 91 | ".hhp project file in $(BUILDDIR)/htmlhelp." 92 | 93 | .PHONY: qthelp 94 | qthelp: 95 | $(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp 96 | @echo 97 | @echo "Build finished; now you can run "qcollectiongenerator" with the" \ 98 | ".qhcp project file in $(BUILDDIR)/qthelp, like this:" 99 | @echo "# qcollectiongenerator $(BUILDDIR)/qthelp/luqum.qhcp" 100 | @echo "To view the help file:" 101 | @echo "# assistant -collectionFile $(BUILDDIR)/qthelp/luqum.qhc" 102 | 103 | .PHONY: applehelp 104 | applehelp: 105 | $(SPHINXBUILD) -b applehelp $(ALLSPHINXOPTS) $(BUILDDIR)/applehelp 106 | @echo 107 | @echo "Build finished. The help book is in $(BUILDDIR)/applehelp." 108 | @echo "N.B. You won't be able to view it unless you put it in" \ 109 | "~/Library/Documentation/Help or install it in your application" \ 110 | "bundle." 111 | 112 | .PHONY: devhelp 113 | devhelp: 114 | $(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILDDIR)/devhelp 115 | @echo 116 | @echo "Build finished." 117 | @echo "To view the help file:" 118 | @echo "# mkdir -p $$HOME/.local/share/devhelp/luqum" 119 | @echo "# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/luqum" 120 | @echo "# devhelp" 121 | 122 | .PHONY: epub 123 | epub: 124 | $(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILDDIR)/epub 125 | @echo 126 | @echo "Build finished. The epub file is in $(BUILDDIR)/epub." 127 | 128 | .PHONY: epub3 129 | epub3: 130 | $(SPHINXBUILD) -b epub3 $(ALLSPHINXOPTS) $(BUILDDIR)/epub3 131 | @echo 132 | @echo "Build finished. The epub3 file is in $(BUILDDIR)/epub3." 133 | 134 | .PHONY: latex 135 | latex: 136 | $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex 137 | @echo 138 | @echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex." 139 | @echo "Run \`make' in that directory to run these through (pdf)latex" \ 140 | "(use \`make latexpdf' here to do that automatically)." 141 | 142 | .PHONY: latexpdf 143 | latexpdf: 144 | $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex 145 | @echo "Running LaTeX files through pdflatex..." 146 | $(MAKE) -C $(BUILDDIR)/latex all-pdf 147 | @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." 148 | 149 | .PHONY: latexpdfja 150 | latexpdfja: 151 | $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex 152 | @echo "Running LaTeX files through platex and dvipdfmx..." 153 | $(MAKE) -C $(BUILDDIR)/latex all-pdf-ja 154 | @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." 155 | 156 | .PHONY: text 157 | text: 158 | $(SPHINXBUILD) -b text $(ALLSPHINXOPTS) $(BUILDDIR)/text 159 | @echo 160 | @echo "Build finished. The text files are in $(BUILDDIR)/text." 161 | 162 | .PHONY: man 163 | man: 164 | $(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man 165 | @echo 166 | @echo "Build finished. The manual pages are in $(BUILDDIR)/man." 167 | 168 | .PHONY: texinfo 169 | texinfo: 170 | $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo 171 | @echo 172 | @echo "Build finished. The Texinfo files are in $(BUILDDIR)/texinfo." 173 | @echo "Run \`make' in that directory to run these through makeinfo" \ 174 | "(use \`make info' here to do that automatically)." 175 | 176 | .PHONY: info 177 | info: 178 | $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo 179 | @echo "Running Texinfo files through makeinfo..." 180 | make -C $(BUILDDIR)/texinfo info 181 | @echo "makeinfo finished; the Info files are in $(BUILDDIR)/texinfo." 182 | 183 | .PHONY: gettext 184 | gettext: 185 | $(SPHINXBUILD) -b gettext $(I18NSPHINXOPTS) $(BUILDDIR)/locale 186 | @echo 187 | @echo "Build finished. The message catalogs are in $(BUILDDIR)/locale." 188 | 189 | .PHONY: changes 190 | changes: 191 | $(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes 192 | @echo 193 | @echo "The overview file is in $(BUILDDIR)/changes." 194 | 195 | .PHONY: linkcheck 196 | linkcheck: 197 | $(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck 198 | @echo 199 | @echo "Link check complete; look for any errors in the above output " \ 200 | "or in $(BUILDDIR)/linkcheck/output.txt." 201 | 202 | .PHONY: doctest 203 | doctest: 204 | $(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest 205 | @echo "Testing of doctests in the sources finished, look at the " \ 206 | "results in $(BUILDDIR)/doctest/output.txt." 207 | 208 | .PHONY: coverage 209 | coverage: 210 | $(SPHINXBUILD) -b coverage $(ALLSPHINXOPTS) $(BUILDDIR)/coverage 211 | @echo "Testing of coverage in the sources finished, look at the " \ 212 | "results in $(BUILDDIR)/coverage/python.txt." 213 | 214 | .PHONY: xml 215 | xml: 216 | $(SPHINXBUILD) -b xml $(ALLSPHINXOPTS) $(BUILDDIR)/xml 217 | @echo 218 | @echo "Build finished. The XML files are in $(BUILDDIR)/xml." 219 | 220 | .PHONY: pseudoxml 221 | pseudoxml: 222 | $(SPHINXBUILD) -b pseudoxml $(ALLSPHINXOPTS) $(BUILDDIR)/pseudoxml 223 | @echo 224 | @echo "Build finished. The pseudo-XML files are in $(BUILDDIR)/pseudoxml." 225 | 226 | .PHONY: dummy 227 | dummy: 228 | $(SPHINXBUILD) -b dummy $(ALLSPHINXOPTS) $(BUILDDIR)/dummy 229 | @echo 230 | @echo "Build finished. Dummy builder generates no files." 231 | -------------------------------------------------------------------------------- /docs/_static/.empty: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/django-es/django-elasticsearch-dsl/e453afffe024725be27e457cdf67b1e8229bccfe/docs/_static/.empty -------------------------------------------------------------------------------- /docs/make.bat: -------------------------------------------------------------------------------- 1 | @ECHO OFF 2 | 3 | REM Command file for Sphinx documentation 4 | 5 | if "%SPHINXBUILD%" == "" ( 6 | set SPHINXBUILD=sphinx-build 7 | ) 8 | set BUILDDIR=build 9 | set ALLSPHINXOPTS=-d %BUILDDIR%/doctrees %SPHINXOPTS% source 10 | set I18NSPHINXOPTS=%SPHINXOPTS% source 11 | if NOT "%PAPER%" == "" ( 12 | set ALLSPHINXOPTS=-D latex_paper_size=%PAPER% %ALLSPHINXOPTS% 13 | set I18NSPHINXOPTS=-D latex_paper_size=%PAPER% %I18NSPHINXOPTS% 14 | ) 15 | 16 | if "%1" == "" goto help 17 | 18 | if "%1" == "help" ( 19 | :help 20 | echo.Please use `make ^` where ^ is one of 21 | echo. html to make standalone HTML files 22 | echo. dirhtml to make HTML files named index.html in directories 23 | echo. singlehtml to make a single large HTML file 24 | echo. pickle to make pickle files 25 | echo. json to make JSON files 26 | echo. htmlhelp to make HTML files and a HTML help project 27 | echo. qthelp to make HTML files and a qthelp project 28 | echo. devhelp to make HTML files and a Devhelp project 29 | echo. epub to make an epub 30 | echo. epub3 to make an epub3 31 | echo. latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter 32 | echo. text to make text files 33 | echo. man to make manual pages 34 | echo. texinfo to make Texinfo files 35 | echo. gettext to make PO message catalogs 36 | echo. changes to make an overview over all changed/added/deprecated items 37 | echo. xml to make Docutils-native XML files 38 | echo. pseudoxml to make pseudoxml-XML files for display purposes 39 | echo. linkcheck to check all external links for integrity 40 | echo. doctest to run all doctests embedded in the documentation if enabled 41 | echo. coverage to run coverage check of the documentation if enabled 42 | echo. dummy to check syntax errors of document sources 43 | goto end 44 | ) 45 | 46 | if "%1" == "clean" ( 47 | for /d %%i in (%BUILDDIR%\*) do rmdir /q /s %%i 48 | del /q /s %BUILDDIR%\* 49 | goto end 50 | ) 51 | 52 | 53 | REM Check if sphinx-build is available and fallback to Python version if any 54 | %SPHINXBUILD% 1>NUL 2>NUL 55 | if errorlevel 9009 goto sphinx_python 56 | goto sphinx_ok 57 | 58 | :sphinx_python 59 | 60 | set SPHINXBUILD=python -m sphinx.__init__ 61 | %SPHINXBUILD% 2> nul 62 | if errorlevel 9009 ( 63 | echo. 64 | echo.The 'sphinx-build' command was not found. Make sure you have Sphinx 65 | echo.installed, then set the SPHINXBUILD environment variable to point 66 | echo.to the full path of the 'sphinx-build' executable. Alternatively you 67 | echo.may add the Sphinx directory to PATH. 68 | echo. 69 | echo.If you don't have Sphinx installed, grab it from 70 | echo.http://sphinx-doc.org/ 71 | exit /b 1 72 | ) 73 | 74 | :sphinx_ok 75 | 76 | 77 | if "%1" == "html" ( 78 | %SPHINXBUILD% -b html %ALLSPHINXOPTS% %BUILDDIR%/html 79 | if errorlevel 1 exit /b 1 80 | echo. 81 | echo.Build finished. The HTML pages are in %BUILDDIR%/html. 82 | goto end 83 | ) 84 | 85 | if "%1" == "dirhtml" ( 86 | %SPHINXBUILD% -b dirhtml %ALLSPHINXOPTS% %BUILDDIR%/dirhtml 87 | if errorlevel 1 exit /b 1 88 | echo. 89 | echo.Build finished. The HTML pages are in %BUILDDIR%/dirhtml. 90 | goto end 91 | ) 92 | 93 | if "%1" == "singlehtml" ( 94 | %SPHINXBUILD% -b singlehtml %ALLSPHINXOPTS% %BUILDDIR%/singlehtml 95 | if errorlevel 1 exit /b 1 96 | echo. 97 | echo.Build finished. The HTML pages are in %BUILDDIR%/singlehtml. 98 | goto end 99 | ) 100 | 101 | if "%1" == "pickle" ( 102 | %SPHINXBUILD% -b pickle %ALLSPHINXOPTS% %BUILDDIR%/pickle 103 | if errorlevel 1 exit /b 1 104 | echo. 105 | echo.Build finished; now you can process the pickle files. 106 | goto end 107 | ) 108 | 109 | if "%1" == "json" ( 110 | %SPHINXBUILD% -b json %ALLSPHINXOPTS% %BUILDDIR%/json 111 | if errorlevel 1 exit /b 1 112 | echo. 113 | echo.Build finished; now you can process the JSON files. 114 | goto end 115 | ) 116 | 117 | if "%1" == "htmlhelp" ( 118 | %SPHINXBUILD% -b htmlhelp %ALLSPHINXOPTS% %BUILDDIR%/htmlhelp 119 | if errorlevel 1 exit /b 1 120 | echo. 121 | echo.Build finished; now you can run HTML Help Workshop with the ^ 122 | .hhp project file in %BUILDDIR%/htmlhelp. 123 | goto end 124 | ) 125 | 126 | if "%1" == "qthelp" ( 127 | %SPHINXBUILD% -b qthelp %ALLSPHINXOPTS% %BUILDDIR%/qthelp 128 | if errorlevel 1 exit /b 1 129 | echo. 130 | echo.Build finished; now you can run "qcollectiongenerator" with the ^ 131 | .qhcp project file in %BUILDDIR%/qthelp, like this: 132 | echo.^> qcollectiongenerator %BUILDDIR%\qthelp\luqum.qhcp 133 | echo.To view the help file: 134 | echo.^> assistant -collectionFile %BUILDDIR%\qthelp\luqum.ghc 135 | goto end 136 | ) 137 | 138 | if "%1" == "devhelp" ( 139 | %SPHINXBUILD% -b devhelp %ALLSPHINXOPTS% %BUILDDIR%/devhelp 140 | if errorlevel 1 exit /b 1 141 | echo. 142 | echo.Build finished. 143 | goto end 144 | ) 145 | 146 | if "%1" == "epub" ( 147 | %SPHINXBUILD% -b epub %ALLSPHINXOPTS% %BUILDDIR%/epub 148 | if errorlevel 1 exit /b 1 149 | echo. 150 | echo.Build finished. The epub file is in %BUILDDIR%/epub. 151 | goto end 152 | ) 153 | 154 | if "%1" == "epub3" ( 155 | %SPHINXBUILD% -b epub3 %ALLSPHINXOPTS% %BUILDDIR%/epub3 156 | if errorlevel 1 exit /b 1 157 | echo. 158 | echo.Build finished. The epub3 file is in %BUILDDIR%/epub3. 159 | goto end 160 | ) 161 | 162 | if "%1" == "latex" ( 163 | %SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex 164 | if errorlevel 1 exit /b 1 165 | echo. 166 | echo.Build finished; the LaTeX files are in %BUILDDIR%/latex. 167 | goto end 168 | ) 169 | 170 | if "%1" == "latexpdf" ( 171 | %SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex 172 | cd %BUILDDIR%/latex 173 | make all-pdf 174 | cd %~dp0 175 | echo. 176 | echo.Build finished; the PDF files are in %BUILDDIR%/latex. 177 | goto end 178 | ) 179 | 180 | if "%1" == "latexpdfja" ( 181 | %SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex 182 | cd %BUILDDIR%/latex 183 | make all-pdf-ja 184 | cd %~dp0 185 | echo. 186 | echo.Build finished; the PDF files are in %BUILDDIR%/latex. 187 | goto end 188 | ) 189 | 190 | if "%1" == "text" ( 191 | %SPHINXBUILD% -b text %ALLSPHINXOPTS% %BUILDDIR%/text 192 | if errorlevel 1 exit /b 1 193 | echo. 194 | echo.Build finished. The text files are in %BUILDDIR%/text. 195 | goto end 196 | ) 197 | 198 | if "%1" == "man" ( 199 | %SPHINXBUILD% -b man %ALLSPHINXOPTS% %BUILDDIR%/man 200 | if errorlevel 1 exit /b 1 201 | echo. 202 | echo.Build finished. The manual pages are in %BUILDDIR%/man. 203 | goto end 204 | ) 205 | 206 | if "%1" == "texinfo" ( 207 | %SPHINXBUILD% -b texinfo %ALLSPHINXOPTS% %BUILDDIR%/texinfo 208 | if errorlevel 1 exit /b 1 209 | echo. 210 | echo.Build finished. The Texinfo files are in %BUILDDIR%/texinfo. 211 | goto end 212 | ) 213 | 214 | if "%1" == "gettext" ( 215 | %SPHINXBUILD% -b gettext %I18NSPHINXOPTS% %BUILDDIR%/locale 216 | if errorlevel 1 exit /b 1 217 | echo. 218 | echo.Build finished. The message catalogs are in %BUILDDIR%/locale. 219 | goto end 220 | ) 221 | 222 | if "%1" == "changes" ( 223 | %SPHINXBUILD% -b changes %ALLSPHINXOPTS% %BUILDDIR%/changes 224 | if errorlevel 1 exit /b 1 225 | echo. 226 | echo.The overview file is in %BUILDDIR%/changes. 227 | goto end 228 | ) 229 | 230 | if "%1" == "linkcheck" ( 231 | %SPHINXBUILD% -b linkcheck %ALLSPHINXOPTS% %BUILDDIR%/linkcheck 232 | if errorlevel 1 exit /b 1 233 | echo. 234 | echo.Link check complete; look for any errors in the above output ^ 235 | or in %BUILDDIR%/linkcheck/output.txt. 236 | goto end 237 | ) 238 | 239 | if "%1" == "doctest" ( 240 | %SPHINXBUILD% -b doctest %ALLSPHINXOPTS% %BUILDDIR%/doctest 241 | if errorlevel 1 exit /b 1 242 | echo. 243 | echo.Testing of doctests in the sources finished, look at the ^ 244 | results in %BUILDDIR%/doctest/output.txt. 245 | goto end 246 | ) 247 | 248 | if "%1" == "coverage" ( 249 | %SPHINXBUILD% -b coverage %ALLSPHINXOPTS% %BUILDDIR%/coverage 250 | if errorlevel 1 exit /b 1 251 | echo. 252 | echo.Testing of coverage in the sources finished, look at the ^ 253 | results in %BUILDDIR%/coverage/python.txt. 254 | goto end 255 | ) 256 | 257 | if "%1" == "xml" ( 258 | %SPHINXBUILD% -b xml %ALLSPHINXOPTS% %BUILDDIR%/xml 259 | if errorlevel 1 exit /b 1 260 | echo. 261 | echo.Build finished. The XML files are in %BUILDDIR%/xml. 262 | goto end 263 | ) 264 | 265 | if "%1" == "pseudoxml" ( 266 | %SPHINXBUILD% -b pseudoxml %ALLSPHINXOPTS% %BUILDDIR%/pseudoxml 267 | if errorlevel 1 exit /b 1 268 | echo. 269 | echo.Build finished. The pseudo-XML files are in %BUILDDIR%/pseudoxml. 270 | goto end 271 | ) 272 | 273 | if "%1" == "dummy" ( 274 | %SPHINXBUILD% -b dummy %ALLSPHINXOPTS% %BUILDDIR%/dummy 275 | if errorlevel 1 exit /b 1 276 | echo. 277 | echo.Build finished. Dummy builder generates no files. 278 | goto end 279 | ) 280 | 281 | :end 282 | -------------------------------------------------------------------------------- /docs/source/about.rst: -------------------------------------------------------------------------------- 1 | ../../README.rst -------------------------------------------------------------------------------- /docs/source/conf.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | # 4 | # This file is execfile()d with the current directory set to its 5 | # containing dir. 6 | # 7 | # Note that not all possible configuration values are present in this 8 | # autogenerated file. 9 | # 10 | # All configuration values have a default; values that are commented out 11 | # serve to show the default. 12 | 13 | import sys 14 | import os 15 | 16 | # If extensions (or modules to document with autodoc) are in another directory, 17 | # add these directories to sys.path here. If the directory is relative to the 18 | # documentation root, use os.path.abspath to make it absolute, like shown here. 19 | #sys.path.insert(0, os.path.abspath('.')) 20 | sys.path.insert(0, os.path.abspath('../..')) 21 | 22 | # -- General configuration ------------------------------------------------ 23 | 24 | # If your documentation needs a minimal Sphinx version, state it here. 25 | #needs_sphinx = '1.0' 26 | 27 | # Add any Sphinx extension module names here, as strings. They can be 28 | # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom 29 | # ones. 30 | extensions = [ 31 | 'sphinx.ext.intersphinx', 32 | 'sphinx.ext.autodoc', 33 | 'alabaster', 34 | ] 35 | 36 | 37 | # Add any paths that contain templates here, relative to this directory. 38 | templates_path = ['../templates'] 39 | 40 | # The suffix(es) of source filenames. 41 | # You can specify multiple suffix as a list of string: 42 | # source_suffix = ['.rst', '.md'] 43 | source_suffix = '.rst' 44 | 45 | # The encoding of source files. 46 | #source_encoding = 'utf-8-sig' 47 | 48 | # The master toctree document. 49 | master_doc = 'index' 50 | 51 | # General information about the project. 52 | project = 'django-elasticsearch-dsl' 53 | copyright = '' 54 | author = 'sabricot and others' 55 | 56 | 57 | # The version info for the project you're documenting, acts as replacement for 58 | # |version| and |release|, also used in various other places throughout the 59 | # built documents. 60 | # 61 | # The full version, including alpha/beta/rc tags. 62 | release = '7.1.1' 63 | # The short X.Y version 64 | version = ".".join(release.split(".", 3)[:2]) 65 | 66 | # The language for content autogenerated by Sphinx. Refer to documentation 67 | # for a list of supported languages. 68 | # 69 | # This is also used if you do content translation via gettext catalogs. 70 | # Usually you set "language" from the command line for these cases. 71 | language = None 72 | 73 | # There are two options for replacing |today|: either, you set today to some 74 | # non-false value, then it is used: 75 | #today = '' 76 | # Else, today_fmt is used as the format for a strftime call. 77 | #today_fmt = '%B %d, %Y' 78 | 79 | # List of patterns, relative to source directory, that match files and 80 | # directories to ignore when looking for source files. 81 | # This patterns also effect to html_static_path and html_extra_path 82 | exclude_patterns = [] 83 | 84 | # The reST default role (used for this markup: `text`) to use for all 85 | # documents. 86 | #default_role = None 87 | 88 | # If true, '()' will be appended to :func: etc. cross-reference text. 89 | #add_function_parentheses = True 90 | 91 | # If true, the current module name will be prepended to all description 92 | # unit titles (such as .. function::). 93 | #add_module_names = True 94 | 95 | # If true, sectionauthor and moduleauthor directives will be shown in the 96 | # output. They are ignored by default. 97 | #show_authors = False 98 | 99 | # The name of the Pygments (syntax highlighting) style to use. 100 | pygments_style = 'sphinx' 101 | 102 | # A list of ignored prefixes for module index sorting. 103 | #modindex_common_prefix = [] 104 | 105 | # If true, keep warnings as "system message" paragraphs in the built documents. 106 | #keep_warnings = False 107 | 108 | # If true, `todo` and `todoList` produce output, else they produce nothing. 109 | todo_include_todos = False 110 | 111 | 112 | # -- Options for HTML output ---------------------------------------------- 113 | 114 | # The theme to use for HTML and HTML Help pages. See the documentation for 115 | # a list of builtin themes. 116 | html_theme = 'alabaster' 117 | 118 | # Theme options are theme-specific and customize the look and feel of a theme 119 | # further. For a list of options available for each theme, see the 120 | # documentation. 121 | #html_theme_options = {} 122 | html_theme_options = { 123 | 'description': 'Using elasticsearch-dsl with Django', 124 | 'github_user': 'sabricot', 125 | 'github_repo': 'django-elasticsearch-dsl', 126 | 'github_banner': True} 127 | 128 | # Add any paths that contain custom themes here, relative to this directory. 129 | #html_theme_path = [] 130 | 131 | # The name for this set of Sphinx documents. 132 | # " v documentation" by default. 133 | #html_title = '' 134 | 135 | # A shorter title for the navigation bar. Default is the same as html_title. 136 | #html_short_title = None 137 | 138 | # The name of an image file (relative to this directory) to place at the top 139 | # of the sidebar. 140 | #html_logo = None 141 | 142 | # The name of an image file (relative to this directory) to use as a favicon of 143 | # the docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 144 | # pixels large. 145 | #html_favicon = None 146 | 147 | # Add any paths that contain custom static files (such as style sheets) here, 148 | # relative to this directory. They are copied after the builtin static files, 149 | # so a file named "default.css" will overwrite the builtin "default.css". 150 | html_static_path = ['../_static'] 151 | 152 | # Add any extra paths that contain custom files (such as robots.txt or 153 | # .htaccess) here, relative to this directory. These files are copied 154 | # directly to the root of the documentation. 155 | #html_extra_path = [] 156 | 157 | # If not None, a 'Last updated on:' timestamp is inserted at every page 158 | # bottom, using the given strftime format. 159 | # The empty string is equivalent to '%b %d, %Y'. 160 | #html_last_updated_fmt = None 161 | 162 | # If true, SmartyPants will be used to convert quotes and dashes to 163 | # typographically correct entities. 164 | #html_use_smartypants = True 165 | 166 | # Custom sidebar templates, maps document names to template names. 167 | #html_sidebars = {} 168 | 169 | 170 | # Additional templates that should be rendered to pages, maps page names to 171 | # template names. 172 | #html_additional_pages = {} 173 | 174 | # If false, no module index is generated. 175 | #html_domain_indices = True 176 | 177 | # If false, no index is generated. 178 | #html_use_index = True 179 | 180 | # If true, the index is split into individual pages for each letter. 181 | #html_split_index = False 182 | 183 | # If true, links to the reST sources are added to the pages. 184 | #html_show_sourcelink = True 185 | 186 | # If true, "Created using Sphinx" is shown in the HTML footer. Default is True. 187 | #html_show_sphinx = True 188 | 189 | # If true, "(C) Copyright ..." is shown in the HTML footer. Default is True. 190 | #html_show_copyright = True 191 | 192 | # If true, an OpenSearch description file will be output, and all pages will 193 | # contain a tag referring to it. The value of this option must be the 194 | # base URL from which the finished HTML is served. 195 | #html_use_opensearch = '' 196 | 197 | # This is the file name suffix for HTML files (e.g. ".xhtml"). 198 | #html_file_suffix = None 199 | 200 | # Language to be used for generating the HTML full-text search index. 201 | # Sphinx supports the following languages: 202 | # 'da', 'de', 'en', 'es', 'fi', 'fr', 'h', 'it', 'ja' 203 | # 'nl', 'no', 'pt', 'ro', 'r', 'sv', 'tr', 'zh' 204 | #html_search_language = 'en' 205 | 206 | # A dictionary with options for the search language support, empty by default. 207 | # 'ja' uses this config value. 208 | # 'zh' user can custom change `jieba` dictionary path. 209 | #html_search_options = {'type': 'default'} 210 | 211 | # The name of a javascript file (relative to the configuration directory) that 212 | # implements a search results scorer. If empty, the default will be used. 213 | #html_search_scorer = 'scorer.js' 214 | 215 | # Output file base name for HTML help builder. 216 | htmlhelp_basename = 'django-elasticsearch-dsl' 217 | 218 | # -- Options for LaTeX output --------------------------------------------- 219 | 220 | latex_elements = { 221 | # The paper size ('letterpaper' or 'a4paper'). 222 | #'papersize': 'letterpaper', 223 | 224 | # The font size ('10pt', '11pt' or '12pt'). 225 | #'pointsize': '10pt', 226 | 227 | # Additional stuff for the LaTeX preamble. 228 | #'preamble': '', 229 | 230 | # Latex figure (float) alignment 231 | #'figure_align': 'htbp', 232 | } 233 | 234 | # Grouping the document tree into LaTeX files. List of tuples 235 | # (source start file, target name, title, 236 | # author, documentclass [howto, manual, or own class]). 237 | latex_documents = [ 238 | (master_doc, 'django_elasticsearch_dsl.tex', 'Django Elasticsearch DSL Documentation', 239 | author, 'manual'), 240 | ] 241 | 242 | # The name of an image file (relative to this directory) to place at the top of 243 | # the title page. 244 | #latex_logo = None 245 | 246 | # For "manual" documents, if this is true, then toplevel headings are parts, 247 | # not chapters. 248 | #latex_use_parts = False 249 | 250 | # If true, show page references after internal links. 251 | #latex_show_pagerefs = False 252 | 253 | # If true, show URL addresses after external links. 254 | #latex_show_urls = False 255 | 256 | # Documents to append as an appendix to all manuals. 257 | #latex_appendices = [] 258 | 259 | # If false, no module index is generated. 260 | #latex_domain_indices = True 261 | 262 | 263 | # -- Options for manual page output --------------------------------------- 264 | 265 | # One entry per manual page. List of tuples 266 | # (source start file, name, description, authors, manual section). 267 | man_pages = [ 268 | (master_doc, 'django_elasticsearch_dsl', 'Django Elasticsearc DSL Documentation', 269 | [author], 1) 270 | ] 271 | 272 | # If true, show URL addresses after external links. 273 | #man_show_urls = False 274 | 275 | 276 | # -- Options for Texinfo output ------------------------------------------- 277 | 278 | # Grouping the document tree into Texinfo files. List of tuples 279 | # (source start file, target name, title, author, 280 | # dir menu entry, description, category) 281 | texinfo_documents = [ 282 | (master_doc, 'django_elasticsearch_dsl', 'Django Elasticsearch DSL Documentation', 283 | author, 'django_elasticsearch_dsl', 'elasticsearch-dsl intégration in Django', 284 | 'Miscellaneous'), 285 | ] 286 | 287 | # Documents to append as an appendix to all manuals. 288 | #texinfo_appendices = [] 289 | 290 | # If false, no module index is generated. 291 | #texinfo_domain_indices = True 292 | 293 | # How to display URL addresses: 'footnote', 'no', or 'inline'. 294 | #texinfo_show_urls = 'footnote' 295 | 296 | # If true, do not generate a @detailmenu in the "Top" node's menu. 297 | #texinfo_no_detailmenu = False 298 | 299 | 300 | # Example configuration for intersphinx: refer to the Python standard library. 301 | intersphinx_mapping = { 302 | 'python': ('https://docs.python.org/', None), 303 | 'es-py': ('https://elasticsearch-py.readthedocs.io/en/master/', None) , 304 | 'es-dsl': ('https://elasticsearch-dsl.readthedocs.io/en/latest/', None), 305 | } 306 | -------------------------------------------------------------------------------- /docs/source/develop.rst: -------------------------------------------------------------------------------- 1 | Contributing 2 | ############ 3 | 4 | We are glad to welcome any contributor. 5 | 6 | Report bugs or propose enhancements through `github bug tracker`_ 7 | 8 | _`github bug tracker`: https://github.com/sabricot/django-elasticsearch-dsl/issues 9 | 10 | 11 | If you want to contribute, the code is on github: 12 | https://github.com/sabricot/django-elasticsearch-dsl 13 | 14 | Testing 15 | ======= 16 | 17 | 18 | You can run the tests by creating a Python virtual environment, installing 19 | the requirements from ``requirements_test.txt`` (``pip install -r requirements_test``):: 20 | 21 | $ python runtests.py 22 | 23 | 24 | For integration testing with a running Elasticsearch server:: 25 | 26 | $ python runtests.py --elasticsearch [localhost:9200] 27 | 28 | TODO 29 | ==== 30 | 31 | - Add support for --using (use another Elasticsearch cluster) in management commands. 32 | - Add management commands for mapping level operations (like update_mapping....). 33 | - Generate ObjectField/NestField properties from a Document class. 34 | - More examples. 35 | - Better ``ESTestCase`` and documentation for testing 36 | 37 | 38 | -------------------------------------------------------------------------------- /docs/source/es_index.rst: -------------------------------------------------------------------------------- 1 | Index 2 | ##### 3 | 4 | In typical scenario using `class Index` on a `Document` class is sufficient to perform any action. 5 | In a few cases though it can be useful to manipulate an Index object directly. 6 | 7 | To define an Elasticsearch index you must instantiate a ``elasticsearch_dsl.Index`` class 8 | and set the name and settings of the index. 9 | After you instantiate your class, 10 | you need to associate it with the Document you want to put in this Elasticsearch index 11 | and also add the `registry.register_document` decorator. 12 | 13 | 14 | .. code-block:: python 15 | 16 | # documents.py 17 | from elasticsearch_dsl import Index 18 | from django_elasticsearch_dsl import Document 19 | from .models import Car, Manufacturer 20 | 21 | # The name of your index 22 | car = Index('cars') 23 | # See Elasticsearch Indices API reference for available settings 24 | car.settings( 25 | number_of_shards=1, 26 | number_of_replicas=0 27 | ) 28 | 29 | @registry.register_document 30 | @car.document 31 | class CarDocument(Document): 32 | class Django: 33 | model = Car 34 | fields = [ 35 | 'name', 36 | 'color', 37 | ] 38 | 39 | @registry.register_document 40 | class ManufacturerDocument(Document): 41 | class Index: 42 | name = 'manufacture' 43 | settings = {'number_of_shards': 1, 44 | 'number_of_replicas': 0} 45 | 46 | class Django: 47 | model = Manufacturer 48 | fields = [ 49 | 'name', 50 | 'country_code', 51 | ] 52 | 53 | When you execute the command:: 54 | 55 | $ ./manage.py search_index --rebuild 56 | 57 | This will create two index named ``cars`` and ``manufacture`` 58 | in Elasticsearch with appropriate mapping. 59 | 60 | ** If your model have huge amount of data, its preferred to use `parallel` indexing. 61 | To do that, you can pass `--parallel` flag while reindexing or populating. 62 | ** 63 | 64 | 65 | Signals 66 | ======= 67 | 68 | * ``django_elasticsearch_dsl.signals.post_index`` 69 | Sent after document indexing is completed. (not applicable for ``parallel`` indexing). 70 | Provides the following arguments: 71 | 72 | ``sender`` 73 | A subclass of ``django_elasticsearch_dsl.documents.DocType`` used 74 | to perform indexing. 75 | 76 | ``instance`` 77 | A ``django_elasticsearch_dsl.documents.DocType`` subclass instance. 78 | 79 | ``actions`` 80 | A generator containing document data that were sent to elasticsearch for indexing. 81 | 82 | ``response`` 83 | The response from ``bulk()`` function of ``elasticsearch-py``, 84 | which includes ``success`` count and ``failed`` count or ``error`` list. 85 | -------------------------------------------------------------------------------- /docs/source/fields.rst: -------------------------------------------------------------------------------- 1 | Fields 2 | ###### 3 | 4 | Once again the ``django_elasticsearch_dsl.fields`` are subclasses of elasticsearch-dsl-py 5 | fields_. They just add support for retrieving data from django models. 6 | 7 | 8 | .. _fields: http://elasticsearch-dsl.readthedocs.io/en/stable/persistence.html#mappings 9 | 10 | Using Different Attributes for Model Fields 11 | =========================================== 12 | 13 | Let's say you don't want to store the type of the car as an integer, but as the 14 | corresponding string instead. You need some way to convert the type field on 15 | the model to a string, so we'll just add a method for it: 16 | 17 | .. code-block:: python 18 | 19 | # models.py 20 | 21 | class Car(models.Model): 22 | # ... # 23 | def type_to_string(self): 24 | """Convert the type field to its string representation 25 | (the boneheaded way). 26 | """ 27 | if self.type == 1: 28 | return "Sedan" 29 | elif self.type == 2: 30 | return "Truck" 31 | else: 32 | return "SUV" 33 | 34 | Now we need to tell our ``Document`` subclass to use that method instead of just 35 | accessing the ``type`` field on the model directly. Change the CarDocument to look 36 | like this: 37 | 38 | .. code-block:: python 39 | 40 | # documents.py 41 | 42 | from django_elasticsearch_dsl import Document, fields 43 | 44 | # ... # 45 | 46 | @registry.register_document 47 | class CarDocument(Document): 48 | # add a string field to the Elasticsearch mapping called type, the 49 | # value of which is derived from the model's type_to_string attribute 50 | type = fields.TextField(attr="type_to_string") 51 | 52 | class Django: 53 | model = Car 54 | # we removed the type field from here 55 | fields = [ 56 | 'name', 57 | 'color', 58 | 'description', 59 | ] 60 | 61 | After a change like this we need to rebuild the index with:: 62 | 63 | $ ./manage.py search_index --rebuild 64 | 65 | Using prepare_field 66 | =================== 67 | 68 | Sometimes, you need to do some extra prepping before a field should be saved to 69 | Elasticsearch. You can add a ``prepare_foo(self, instance)`` method to a Document 70 | (where foo is the name of the field), and that will be called when the field 71 | needs to be saved. 72 | 73 | .. code-block:: python 74 | 75 | # documents.py 76 | 77 | # ... # 78 | 79 | class CarDocument(Document): 80 | # ... # 81 | 82 | foo = TextField() 83 | 84 | def prepare_foo(self, instance): 85 | return " ".join(instance.foos) 86 | 87 | Handle relationship with NestedField/ObjectField 88 | ================================================ 89 | 90 | For example for a model with ForeignKey relationships. 91 | 92 | .. code-block:: python 93 | 94 | # models.py 95 | 96 | class Car(models.Model): 97 | name = models.CharField() 98 | color = models.CharField() 99 | manufacturer = models.ForeignKey('Manufacturer') 100 | 101 | class Manufacturer(models.Model): 102 | name = models.CharField() 103 | country_code = models.CharField(max_length=2) 104 | created = models.DateField() 105 | 106 | class Ad(models.Model): 107 | title = models.CharField() 108 | description = models.TextField() 109 | created = models.DateField(auto_now_add=True) 110 | modified = models.DateField(auto_now=True) 111 | url = models.URLField() 112 | car = models.ForeignKey('Car', related_name='ads') 113 | 114 | 115 | You can use an ObjectField or a NestedField. 116 | 117 | .. code-block:: python 118 | 119 | # documents.py 120 | 121 | from django_elasticsearch_dsl import Document, fields 122 | from .models import Car, Manufacturer, Ad 123 | 124 | @registry.register_document 125 | class CarDocument(Document): 126 | manufacturer = fields.ObjectField(properties={ 127 | 'name': fields.TextField(), 128 | 'country_code': fields.TextField(), 129 | }) 130 | ads = fields.NestedField(properties={ 131 | 'description': fields.TextField(analyzer=html_strip), 132 | 'title': fields.TextField(), 133 | 'pk': fields.IntegerField(), 134 | }) 135 | 136 | class Index: 137 | name = 'cars' 138 | 139 | class Django: 140 | model = Car 141 | fields = [ 142 | 'name', 143 | 'color', 144 | ] 145 | related_models = [Manufacturer, Ad] # Optional: to ensure the Car will be re-saved when Manufacturer or Ad is updated 146 | 147 | def get_queryset(self): 148 | """Not mandatory but to improve performance we can select related in one sql request""" 149 | return super(CarDocument, self).get_queryset().select_related( 150 | 'manufacturer' 151 | ) 152 | 153 | def get_instances_from_related(self, related_instance): 154 | """If related_models is set, define how to retrieve the Car instance(s) from the related model. 155 | The related_models option should be used with caution because it can lead in the index 156 | to the updating of a lot of items. 157 | """ 158 | if isinstance(related_instance, Manufacturer): 159 | return related_instance.car_set.all() 160 | elif isinstance(related_instance, Ad): 161 | return related_instance.car 162 | 163 | 164 | Field Classes 165 | ============= 166 | 167 | Most Elasticsearch field types_ are supported. The ``attr`` argument is a dotted 168 | "attribute path" which will be looked up on the model using Django template 169 | semantics (dict lookup, attribute lookup, list index lookup). By default the attr 170 | argument is set to the field name. 171 | 172 | For the rest, the field properties are the same as elasticsearch-dsl 173 | fields_. 174 | 175 | So for example you can use a custom analyzer_: 176 | 177 | .. _analyzer: http://elasticsearch-dsl.readthedocs.io/en/stable/persistence.html#analysis 178 | .. _types: https://www.elastic.co/guide/en/elasticsearch/reference/5.4/mapping-types.html 179 | 180 | .. code-block:: python 181 | 182 | # documents.py 183 | 184 | # ... # 185 | 186 | html_strip = analyzer( 187 | 'html_strip', 188 | tokenizer="standard", 189 | filter=["lowercase", "stop", "snowball"], 190 | char_filter=["html_strip"] 191 | ) 192 | 193 | @registry.register_document 194 | class CarDocument(Document): 195 | description = fields.TextField( 196 | analyzer=html_strip, 197 | fields={'raw': fields.KeywordField()} 198 | ) 199 | 200 | class Django: 201 | model = Car 202 | fields = [ 203 | 'name', 204 | 'color', 205 | ] 206 | 207 | 208 | Available Fields 209 | ================ 210 | 211 | - Simple Fields 212 | 213 | - ``BooleanField(attr=None, **elasticsearch_properties)`` 214 | - ``ByteField(attr=None, **elasticsearch_properties)`` 215 | - ``CompletionField(attr=None, **elasticsearch_properties)`` 216 | - ``DateField(attr=None, **elasticsearch_properties)`` 217 | - ``DoubleField(attr=None, **elasticsearch_properties)`` 218 | - ``FileField(attr=None, **elasticsearch_properties)`` 219 | - ``FloatField(attr=None, **elasticsearch_properties)`` 220 | - ``IntegerField(attr=None, **elasticsearch_properties)`` 221 | - ``IpField(attr=None, **elasticsearch_properties)`` 222 | - ``KeywordField(attr=None, **elasticsearch_properties)`` 223 | - ``GeoPointField(attr=None, **elasticsearch_properties)`` 224 | - ``GeoShapeField(attr=None, **elasticsearch_properties)`` 225 | - ``ShortField(attr=None, **elasticsearch_properties)`` 226 | - ``TextField(attr=None, **elasticsearch_properties)`` 227 | 228 | - Complex Fields 229 | 230 | - ``ObjectField(properties, attr=None, **elasticsearch_properties)`` 231 | - ``NestedField(properties, attr=None, **elasticsearch_properties)`` 232 | 233 | ``properties`` is a dict where the key is a field name, and the value is a field 234 | instance. 235 | 236 | 237 | Field Mapping 238 | ============= 239 | Django Elasticsearch DSL maps most of the django fields 240 | appropriate Elasticsearch Field. You can find the field 241 | mapping on `documents.py` file in the `model_field_class_to_field_class` 242 | variable. If you need to change the behavior of this mapping, or add mapping 243 | for your custom field, you can do so by overwriting the classmethod 244 | `get_model_field_class_to_field_class`. Remember, you need to inherit 245 | `django_elasticsearch_dsl.fields.DEDField` for your custom field. 246 | Like following 247 | 248 | .. code-block:: python 249 | 250 | from django_elasticsearch_dsl.fields import DEDField 251 | 252 | class MyCustomDEDField(DEDField, ElasticsearchField): 253 | pass 254 | 255 | @classmethod 256 | def get_model_field_class_to_field_class(cls): 257 | field_mapping = super().get_model_field_class_to_field_class() 258 | field_mapping[MyCustomDjangoField] = MyCustomDEDField 259 | 260 | 261 | Document id 262 | =========== 263 | 264 | The elasticsearch document id (``_id``) is not strictly speaking a field, as it is not 265 | part of the document itself. The default behavior of ``django_elasticsearch_dsl`` 266 | is to use the primary key of the model as the document's id (``pk`` or ``id``). 267 | Nevertheless, it can sometimes be useful to change this default behavior. For this, one 268 | can redefine the ``generate_id(cls, instance)`` class method of the ``Document`` class. 269 | 270 | For example, to use an article's slug as the elasticsearch ``_id`` instead of the 271 | article's integer id, one could use: 272 | 273 | .. code-block:: python 274 | 275 | # models.py 276 | 277 | from django.db import models 278 | 279 | class Article(models.Model): 280 | # ... # 281 | 282 | slug = models.SlugField( 283 | max_length=255, 284 | unique=True, 285 | ) 286 | 287 | # ... # 288 | 289 | 290 | # documents.py 291 | 292 | from .models import Article 293 | 294 | class ArticleDocument(Document): 295 | class Django: 296 | model = Article 297 | 298 | # ... # 299 | 300 | @classmethod 301 | def generate_id(cls, article): 302 | return article.slug 303 | -------------------------------------------------------------------------------- /docs/source/index.rst: -------------------------------------------------------------------------------- 1 | Django Elasticsearch DSL 2 | ######################### 3 | 4 | Contents: 5 | 6 | .. toctree:: 7 | :maxdepth: 2 8 | 9 | about 10 | quickstart 11 | es_index 12 | fields 13 | settings 14 | management 15 | develop 16 | 17 | 18 | Indices and tables 19 | ================== 20 | 21 | * :ref:`genindex` 22 | * :ref:`modindex` 23 | * :ref:`search` 24 | 25 | -------------------------------------------------------------------------------- /docs/source/management.rst: -------------------------------------------------------------------------------- 1 | Management Commands 2 | ################### 3 | 4 | Delete all indices in Elasticsearch or only the indices associate with a model (``--models``): 5 | 6 | :: 7 | 8 | $ search_index --delete [-f] [--models [app[.model] app[.model] ...]] 9 | 10 | 11 | Create the indices and their mapping in Elasticsearch: 12 | 13 | :: 14 | 15 | $ search_index --create [--models [app[.model] app[.model] ...]] 16 | 17 | Populate the Elasticsearch mappings with the Django models data (index need to be existing): 18 | 19 | :: 20 | 21 | $ search_index --populate [--models [app[.model] app[.model] ...]] [--parallel] [--refresh] 22 | 23 | Recreate and repopulate the indices: 24 | 25 | :: 26 | 27 | $ search_index --rebuild [-f] [--models [app[.model] app[.model] ...]] [--parallel] [--refresh] 28 | 29 | Recreate and repopulate the indices using aliases: 30 | 31 | :: 32 | 33 | $ search_index --rebuild --use-alias [--models [app[.model] app[.model] ...]] [--parallel] [--refresh] 34 | 35 | Recreate and repopulate the indices using aliases, but not deleting the indices that previously pointed to the aliases: 36 | 37 | :: 38 | 39 | $ search_index --rebuild --use-alias --use-alias-keep-index [--models [app[.model] app[.model] ...]] [--parallel] [--refresh] 40 | -------------------------------------------------------------------------------- /docs/source/quickstart.rst: -------------------------------------------------------------------------------- 1 | 2 | Quickstart 3 | ########## 4 | 5 | Install and configure 6 | ===================== 7 | 8 | Install Django Elasticsearch DSL:: 9 | 10 | pip install django-elasticsearch-dsl 11 | 12 | 13 | Then add ``django_elasticsearch_dsl`` to the INSTALLED_APPS 14 | 15 | You must define ``ELASTICSEARCH_DSL`` in your django settings. 16 | 17 | For example: 18 | 19 | .. code-block:: python 20 | 21 | ELASTICSEARCH_DSL={ 22 | 'default': { 23 | 'hosts': 'localhost:9200', 24 | 'http_auth': ('username', 'password') 25 | } 26 | } 27 | 28 | ``ELASTICSEARCH_DSL`` is then passed to ``elasticsearch-dsl-py.connections.configure`` (see here_). 29 | 30 | .. _here: http://elasticsearch-dsl.readthedocs.io/en/stable/configuration.html#multiple-clusters 31 | 32 | Declare data to index 33 | ===================== 34 | 35 | Then for a model: 36 | 37 | .. code-block:: python 38 | 39 | # models.py 40 | 41 | class Car(models.Model): 42 | name = models.CharField() 43 | color = models.CharField() 44 | description = models.TextField() 45 | type = models.IntegerField(choices=[ 46 | (1, "Sedan"), 47 | (2, "Truck"), 48 | (4, "SUV"), 49 | ]) 50 | 51 | To make this model work with Elasticsearch, 52 | create a subclass of ``django_elasticsearch_dsl.Document``, 53 | create a ``class Index`` inside the ``Document`` class 54 | to define your Elasticsearch indices, names, settings etc 55 | and at last register the class using ``registry.register_document`` decorator. 56 | It is required to define ``Document`` class in ``documents.py`` in your app directory. 57 | 58 | .. code-block:: python 59 | 60 | # documents.py 61 | 62 | from django_elasticsearch_dsl import Document 63 | from django_elasticsearch_dsl.registries import registry 64 | from .models import Car 65 | 66 | 67 | @registry.register_document 68 | class CarDocument(Document): 69 | class Index: 70 | # Name of the Elasticsearch index 71 | name = 'cars' 72 | # See Elasticsearch Indices API reference for available settings 73 | settings = {'number_of_shards': 1, 74 | 'number_of_replicas': 0} 75 | 76 | class Django: 77 | model = Car # The model associated with this Document 78 | 79 | # The fields of the model you want to be indexed in Elasticsearch 80 | fields = [ 81 | 'name', 82 | 'color', 83 | 'description', 84 | 'type', 85 | ] 86 | 87 | # Ignore auto updating of Elasticsearch when a model is saved 88 | # or deleted: 89 | # ignore_signals = True 90 | 91 | # Configure how the index should be refreshed after an update. 92 | # See Elasticsearch documentation for supported options: 93 | # https://www.elastic.co/guide/en/elasticsearch/reference/master/docs-refresh.html 94 | # This per-Document setting overrides settings.ELASTICSEARCH_DSL_AUTO_REFRESH. 95 | # auto_refresh = False 96 | 97 | # Paginate the django queryset used to populate the index with the specified size 98 | # (by default it uses the database driver's default setting) 99 | # queryset_pagination = 5000 100 | 101 | Populate 102 | ======== 103 | 104 | To create and populate the Elasticsearch index and mapping use the search_index command:: 105 | 106 | $ ./manage.py search_index --rebuild 107 | 108 | Now, when you do something like: 109 | 110 | .. code-block:: python 111 | 112 | car = Car( 113 | name="Car one", 114 | color="red", 115 | type=1, 116 | description="A beautiful car" 117 | ) 118 | car.save() 119 | 120 | The object will be saved in Elasticsearch too (using a signal handler). 121 | 122 | Search 123 | ====== 124 | 125 | To get an elasticsearch-dsl-py Search_ instance, use: 126 | 127 | .. code-block:: python 128 | 129 | s = CarDocument.search().filter("term", color="red") 130 | 131 | # or 132 | 133 | s = CarDocument.search().query("match", description="beautiful") 134 | 135 | for hit in s: 136 | print( 137 | "Car name : {}, description {}".format(hit.name, hit.description) 138 | ) 139 | 140 | The previous example returns a result specific to elasticsearch_dsl_, 141 | but it is also possible to convert the elastisearch result into a real django queryset, 142 | just be aware that this costs a sql request to retrieve the model instances 143 | with the ids returned by the elastisearch query. 144 | 145 | .. _Search: https://elasticsearch-dsl.readthedocs.io/en/latest/search_dsl.html#the-search-object 146 | .. _elasticsearch_dsl: http://elasticsearch-dsl.readthedocs.io/en/latest/search_dsl.html#response 147 | 148 | .. code-block:: python 149 | 150 | s = CarDocument.search().filter("term", color="blue")[:30] 151 | qs = s.to_queryset() 152 | # qs is just a django queryset and it is called with order_by to keep 153 | # the same order as the elasticsearch result. 154 | for car in qs: 155 | print(car.name) 156 | -------------------------------------------------------------------------------- /docs/source/settings.rst: -------------------------------------------------------------------------------- 1 | Settings 2 | ######## 3 | 4 | 5 | ELASTICSEARCH_DSL_AUTOSYNC 6 | ========================== 7 | 8 | Default: ``True`` 9 | 10 | Set to ``False`` to globally disable auto-syncing. 11 | 12 | ELASTICSEARCH_DSL_INDEX_SETTINGS 13 | ================================ 14 | 15 | Default: ``{}`` 16 | 17 | Additional options passed to the elasticsearch-dsl Index settings (like ``number_of_replicas`` or ``number_of_shards``). 18 | 19 | ELASTICSEARCH_DSL_AUTO_REFRESH 20 | ============================== 21 | 22 | Default: ``True`` 23 | 24 | Set to ``False`` not force an `index refresh `_ with every save. 25 | 26 | ELASTICSEARCH_DSL_SIGNAL_PROCESSOR 27 | ================================== 28 | 29 | This (optional) setting controls what SignalProcessor class is used to handle 30 | Django's signals and keep the search index up-to-date. 31 | 32 | An example: 33 | 34 | .. code-block:: python 35 | 36 | ELASTICSEARCH_DSL_SIGNAL_PROCESSOR = 'django_elasticsearch_dsl.signals.RealTimeSignalProcessor' 37 | 38 | Defaults to ``django_elasticsearch_dsl.signals.RealTimeSignalProcessor``. 39 | 40 | Options: ``django_elasticsearch_dsl.signals.RealTimeSignalProcessor`` \ ``django_elasticsearch_dsl.signals.CelerySignalProcessor`` 41 | 42 | In this ``CelerySignalProcessor`` implementation, 43 | Create and update operations will record the updated data primary key from the database and delay the time to find the association to ensure eventual consistency. 44 | Delete operations are processed to obtain associated data before database records are deleted. 45 | And celery needs to be pre-configured in the django project, for example `Using Celery with Django `. 46 | 47 | You could, for instance, make a ``CustomSignalProcessor`` which would apply 48 | update jobs as your wish. 49 | 50 | ELASTICSEARCH_DSL_PARALLEL 51 | ========================== 52 | 53 | Default: ``False`` 54 | 55 | Run indexing (populate and rebuild) in parallel using ES' parallel_bulk() method. 56 | Note that some databases (e.g. sqlite) do not play well with this option. 57 | -------------------------------------------------------------------------------- /example/README.rst: -------------------------------------------------------------------------------- 1 | ================================= 2 | Django Elasticsearch DSL Test App 3 | ================================= 4 | 5 | Simple django app for test some django-elasticsearch-dsl features. 6 | 7 | 8 | Installation 9 | ------------ 10 | 11 | In a python virtualenv run:: 12 | 13 | $ pip install -r requirements.txt 14 | 15 | 16 | You need an Elasticsearch server running. Then change the Elasticsearch 17 | connections setting in example/settings.py. 18 | 19 | .. code:: python 20 | 21 | ELASTICSEARCH_DSL={ 22 | 'default': { 23 | 'hosts': 'localhost:9200' 24 | }, 25 | } 26 | 27 | To launch a functional server:: 28 | 29 | $ ./manage.py migrate 30 | $ ./manage.py createsuperuser 31 | 32 | You can use django-autofixture for populate django models with fake data:: 33 | 34 | $ ./manage.py loadtestdata test_app.Manufacturer:10 test_app.Car:100 test_app.Ad:500 35 | 36 | Then build the Elasticsearch index with:: 37 | 38 | $ ./manage.py search_index --rebuild 39 | 40 | And run the server (there is django admin but not any views yet):: 41 | 42 | $ ./manage.py runserver 43 | -------------------------------------------------------------------------------- /example/example/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/django-es/django-elasticsearch-dsl/e453afffe024725be27e457cdf67b1e8229bccfe/example/example/__init__.py -------------------------------------------------------------------------------- /example/example/settings.py: -------------------------------------------------------------------------------- 1 | """ 2 | Django settings for example project. 3 | 4 | For more information on this file, see 5 | https://docs.djangoproject.com/en/1.9/topics/settings/ 6 | 7 | For the full list of settings and their values, see 8 | https://docs.djangoproject.com/en/1.9/ref/settings/ 9 | """ 10 | 11 | import os 12 | 13 | # Build paths inside the project like this: os.path.join(BASE_DIR, ...) 14 | BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) 15 | 16 | # Quick-start development settings - unsuitable for production 17 | # See https://docs.djangoproject.com/en/1.9/howto/deployment/checklist/ 18 | 19 | # SECURITY WARNING: keep the secret key used in production secret! 20 | SECRET_KEY = '%b&p!s^@301dc=@y^*x92ff*hzelqm0)m0vy29-dwexs=e1w+t' 21 | 22 | # SECURITY WARNING: don't run with debug turned on in production! 23 | DEBUG = True 24 | 25 | ALLOWED_HOSTS = [] 26 | 27 | # Application definition 28 | 29 | INSTALLED_APPS = [ 30 | 'django.contrib.admin', 31 | 'django.contrib.auth', 32 | 'django.contrib.contenttypes', 33 | 'django.contrib.sessions', 34 | 'django.contrib.messages', 35 | 'django.contrib.staticfiles', 36 | 37 | 'django_elasticsearch_dsl', 38 | 39 | # if your app has other dependencies that need to be added to the site 40 | # they should be added here 41 | 'test_app', 42 | ] 43 | 44 | MIDDLEWARE = [ 45 | 'django.middleware.security.SecurityMiddleware', 46 | 'django.contrib.sessions.middleware.SessionMiddleware', 47 | 'django.middleware.common.CommonMiddleware', 48 | 'django.middleware.csrf.CsrfViewMiddleware', 49 | 'django.contrib.auth.middleware.AuthenticationMiddleware', 50 | 'django.contrib.messages.middleware.MessageMiddleware', 51 | 'django.middleware.clickjacking.XFrameOptionsMiddleware', 52 | ] 53 | 54 | ROOT_URLCONF = 'example.urls' 55 | 56 | TEMPLATES = [ 57 | { 58 | 'BACKEND': 'django.template.backends.django.DjangoTemplates', 59 | 'DIRS': [os.path.join(BASE_DIR, 'templates'), ], 60 | 'APP_DIRS': True, 61 | 'OPTIONS': { 62 | 'context_processors': [ 63 | 'django.template.context_processors.debug', 64 | 'django.template.context_processors.request', 65 | 'django.contrib.auth.context_processors.auth', 66 | 'django.contrib.messages.context_processors.messages', 67 | ], 68 | }, 69 | }, 70 | ] 71 | 72 | WSGI_APPLICATION = 'example.wsgi.application' 73 | 74 | # Database 75 | # https://docs.djangoproject.com/en/1.9/ref/settings/#databases 76 | 77 | DATABASES = { 78 | 'default': { 79 | 'ENGINE': 'django.db.backends.sqlite3', 80 | 'NAME': os.path.join(BASE_DIR, 'db.sqlite3'), 81 | } 82 | } 83 | 84 | ELASTICSEARCH_DSL = { 85 | 'default': { 86 | 'hosts': 'localhost:9200' 87 | }, 88 | } 89 | 90 | ELASTICSEARCH_DSL_INDEX_SETTINGS = { 91 | 'number_of_shards': 1 92 | } 93 | 94 | # Password validation 95 | # https://docs.djangoproject.com/en/1.9/ref/settings/#auth-password-validators 96 | 97 | AUTH_PASSWORD_VALIDATORS = [ 98 | { 99 | 'NAME': 'django.contrib.auth.password_validation.UserAttributeSimilarityValidator', 100 | }, 101 | { 102 | 'NAME': 'django.contrib.auth.password_validation.MinimumLengthValidator', 103 | }, 104 | { 105 | 'NAME': 'django.contrib.auth.password_validation.CommonPasswordValidator', 106 | }, 107 | { 108 | 'NAME': 'django.contrib.auth.password_validation.NumericPasswordValidator', 109 | }, 110 | ] 111 | 112 | # Internationalization 113 | # https://docs.djangoproject.com/en/1.9/topics/i18n/ 114 | 115 | LANGUAGE_CODE = 'en-us' 116 | 117 | TIME_ZONE = 'UTC' 118 | 119 | USE_I18N = True 120 | 121 | USE_L10N = True 122 | 123 | USE_TZ = True 124 | 125 | # Static files (CSS, JavaScript, Images) 126 | # https://docs.djangoproject.com/en/1.9/howto/static-files/ 127 | 128 | STATIC_URL = '/static/' 129 | 130 | # Default primary key field type 131 | # https://docs.djangoproject.com/en/3.2/ref/settings/#default-auto-field 132 | 133 | DEFAULT_AUTO_FIELD = 'django.db.models.BigAutoField' 134 | -------------------------------------------------------------------------------- /example/example/urls.py: -------------------------------------------------------------------------------- 1 | """example URL Configuration 2 | 3 | The `urlpatterns` list routes URLs to views. For more information please see: 4 | https://docs.djangoproject.com/en/1.9/topics/http/urls/ 5 | Examples: 6 | Function views 7 | 1. Add an import: from my_app import views 8 | 2. Add a URL to urlpatterns: url(r'^$', views.home, name='home') 9 | Class-based views 10 | 1. Add an import: from other_app.views import Home 11 | 2. Add a URL to urlpatterns: url(r'^$', Home.as_view(), name='home') 12 | Including another URLconf 13 | 1. Import the include() function: from django.conf.urls import url, include 14 | 2. Add a URL to urlpatterns: url(r'^blog/', include('blog.urls')) 15 | """ 16 | from django.urls import path 17 | from django.contrib import admin 18 | 19 | urlpatterns = [ 20 | path('admin/', admin.site.urls), 21 | ] 22 | -------------------------------------------------------------------------------- /example/example/wsgi.py: -------------------------------------------------------------------------------- 1 | """ 2 | WSGI config for example project. 3 | 4 | It exposes the WSGI callable as a module-level variable named ``application``. 5 | 6 | For more information on this file, see 7 | https://docs.djangoproject.com/en/1.9/howto/deployment/wsgi/ 8 | """ 9 | 10 | import os 11 | 12 | from django.core.wsgi import get_wsgi_application 13 | 14 | os.environ.setdefault("DJANGO_SETTINGS_MODULE", "example.settings") 15 | 16 | application = get_wsgi_application() 17 | -------------------------------------------------------------------------------- /example/manage.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | import os 3 | import sys 4 | 5 | if __name__ == "__main__": 6 | os.environ.setdefault("DJANGO_SETTINGS_MODULE", "example.settings") 7 | 8 | from django.core.management import execute_from_command_line 9 | 10 | execute_from_command_line(sys.argv) 11 | -------------------------------------------------------------------------------- /example/requirements.txt: -------------------------------------------------------------------------------- 1 | # Your app requirements. 2 | -r ../requirements_test.txt 3 | # Your app in editable mode. 4 | -e ../ 5 | 6 | django-autofixture==0.12.1 7 | Pillow==6.2.2 8 | django==4.1.2 9 | -------------------------------------------------------------------------------- /example/test_app/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/django-es/django-elasticsearch-dsl/e453afffe024725be27e457cdf67b1e8229bccfe/example/test_app/__init__.py -------------------------------------------------------------------------------- /example/test_app/admin.py: -------------------------------------------------------------------------------- 1 | from django.contrib import admin 2 | 3 | from .models import Ad, Category, Car, Manufacturer 4 | 5 | 6 | admin.site.register(Ad) 7 | admin.site.register(Category) 8 | admin.site.register(Car) 9 | admin.site.register(Manufacturer) 10 | -------------------------------------------------------------------------------- /example/test_app/documents.py: -------------------------------------------------------------------------------- 1 | from elasticsearch_dsl import analyzer 2 | from django_elasticsearch_dsl import Document, Index, fields 3 | from django_elasticsearch_dsl.registries import registry 4 | 5 | from .models import Ad, Car, Manufacturer 6 | 7 | 8 | car = Index('test_cars') 9 | car.settings( 10 | number_of_shards=1, 11 | number_of_replicas=0 12 | ) 13 | 14 | 15 | html_strip = analyzer( 16 | 'html_strip', 17 | tokenizer="standard", 18 | filter=["lowercase", "stop", "snowball"], 19 | char_filter=["html_strip"] 20 | ) 21 | 22 | 23 | @registry.register_document 24 | class CarDocument(Document): 25 | manufacturer = fields.ObjectField(properties={ 26 | 'name': fields.TextField(), 27 | 'country': fields.TextField(), 28 | 'logo': fields.FileField(), 29 | }) 30 | 31 | ads = fields.NestedField(properties={ 32 | 'description': fields.TextField(analyzer=html_strip), 33 | 'title': fields.TextField(), 34 | 'pk': fields.IntegerField(), 35 | }) 36 | 37 | categories = fields.NestedField(properties={ 38 | 'title': fields.TextField(), 39 | }) 40 | 41 | class Django: 42 | model = Car 43 | fields = [ 44 | 'name', 45 | 'launched', 46 | 'type', 47 | ] 48 | 49 | class Index: 50 | name = "car" 51 | 52 | def get_instances_from_related(self, related_instance): 53 | if isinstance(related_instance, Ad): 54 | return related_instance.car 55 | 56 | # otherwise it's a Manufacturer or a Category 57 | return related_instance.car_set.all() 58 | 59 | 60 | @registry.register_document 61 | class ManufacturerDocument(Document): 62 | country = fields.TextField() 63 | 64 | class Django: 65 | model = Manufacturer 66 | fields = [ 67 | 'name', 68 | 'created', 69 | 'country_code', 70 | 'logo', 71 | ] 72 | 73 | class Index: 74 | name = "manufacturer" 75 | 76 | 77 | # @registry.register_document 78 | # class CarWithPrepareDocument(Document): 79 | # manufacturer = fields.ObjectField(properties={ 80 | # 'name': fields.TextField(), 81 | # 'country': fields.TextField(), 82 | # }) 83 | # 84 | # manufacturer_short = fields.ObjectField(properties={ 85 | # 'name': fields.TextField(), 86 | # }) 87 | # 88 | # class Index: 89 | # name = "car_with_prepare_index" 90 | # 91 | # class Django: 92 | # model = Car 93 | # related_models = [Manufacturer] 94 | # fields = [ 95 | # 'name', 96 | # 'launched', 97 | # 'type', 98 | # ] 99 | # 100 | # def prepare_manufacturer_with_related(self, car, related_to_ignore): 101 | # if (car.manufacturer is not None and car.manufacturer != 102 | # related_to_ignore): 103 | # return { 104 | # 'name': car.manufacturer.name, 105 | # 'country': car.manufacturer.country(), 106 | # } 107 | # return {} 108 | # 109 | # def prepare_manufacturer_short(self, car): 110 | # if car.manufacturer is not None: 111 | # return { 112 | # 'name': car.manufacturer.name, 113 | # } 114 | # return {} 115 | # 116 | # def get_instances_from_related(self, related_instance): 117 | # return related_instance.car_set.all() 118 | 119 | 120 | @registry.register_document 121 | class AdDocument(Document): 122 | description = fields.TextField( 123 | analyzer=html_strip, 124 | fields={'raw': fields.KeywordField()} 125 | ) 126 | 127 | class Django: 128 | model = Ad 129 | index = 'test_ads' 130 | fields = [ 131 | 'title', 132 | 'created', 133 | 'modified', 134 | 'url', 135 | ] 136 | 137 | class Index: 138 | name = "add" 139 | 140 | 141 | @registry.register_document 142 | class AdDocument2(Document): 143 | def __init__(self, *args, **kwargs): 144 | super(AdDocument2, self).__init__(*args, **kwargs) 145 | 146 | class Django: 147 | model = Ad 148 | index = 'test_ads2' 149 | fields = [ 150 | 'title', 151 | ] 152 | 153 | class Index: 154 | name = "add2" 155 | -------------------------------------------------------------------------------- /example/test_app/migrations/0001_initial.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Generated by Django 1.11.5 on 2017-11-21 18:46 3 | from __future__ import unicode_literals 4 | 5 | from django.db import migrations, models 6 | import django.db.models.deletion 7 | 8 | 9 | class Migration(migrations.Migration): 10 | 11 | initial = True 12 | 13 | dependencies = [ 14 | ] 15 | 16 | operations = [ 17 | migrations.CreateModel( 18 | name='Ad', 19 | fields=[ 20 | ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), 21 | ('title', models.CharField(max_length=255)), 22 | ('description', models.TextField()), 23 | ('created', models.DateField(auto_now_add=True)), 24 | ('modified', models.DateField(auto_now=True)), 25 | ('url', models.URLField()), 26 | ], 27 | ), 28 | migrations.CreateModel( 29 | name='Car', 30 | fields=[ 31 | ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), 32 | ('name', models.CharField(max_length=255)), 33 | ('launched', models.DateField()), 34 | ('type', models.CharField(choices=[('se', 'Sedan'), ('br', 'Break'), ('4x', '4x4'), ('co', 'Coupé')], default='se', max_length=2)), 35 | ], 36 | ), 37 | migrations.CreateModel( 38 | name='Category', 39 | fields=[ 40 | ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), 41 | ('title', models.CharField(max_length=255)), 42 | ('slug', models.CharField(max_length=255)), 43 | ], 44 | ), 45 | migrations.CreateModel( 46 | name='Manufacturer', 47 | fields=[ 48 | ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), 49 | ('name', models.CharField(max_length=255)), 50 | ('country_code', models.CharField(max_length=2)), 51 | ('logo', models.ImageField(blank=True, upload_to='')), 52 | ('created', models.DateField()), 53 | ], 54 | ), 55 | migrations.AddField( 56 | model_name='car', 57 | name='categories', 58 | field=models.ManyToManyField(to='test_app.Category'), 59 | ), 60 | migrations.AddField( 61 | model_name='car', 62 | name='manufacturer', 63 | field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.CASCADE, to='test_app.Manufacturer'), 64 | ), 65 | migrations.AddField( 66 | model_name='ad', 67 | name='car', 68 | field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.CASCADE, related_name='ads', to='test_app.Car'), 69 | ), 70 | ] 71 | -------------------------------------------------------------------------------- /example/test_app/migrations/0002_alter_ad_car_alter_ad_id_alter_car_id_and_more.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 4.1.2 on 2022-10-31 22:57 2 | 3 | from django.db import migrations, models 4 | import django.db.models.deletion 5 | 6 | 7 | class Migration(migrations.Migration): 8 | 9 | dependencies = [ 10 | ('test_app', '0001_initial'), 11 | ] 12 | 13 | operations = [ 14 | migrations.AlterField( 15 | model_name='ad', 16 | name='car', 17 | field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='ads', to='test_app.car'), 18 | ), 19 | migrations.AlterField( 20 | model_name='ad', 21 | name='id', 22 | field=models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID'), 23 | ), 24 | migrations.AlterField( 25 | model_name='car', 26 | name='id', 27 | field=models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID'), 28 | ), 29 | migrations.AlterField( 30 | model_name='car', 31 | name='manufacturer', 32 | field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, to='test_app.manufacturer'), 33 | ), 34 | migrations.AlterField( 35 | model_name='category', 36 | name='id', 37 | field=models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID'), 38 | ), 39 | migrations.AlterField( 40 | model_name='manufacturer', 41 | name='id', 42 | field=models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID'), 43 | ), 44 | ] 45 | -------------------------------------------------------------------------------- /example/test_app/migrations/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/django-es/django-elasticsearch-dsl/e453afffe024725be27e457cdf67b1e8229bccfe/example/test_app/migrations/__init__.py -------------------------------------------------------------------------------- /example/test_app/models.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | from __future__ import unicode_literals 3 | from django.db import models 4 | from six import python_2_unicode_compatible 5 | 6 | 7 | @python_2_unicode_compatible 8 | class Car(models.Model): 9 | TYPE_CHOICES = ( 10 | ('se', "Sedan"), 11 | ('br', "Break"), 12 | ('4x', "4x4"), 13 | ('co', "Coupé"), 14 | ) 15 | 16 | name = models.CharField(max_length=255) 17 | launched = models.DateField() 18 | type = models.CharField( 19 | max_length=2, 20 | choices=TYPE_CHOICES, 21 | default='se', 22 | ) 23 | manufacturer = models.ForeignKey( 24 | 'Manufacturer', null=True, on_delete=models.SET_NULL 25 | ) 26 | categories = models.ManyToManyField('Category') 27 | 28 | def __str__(self): 29 | return self.name 30 | 31 | 32 | COUNTRIES = { 33 | 'FR': 'France', 34 | 'UK': 'United Kingdom', 35 | 'ES': 'Spain', 36 | 'IT': 'Italya', 37 | } 38 | 39 | 40 | @python_2_unicode_compatible 41 | class Manufacturer(models.Model): 42 | name = models.CharField(max_length=255) 43 | country_code = models.CharField(max_length=2) 44 | logo = models.ImageField(blank=True) 45 | created = models.DateField() 46 | 47 | def country(self): 48 | return COUNTRIES.get(self.country_code, self.country_code) 49 | 50 | def __str__(self): 51 | return self.name 52 | 53 | 54 | @python_2_unicode_compatible 55 | class Category(models.Model): 56 | title = models.CharField(max_length=255) 57 | slug = models.CharField(max_length=255) 58 | 59 | def __str__(self): 60 | return self.title 61 | 62 | 63 | @python_2_unicode_compatible 64 | class Ad(models.Model): 65 | title = models.CharField(max_length=255) 66 | description = models.TextField() 67 | created = models.DateField(auto_now_add=True) 68 | modified = models.DateField(auto_now=True) 69 | url = models.URLField() 70 | car = models.ForeignKey( 71 | 'Car', related_name='ads', null=True, on_delete=models.SET_NULL 72 | ) 73 | 74 | def __str__(self): 75 | return self.title 76 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | django>=3.2 2 | elasticsearch-dsl>=8.0.0,<9.0.0 3 | -------------------------------------------------------------------------------- /requirements_dev.txt: -------------------------------------------------------------------------------- 1 | bumpversion==0.6.0 2 | wheel==0.41.2 3 | django>=3.2 4 | elasticsearch-dsl>=7.0.0,<8.0.0 5 | twine 6 | sphinx 7 | -e . 8 | -------------------------------------------------------------------------------- /requirements_test.txt: -------------------------------------------------------------------------------- 1 | coverage==7.3.1 2 | mock>=1.0.1 3 | flake8>=2.1.0 4 | tox>=1.7.0 5 | Pillow==10.0.0 6 | celery>=4.1.0 7 | 8 | # Additional test requirements go here 9 | -------------------------------------------------------------------------------- /runtests.py: -------------------------------------------------------------------------------- 1 | import os 2 | import sys 3 | import argparse 4 | 5 | from celery import Celery 6 | 7 | try: 8 | from django.conf import settings 9 | from django.test.utils import get_runner 10 | 11 | def get_settings(signal_processor): 12 | elasticsearch_dsl_default_settings = { 13 | 'hosts': os.environ.get( 14 | 'ELASTICSEARCH_URL', 15 | 'https://127.0.0.1:9200' 16 | ), 17 | 'basic_auth': ( 18 | os.environ.get('ELASTICSEARCH_USERNAME'), 19 | os.environ.get('ELASTICSEARCH_PASSWORD') 20 | ) 21 | } 22 | 23 | elasticsearch_certs_path = os.environ.get( 24 | 'ELASTICSEARCH_CERTS_PATH' 25 | ) 26 | if elasticsearch_certs_path: 27 | elasticsearch_dsl_default_settings['ca_certs'] = ( 28 | elasticsearch_certs_path 29 | ) 30 | else: 31 | elasticsearch_dsl_default_settings['verify_certs'] = False 32 | 33 | PROCESSOR_CLASSES = { 34 | 'realtime': 'django_elasticsearch_dsl.signals.RealTimeSignalProcessor', 35 | 'celery': 'django_elasticsearch_dsl.signals.CelerySignalProcessor', 36 | } 37 | 38 | signal_processor = PROCESSOR_CLASSES[signal_processor] 39 | settings.configure( 40 | DEBUG=True, 41 | USE_TZ=True, 42 | DATABASES={ 43 | "default": { 44 | "ENGINE": "django.db.backends.sqlite3", 45 | } 46 | }, 47 | INSTALLED_APPS=[ 48 | "django.contrib.auth", 49 | "django.contrib.contenttypes", 50 | "django.contrib.sites", 51 | "django_elasticsearch_dsl", 52 | "tests", 53 | ], 54 | SITE_ID=1, 55 | MIDDLEWARE_CLASSES=(), 56 | ELASTICSEARCH_DSL={ 57 | 'default': elasticsearch_dsl_default_settings 58 | }, 59 | DEFAULT_AUTO_FIELD="django.db.models.BigAutoField", 60 | CELERY_BROKER_URL='memory://localhost/', 61 | CELERY_TASK_ALWAYS_EAGER=True, 62 | CELERY_EAGER_PROPAGATES_EXCEPTIONS=True, 63 | ELASTICSEARCH_DSL_SIGNAL_PROCESSOR=signal_processor 64 | ) 65 | 66 | try: 67 | import django 68 | setup = django.setup 69 | except AttributeError: 70 | pass 71 | else: 72 | setup() 73 | 74 | app = Celery() 75 | app.config_from_object('django.conf:settings', namespace='CELERY') 76 | app.autodiscover_tasks(lambda: settings.INSTALLED_APPS) 77 | return settings 78 | 79 | except ImportError: 80 | import traceback 81 | traceback.print_exc() 82 | msg = "To fix this error, run: pip install -r requirements_test.txt" 83 | raise ImportError(msg) 84 | 85 | 86 | def make_parser(): 87 | parser = argparse.ArgumentParser() 88 | parser.add_argument( 89 | '--elasticsearch', 90 | nargs='?', 91 | metavar='localhost:9200', 92 | const='localhost:9200', 93 | help="To run integration test against an Elasticsearch server", 94 | ) 95 | parser.add_argument( 96 | '--signal-processor', 97 | nargs='?', 98 | default='realtime', 99 | choices=('realtime', 'celery'), 100 | help='Defines which signal backend to choose' 101 | ) 102 | parser.add_argument( 103 | '--elasticsearch-username', 104 | nargs='?', 105 | help="Username for Elasticsearch user" 106 | ) 107 | parser.add_argument( 108 | '--elasticsearch-password', 109 | nargs='?', 110 | help="Password for Elasticsearch user" 111 | ) 112 | parser.add_argument( 113 | '--elasticsearch-certs-path', 114 | nargs='?', 115 | help="Path to CA certificates for Elasticsearch" 116 | ) 117 | return parser 118 | 119 | 120 | def run_tests(*test_args): 121 | args, test_args = make_parser().parse_known_args(test_args) 122 | if args.elasticsearch: 123 | os.environ.setdefault('ELASTICSEARCH_URL', "https://127.0.0.1:9200") 124 | 125 | username = args.elasticsearch_username or "elastic" 126 | password = args.elasticsearch_password or "changeme" 127 | os.environ.setdefault( 128 | 'ELASTICSEARCH_USERNAME', username 129 | ) 130 | os.environ.setdefault( 131 | 'ELASTICSEARCH_PASSWORD', password 132 | ) 133 | 134 | if args.elasticsearch_certs_path: 135 | os.environ.setdefault( 136 | 'ELASTICSEARCH_CERTS_PATH', args.elasticsearch_certs_path 137 | ) 138 | 139 | if not test_args: 140 | test_args = ['tests'] 141 | 142 | signal_processor = args.signal_processor 143 | 144 | settings = get_settings(signal_processor) 145 | TestRunner = get_runner(settings) 146 | test_runner = TestRunner() 147 | 148 | failures = test_runner.run_tests(test_args) 149 | 150 | if failures: 151 | sys.exit(bool(failures)) 152 | 153 | 154 | if __name__ == '__main__': 155 | run_tests(*sys.argv[1:]) 156 | -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [bumpversion] 2 | current_version = 7.1.1 3 | commit = True 4 | tag = True 5 | 6 | [bumpversion:file:setup.py] 7 | 8 | [bumpversion:file:django_elasticsearch_dsl/__init__.py] 9 | 10 | [bumpversion:file:docs/source/conf.py] 11 | 12 | [wheel] 13 | universal = 1 14 | 15 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | import os 4 | import sys 5 | 6 | try: 7 | from setuptools import setup 8 | except ImportError: 9 | from distutils.core import setup 10 | 11 | version = '8.0' 12 | 13 | if sys.argv[-1] == 'publish': 14 | try: 15 | import wheel 16 | print("Wheel version: ", wheel.__version__) 17 | except ImportError: 18 | print('Wheel library missing. Please run "pip install wheel"') 19 | sys.exit() 20 | os.system('python setup.py sdist upload') 21 | os.system('python setup.py bdist_wheel upload') 22 | sys.exit() 23 | 24 | if sys.argv[-1] == 'tag': 25 | print("Tagging the version on git:") 26 | os.system("git tag -a %s -m 'version %s'" % (version, version)) 27 | os.system("git push --tags") 28 | sys.exit() 29 | 30 | readme = open('README.rst').read() 31 | history = open('HISTORY.rst').read().replace('.. :changelog:', '') 32 | 33 | setup( 34 | name='django-elasticsearch-dsl', 35 | version=version, 36 | python_requires=">=3.8", 37 | description="""Wrapper around elasticsearch-dsl-py for django models""", 38 | long_description=readme + '\n\n' + history, 39 | author='Sabricot', 40 | url='https://github.com/sabricot/django-elasticsearch-dsl', 41 | packages=[ 42 | 'django_elasticsearch_dsl', 43 | ], 44 | include_package_data=True, 45 | install_requires=[ 46 | 'elasticsearch-dsl>=8.9.0,<9.0.0', 47 | 'six', 48 | ], 49 | license="Apache Software License 2.0", 50 | zip_safe=False, 51 | keywords='django elasticsearch elasticsearch-dsl', 52 | classifiers=[ 53 | 'Development Status :: 3 - Alpha', 54 | 'Framework :: Django', 55 | 'Framework :: Django :: 3.2', 56 | 'Framework :: Django :: 4.1', 57 | 'Framework :: Django :: 4.2', 58 | 'Intended Audience :: Developers', 59 | 'License :: OSI Approved :: BSD License', 60 | 'Natural Language :: English', 61 | 'Programming Language :: Python :: 3', 62 | 'Programming Language :: Python :: 3.8', 63 | 'Programming Language :: Python :: 3.9', 64 | 'Programming Language :: Python :: 3.10', 65 | 'Programming Language :: Python :: 3.11', 66 | ], 67 | extras_require={ 68 | 'celery': ["celery>=4.1.0"], 69 | } 70 | ) 71 | -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- 1 | from elasticsearch_dsl import VERSION 2 | 3 | ES_MAJOR_VERSION = VERSION[0] 4 | -------------------------------------------------------------------------------- /tests/documents.py: -------------------------------------------------------------------------------- 1 | from elasticsearch_dsl import analyzer 2 | from django_elasticsearch_dsl import Document, fields 3 | from django_elasticsearch_dsl.registries import registry 4 | 5 | from .models import Ad, Category, Car, Manufacturer, Article 6 | 7 | index_settings = { 8 | 'number_of_shards': 1, 9 | 'number_of_replicas': 0, 10 | } 11 | 12 | 13 | html_strip = analyzer( 14 | 'html_strip', 15 | tokenizer="standard", 16 | filter=["lowercase", "stop", "snowball"], 17 | char_filter=["html_strip"] 18 | ) 19 | 20 | 21 | @registry.register_document 22 | class CarDocument(Document): 23 | # test can override __init__ 24 | def __init__(self, *args, **kwargs): 25 | super(CarDocument, self).__init__(*args, **kwargs) 26 | 27 | manufacturer = fields.ObjectField(properties={ 28 | 'name': fields.TextField(), 29 | 'country': fields.TextField(), 30 | }) 31 | 32 | ads = fields.NestedField(properties={ 33 | 'description': fields.TextField(analyzer=html_strip), 34 | 'title': fields.TextField(), 35 | 'pk': fields.IntegerField(), 36 | }) 37 | 38 | categories = fields.NestedField(properties={ 39 | 'title': fields.TextField(), 40 | 'slug': fields.TextField(), 41 | 'icon': fields.FileField(), 42 | }) 43 | 44 | class Django: 45 | model = Car 46 | related_models = [Ad, Manufacturer, Category] 47 | fields = [ 48 | 'name', 49 | 'launched', 50 | 'type', 51 | ] 52 | 53 | class Index: 54 | name = 'test_cars' 55 | settings = index_settings 56 | 57 | def get_queryset(self): 58 | return super(CarDocument, self).get_queryset().select_related( 59 | 'manufacturer') 60 | 61 | def get_instances_from_related(self, related_instance): 62 | if isinstance(related_instance, Ad): 63 | return related_instance.car 64 | 65 | # otherwise it's a Manufacturer or a Category 66 | return related_instance.car_set.all() 67 | 68 | 69 | @registry.register_document 70 | class ManufacturerDocument(Document): 71 | country = fields.TextField() 72 | 73 | class Django: 74 | model = Manufacturer 75 | fields = [ 76 | 'name', 77 | 'created', 78 | 'country_code', 79 | 'logo', 80 | ] 81 | 82 | class Index: 83 | name = 'index_settings' 84 | settings = index_settings 85 | 86 | 87 | @registry.register_document 88 | class CarWithPrepareDocument(Document): 89 | manufacturer = fields.ObjectField(properties={ 90 | 'name': fields.TextField(), 91 | 'country': fields.TextField(), 92 | }) 93 | 94 | manufacturer_short = fields.ObjectField(properties={ 95 | 'name': fields.TextField(), 96 | }) 97 | 98 | class Django: 99 | model = Car 100 | related_models = [Manufacturer] 101 | fields = [ 102 | 'name', 103 | 'launched', 104 | 'type', 105 | ] 106 | 107 | class Index: 108 | name = 'car_with_prepare_index' 109 | 110 | def prepare_manufacturer_with_related(self, car, related_to_ignore): 111 | if (car.manufacturer is not None and car.manufacturer != 112 | related_to_ignore): 113 | return { 114 | 'name': car.manufacturer.name, 115 | 'country': car.manufacturer.country(), 116 | } 117 | return {} 118 | 119 | def prepare_manufacturer_short(self, car): 120 | if car.manufacturer is not None: 121 | return { 122 | 'name': car.manufacturer.name, 123 | } 124 | return {} 125 | 126 | def get_instances_from_related(self, related_instance): 127 | return related_instance.car_set.all() 128 | 129 | 130 | @registry.register_document 131 | class AdDocument(Document): 132 | description = fields.TextField( 133 | analyzer=html_strip, 134 | fields={'raw': fields.KeywordField()} 135 | ) 136 | 137 | class Django: 138 | model = Ad 139 | fields = [ 140 | 'title', 141 | 'created', 142 | 'modified', 143 | 'url', 144 | ] 145 | 146 | class Index: 147 | name = 'test_ads' 148 | settings = index_settings 149 | 150 | 151 | @registry.register_document 152 | class ArticleDocument(Document): 153 | class Django: 154 | model = Article 155 | fields = [ 156 | 'slug', 157 | ] 158 | 159 | class Index: 160 | name = 'test_articles' 161 | settings = index_settings 162 | 163 | 164 | @registry.register_document 165 | class ArticleWithSlugAsIdDocument(Document): 166 | class Django: 167 | model = Article 168 | fields = [ 169 | 'slug', 170 | ] 171 | 172 | class Index: 173 | name = 'test_articles_with_slugs_as_doc_ids' 174 | settings = index_settings 175 | 176 | @classmethod 177 | def generate_id(cls, article): 178 | return article.slug 179 | 180 | 181 | ad_index = AdDocument._index 182 | car_index = CarDocument._index 183 | -------------------------------------------------------------------------------- /tests/fixtures.py: -------------------------------------------------------------------------------- 1 | from mock import Mock 2 | 3 | from django.db import models 4 | 5 | from django_elasticsearch_dsl.documents import DocType 6 | 7 | 8 | class WithFixturesMixin(object): 9 | 10 | class ModelA(models.Model): 11 | class Meta: 12 | app_label = 'foo' 13 | 14 | class ModelB(models.Model): 15 | class Meta: 16 | app_label = 'foo' 17 | 18 | class ModelC(models.Model): 19 | class Meta: 20 | app_label = 'bar' 21 | 22 | class ModelD(models.Model): 23 | pass 24 | 25 | class ModelE(models.Model): 26 | pass 27 | 28 | def _generate_doc_mock(self, _model, index=None, mock_qs=None, 29 | _ignore_signals=False, _related_models=None): 30 | _index = index 31 | 32 | class Doc(DocType): 33 | 34 | class Django: 35 | model = _model 36 | related_models = _related_models if _related_models is not None else [] 37 | ignore_signals = _ignore_signals 38 | 39 | if _index: 40 | _index.document(Doc) 41 | self.registry.register_document(Doc) 42 | 43 | Doc.update = Mock() 44 | if mock_qs: 45 | Doc.get_queryset = Mock(return_value=mock_qs) 46 | if _related_models: 47 | Doc.get_instances_from_related = Mock() 48 | 49 | return Doc 50 | -------------------------------------------------------------------------------- /tests/models.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | from __future__ import unicode_literals 3 | 4 | import django 5 | from django.db import models 6 | if django.VERSION < (4, 0): 7 | from django.utils.translation import ugettext_lazy as _ 8 | else: 9 | from django.utils.translation import gettext_lazy as _ 10 | from six import python_2_unicode_compatible 11 | 12 | 13 | @python_2_unicode_compatible 14 | class Car(models.Model): 15 | TYPE_CHOICES = ( 16 | ('se', "Sedan"), 17 | ('br', "Break"), 18 | ('4x', "4x4"), 19 | ('co', "Coupé"), 20 | ) 21 | 22 | name = models.CharField(max_length=255) 23 | launched = models.DateField() 24 | type = models.CharField( 25 | max_length=2, 26 | choices=TYPE_CHOICES, 27 | default='se', 28 | ) 29 | manufacturer = models.ForeignKey( 30 | 'Manufacturer', null=True, on_delete=models.SET_NULL 31 | ) 32 | categories = models.ManyToManyField('Category') 33 | 34 | class Meta: 35 | app_label = 'tests' 36 | 37 | def __str__(self): 38 | return self.name 39 | 40 | 41 | COUNTRIES = { 42 | 'FR': 'France', 43 | 'UK': 'United Kingdom', 44 | 'ES': 'Spain', 45 | 'IT': 'Italya', 46 | } 47 | 48 | 49 | @python_2_unicode_compatible 50 | class Manufacturer(models.Model): 51 | name = models.CharField(max_length=255, default=_("Test lazy tanslation")) 52 | country_code = models.CharField(max_length=2) 53 | created = models.DateField() 54 | logo = models.ImageField(blank=True) 55 | 56 | class meta: 57 | app_label = 'tests' 58 | 59 | def country(self): 60 | return COUNTRIES.get(self.country_code, self.country_code) 61 | 62 | def __str__(self): 63 | return self.name 64 | 65 | 66 | @python_2_unicode_compatible 67 | class Category(models.Model): 68 | title = models.CharField(max_length=255) 69 | slug = models.CharField(max_length=255) 70 | icon = models.ImageField(blank=True) 71 | 72 | class Meta: 73 | app_label = 'tests' 74 | 75 | def __str__(self): 76 | return self.title 77 | 78 | 79 | @python_2_unicode_compatible 80 | class Ad(models.Model): 81 | title = models.CharField(max_length=255) 82 | description = models.TextField() 83 | created = models.DateField(auto_now_add=True) 84 | modified = models.DateField(auto_now=True) 85 | url = models.URLField() 86 | car = models.ForeignKey( 87 | 'Car', related_name='ads', null=True, on_delete=models.SET_NULL 88 | ) 89 | 90 | class Meta: 91 | app_label = 'tests' 92 | 93 | def __str__(self): 94 | return self.title 95 | 96 | 97 | @python_2_unicode_compatible 98 | class Article(models.Model): 99 | slug = models.CharField( 100 | max_length=255, 101 | unique=True, 102 | ) 103 | 104 | class Meta: 105 | app_label = 'tests' 106 | 107 | def __str__(self): 108 | return self.slug 109 | -------------------------------------------------------------------------------- /tests/test_commands.py: -------------------------------------------------------------------------------- 1 | from mock import DEFAULT, Mock, patch 2 | from unittest import TestCase 3 | 4 | from django.core.management.base import CommandError 5 | from django.core.management import call_command 6 | from six import StringIO 7 | 8 | from django_elasticsearch_dsl import Index 9 | from django_elasticsearch_dsl.management.commands.search_index import Command 10 | from django_elasticsearch_dsl.registries import DocumentRegistry 11 | 12 | from .fixtures import WithFixturesMixin 13 | 14 | 15 | class SearchIndexTestCase(WithFixturesMixin, TestCase): 16 | def _mock_setup(self): 17 | # Mock Patch object 18 | patch_registry = patch( 19 | 'django_elasticsearch_dsl.management.commands.search_index.registry', self.registry) 20 | 21 | patch_registry.start() 22 | 23 | methods = ['delete', 'create'] 24 | for index in [self.index_a, self.index_b]: 25 | for method in methods: 26 | obj_patch = patch.object(index, method) 27 | obj_patch.start() 28 | 29 | self.addCleanup(patch.stopall) 30 | 31 | def setUp(self): 32 | self.out = StringIO() 33 | self.registry = DocumentRegistry() 34 | self.index_a = Index('foo') 35 | self.index_b = Index('bar') 36 | 37 | self.doc_a1_qs = Mock() 38 | self.doc_a1 = self._generate_doc_mock( 39 | self.ModelA, self.index_a, self.doc_a1_qs 40 | ) 41 | 42 | self.doc_a2_qs = Mock() 43 | self.doc_a2 = self._generate_doc_mock( 44 | self.ModelA, self.index_a, self.doc_a2_qs 45 | ) 46 | 47 | self.doc_b1_qs = Mock() 48 | self.doc_b1 = self._generate_doc_mock( 49 | self.ModelB, self.index_a, self.doc_b1_qs 50 | ) 51 | 52 | self.doc_c1_qs = Mock() 53 | self.doc_c1 = self._generate_doc_mock( 54 | self.ModelC, self.index_b, self.doc_c1_qs 55 | ) 56 | 57 | self._mock_setup() 58 | 59 | def test_get_models(self): 60 | cmd = Command() 61 | self.assertEqual( 62 | cmd._get_models(['foo']), 63 | set([self.ModelA, self.ModelB]) 64 | ) 65 | 66 | self.assertEqual( 67 | cmd._get_models(['foo', 'bar.ModelC']), 68 | set([self.ModelA, self.ModelB, self.ModelC]) 69 | ) 70 | 71 | self.assertEqual( 72 | cmd._get_models([]), 73 | set([self.ModelA, self.ModelB, self.ModelC]) 74 | ) 75 | with self.assertRaises(CommandError): 76 | cmd._get_models(['unknown']) 77 | 78 | def test_no_action_error(self): 79 | cmd = Command() 80 | with self.assertRaises(CommandError): 81 | cmd.handle(action="") 82 | 83 | def test_delete_foo_index(self): 84 | 85 | with patch( 86 | 'django_elasticsearch_dsl.management.commands.search_index.input', 87 | Mock(return_value="y") 88 | ): 89 | call_command('search_index', stdout=self.out, 90 | action='delete', models=['foo']) 91 | self.index_a.delete.assert_called_once() 92 | self.assertFalse(self.index_b.delete.called) 93 | 94 | def test_force_delete_all_indices(self): 95 | 96 | call_command('search_index', stdout=self.out, 97 | action='delete', force=True) 98 | self.index_a.delete.assert_called_once() 99 | self.index_b.delete.assert_called_once() 100 | 101 | def test_force_delete_bar_model_c_index(self): 102 | call_command('search_index', stdout=self.out, 103 | models=[self.ModelC._meta.label], 104 | action='delete', force=True) 105 | self.index_b.delete.assert_called_once() 106 | self.assertFalse(self.index_a.delete.called) 107 | 108 | def test_create_all_indices(self): 109 | call_command('search_index', stdout=self.out, action='create') 110 | self.index_a.create.assert_called_once() 111 | self.index_b.create.assert_called_once() 112 | 113 | def test_populate_all_doc_type(self): 114 | call_command('search_index', stdout=self.out, action='populate') 115 | expected_kwargs = {'parallel': False, 'refresh': None} 116 | # One call for "Indexing NNN documents", one for indexing itself (via get_index_queryset). 117 | assert self.doc_a1.get_queryset.call_count == 2 118 | self.doc_a1.update.assert_called_once_with(self.doc_a1_qs.iterator(), **expected_kwargs) 119 | assert self.doc_a2.get_queryset.call_count == 2 120 | self.doc_a2.update.assert_called_once_with(self.doc_a2_qs.iterator(), **expected_kwargs) 121 | assert self.doc_b1.get_queryset.call_count == 2 122 | self.doc_b1.update.assert_called_once_with(self.doc_b1_qs.iterator(), **expected_kwargs) 123 | assert self.doc_c1.get_queryset.call_count == 2 124 | self.doc_c1.update.assert_called_once_with(self.doc_c1_qs.iterator(), **expected_kwargs) 125 | 126 | def test_populate_all_doc_type_refresh(self): 127 | call_command('search_index', stdout=self.out, action='populate', refresh=True) 128 | expected_kwargs = {'parallel': False, 'refresh': True} 129 | self.doc_a1.update.assert_called_once_with(self.doc_a1_qs.iterator(), **expected_kwargs) 130 | self.doc_a2.update.assert_called_once_with(self.doc_a2_qs.iterator(), **expected_kwargs) 131 | self.doc_b1.update.assert_called_once_with(self.doc_b1_qs.iterator(), **expected_kwargs) 132 | self.doc_c1.update.assert_called_once_with(self.doc_c1_qs.iterator(), **expected_kwargs) 133 | 134 | def test_rebuild_indices(self): 135 | 136 | with patch.multiple( 137 | Command, _create=DEFAULT, _delete=DEFAULT, _populate=DEFAULT 138 | ) as handles: 139 | handles['_delete'].return_value = True 140 | call_command('search_index', stdout=self.out, action='rebuild') 141 | handles['_delete'].assert_called() 142 | handles['_create'].assert_called() 143 | handles['_populate'].assert_called() 144 | 145 | def test_rebuild_indices_aborted(self): 146 | 147 | with patch.multiple( 148 | Command, _create=DEFAULT, _delete=DEFAULT, _populate=DEFAULT 149 | ) as handles: 150 | handles['_delete'].return_value = False 151 | call_command('search_index', stdout=self.out, action='rebuild') 152 | handles['_delete'].assert_called() 153 | handles['_create'].assert_not_called() 154 | handles['_populate'].assert_not_called() 155 | -------------------------------------------------------------------------------- /tests/test_fields.py: -------------------------------------------------------------------------------- 1 | from unittest import TestCase 2 | 3 | import django 4 | from django.db.models.fields.files import FieldFile 5 | if django.VERSION < (4, 0): 6 | from django.utils.translation import ugettext_lazy as _ 7 | else: 8 | from django.utils.translation import gettext_lazy as _ 9 | from mock import Mock, NonCallableMock 10 | from six import string_types 11 | 12 | from django_elasticsearch_dsl.exceptions import VariableLookupError 13 | from django_elasticsearch_dsl.fields import (BooleanField, ByteField, CompletionField, DEDField, 14 | DateField, DoubleField, FileField, FloatField, 15 | GeoPointField, 16 | GeoShapeField, IntegerField, IpField, KeywordField, 17 | ListField, LongField, 18 | NestedField, ObjectField, ScaledFloatField, ShortField, TextField 19 | ) 20 | from tests import ES_MAJOR_VERSION 21 | 22 | 23 | class DEDFieldTestCase(TestCase): 24 | def test_attr_to_path(self): 25 | field = DEDField(attr='field') 26 | self.assertEqual(field._path, ['field']) 27 | 28 | field = DEDField(attr='obj.field') 29 | self.assertEqual(field._path, ['obj', 'field']) 30 | 31 | def test_get_value_from_instance_attr(self): 32 | field = DEDField(attr='attr1') 33 | instance = NonCallableMock(attr1="foo", attr2="bar") 34 | self.assertEqual(field.get_value_from_instance(instance), "foo") 35 | 36 | def test_get_value_from_instance_related_attr(self): 37 | field = DEDField(attr='related.attr1') 38 | instance = NonCallableMock(attr1="foo", 39 | related=NonCallableMock(attr1="bar")) 40 | self.assertEqual(field.get_value_from_instance(instance), "bar") 41 | 42 | def test_get_value_from_instance_callable(self): 43 | field = DEDField(attr='callable') 44 | instance = NonCallableMock(callable=Mock(return_value="bar")) 45 | self.assertEqual(field.get_value_from_instance(instance), "bar") 46 | 47 | def test_get_value_from_instance_related_callable(self): 48 | field = DEDField(attr='related.callable') 49 | instance = NonCallableMock(related=NonCallableMock( 50 | callable=Mock(return_value="bar"), attr1="foo")) 51 | self.assertEqual(field.get_value_from_instance(instance), "bar") 52 | 53 | def test_get_value_from_instance_with_unknown_attr(self): 54 | class Dummy: 55 | attr1 = "foo" 56 | 57 | field = DEDField(attr='attr2', required=True) 58 | self.assertRaises( 59 | VariableLookupError, field.get_value_from_instance, Dummy() 60 | ) 61 | 62 | def test_get_value_from_none(self): 63 | field = DEDField(attr='related.none') 64 | instance = NonCallableMock(attr1="foo", related=None) 65 | self.assertEqual(field.get_value_from_instance(instance), None) 66 | 67 | def test_get_value_from_lazy_object(self): 68 | field = DEDField(attr='translation') 69 | instance = NonCallableMock(translation=_("foo")) 70 | self.assertIsInstance( 71 | field.get_value_from_instance(instance), string_types 72 | ) 73 | self.assertEqual(field.get_value_from_instance(instance), "foo") 74 | 75 | 76 | class ObjectFieldTestCase(TestCase): 77 | def test_get_mapping(self): 78 | field = ObjectField(attr='person', properties={ 79 | 'first_name': TextField(analyzer='foo'), 80 | 'last_name': TextField() 81 | }) 82 | 83 | expected_type = 'string' if ES_MAJOR_VERSION == 2 else 'text' 84 | 85 | self.assertEqual({ 86 | 'type': 'object', 87 | 'properties': { 88 | 'first_name': {'type': expected_type, 'analyzer': 'foo'}, 89 | 'last_name': {'type': expected_type}, 90 | } 91 | }, field.to_dict()) 92 | 93 | def test_get_value_from_instance(self): 94 | field = ObjectField(attr='person', properties={ 95 | 'first_name': TextField(analyzer='foo'), 96 | 'last_name': TextField() 97 | }) 98 | 99 | instance = NonCallableMock(person=NonCallableMock( 100 | first_name='foo', last_name='bar')) 101 | 102 | self.assertEqual(field.get_value_from_instance(instance), { 103 | 'first_name': "foo", 104 | 'last_name': "bar", 105 | }) 106 | 107 | def test_get_value_from_instance_with_partial_properties(self): 108 | field = ObjectField( 109 | attr='person', 110 | properties={ 111 | 'first_name': TextField(analyzer='foo') 112 | } 113 | ) 114 | 115 | instance = NonCallableMock( 116 | person=NonCallableMock(first_name='foo', last_name='bar') 117 | ) 118 | 119 | self.assertEqual(field.get_value_from_instance(instance), { 120 | 'first_name': "foo" 121 | }) 122 | 123 | def test_get_value_from_instance_without_properties(self): 124 | field = ObjectField(attr='person') 125 | 126 | instance = NonCallableMock( 127 | person={'first_name': 'foo', 'last_name': 'bar'} 128 | ) 129 | 130 | self.assertEqual(field.get_value_from_instance(instance), 131 | { 132 | 'first_name': "foo", 133 | 'last_name': "bar" 134 | } 135 | ) 136 | 137 | def test_get_value_from_instance_with_inner_objectfield(self): 138 | field = ObjectField(attr='person', properties={ 139 | 'first_name': TextField(analyzer='foo'), 140 | 'last_name': TextField(), 141 | 'additional': ObjectField(properties={ 142 | 'age': IntegerField() 143 | }) 144 | }) 145 | 146 | instance = NonCallableMock(person=NonCallableMock( 147 | first_name="foo", last_name="bar", 148 | additional=NonCallableMock(age=12) 149 | )) 150 | 151 | self.assertEqual(field.get_value_from_instance(instance), { 152 | 'first_name': "foo", 153 | 'last_name': "bar", 154 | 'additional': {'age': 12} 155 | }) 156 | 157 | def test_get_value_from_instance_with_inner_objectfield_without_properties(self): 158 | field = ObjectField( 159 | attr='person', 160 | properties={ 161 | 'first_name': TextField(analyzer='foo'), 162 | 'last_name': TextField(), 163 | 'additional': ObjectField() 164 | } 165 | ) 166 | 167 | instance = NonCallableMock(person=NonCallableMock( 168 | first_name="foo", 169 | last_name="bar", 170 | additional={'age': 12} 171 | )) 172 | 173 | self.assertEqual(field.get_value_from_instance(instance), 174 | { 175 | 'first_name': "foo", 176 | 'last_name': "bar", 177 | 'additional': {'age': 12} 178 | } 179 | ) 180 | 181 | def test_get_value_from_instance_with_none_inner_objectfield(self): 182 | field = ObjectField(attr='person', properties={ 183 | 'first_name': TextField(analyzer='foo'), 184 | 'last_name': TextField(), 185 | 'additional': ObjectField(properties={ 186 | 'age': IntegerField() 187 | }) 188 | }) 189 | 190 | instance = NonCallableMock(person=NonCallableMock( 191 | first_name="foo", last_name="bar", 192 | additional=None 193 | )) 194 | 195 | self.assertEqual(field.get_value_from_instance(instance), { 196 | 'first_name': "foo", 197 | 'last_name': "bar", 198 | 'additional': {} 199 | }) 200 | 201 | def test_get_value_from_iterable(self): 202 | field = ObjectField(attr='person', properties={ 203 | 'first_name': TextField(analyzer='foo'), 204 | 'last_name': TextField() 205 | }) 206 | 207 | instance = NonCallableMock( 208 | person=[ 209 | NonCallableMock( 210 | first_name="foo1", last_name="bar1" 211 | ), 212 | NonCallableMock( 213 | first_name="foo2", last_name="bar2" 214 | ) 215 | ] 216 | ) 217 | 218 | self.assertEqual(field.get_value_from_instance(instance), [ 219 | { 220 | 'first_name': "foo1", 221 | 'last_name': "bar1", 222 | }, 223 | { 224 | 'first_name': "foo2", 225 | 'last_name': "bar2", 226 | } 227 | ]) 228 | 229 | def test_get_value_from_iterable_without_properties(self): 230 | field = ObjectField(attr='person') 231 | 232 | instance = NonCallableMock( 233 | person=[ 234 | {'first_name': "foo1", 'last_name': "bar1"}, 235 | {'first_name': "foo2", 'last_name': "bar2"} 236 | ] 237 | ) 238 | 239 | self.assertEqual(field.get_value_from_instance(instance), 240 | [ 241 | { 242 | 'first_name': "foo1", 243 | 'last_name': "bar1", 244 | }, 245 | { 246 | 'first_name': "foo2", 247 | 'last_name': "bar2", 248 | } 249 | ] 250 | ) 251 | 252 | 253 | class NestedFieldTestCase(TestCase): 254 | def test_get_mapping(self): 255 | field = NestedField(attr='person', properties={ 256 | 'first_name': TextField(analyzer='foo'), 257 | 'last_name': TextField() 258 | }) 259 | 260 | expected_type = 'string' if ES_MAJOR_VERSION == 2 else 'text' 261 | 262 | self.assertEqual({ 263 | 'type': 'nested', 264 | 'properties': { 265 | 'first_name': {'type': expected_type, 'analyzer': 'foo'}, 266 | 'last_name': {'type': expected_type}, 267 | } 268 | }, field.to_dict()) 269 | 270 | 271 | class BooleanFieldTestCase(TestCase): 272 | def test_get_mapping(self): 273 | field = BooleanField() 274 | 275 | self.assertEqual({ 276 | 'type': 'boolean', 277 | }, field.to_dict()) 278 | 279 | 280 | class DateFieldTestCase(TestCase): 281 | def test_get_mapping(self): 282 | field = DateField() 283 | 284 | self.assertEqual({ 285 | 'type': 'date', 286 | }, field.to_dict()) 287 | 288 | 289 | class CompletionFieldTestCase(TestCase): 290 | def test_get_mapping(self): 291 | field = CompletionField() 292 | 293 | self.assertEqual({ 294 | 'type': 'completion', 295 | }, field.to_dict()) 296 | 297 | 298 | class GeoPointFieldTestCase(TestCase): 299 | def test_get_mapping(self): 300 | field = GeoPointField() 301 | 302 | self.assertEqual({ 303 | 'type': 'geo_point', 304 | }, field.to_dict()) 305 | 306 | 307 | class GeoShapeFieldTestCase(TestCase): 308 | def test_get_mapping(self): 309 | field = GeoShapeField() 310 | 311 | self.assertEqual({ 312 | 'type': 'geo_shape' 313 | }, field.to_dict()) 314 | 315 | 316 | class ByteFieldTestCase(TestCase): 317 | def test_get_mapping(self): 318 | field = ByteField() 319 | 320 | self.assertEqual({ 321 | 'type': 'byte', 322 | }, field.to_dict()) 323 | 324 | 325 | class LongFieldTestCase(TestCase): 326 | def test_get_mapping(self): 327 | field = LongField() 328 | 329 | self.assertEqual({ 330 | 'type': 'long', 331 | }, field.to_dict()) 332 | 333 | 334 | class DoubleFieldTestCase(TestCase): 335 | def test_get_mapping(self): 336 | field = DoubleField() 337 | 338 | self.assertEqual({ 339 | 'type': 'double', 340 | }, field.to_dict()) 341 | 342 | 343 | class FloatFieldTestCase(TestCase): 344 | def test_get_mapping(self): 345 | field = FloatField() 346 | 347 | self.assertEqual({ 348 | 'type': 'float', 349 | }, field.to_dict()) 350 | 351 | 352 | class ScaledFloatFieldTestCase(TestCase): 353 | def test_get_mapping(self): 354 | field = ScaledFloatField(scaling_factor=100) 355 | 356 | self.assertEqual({ 357 | 'type': 'scaled_float', 358 | 'scaling_factor': 100, 359 | }, field.to_dict()) 360 | 361 | 362 | class IpFieldTestCase(TestCase): 363 | def test_get_mapping(self): 364 | field = IpField() 365 | 366 | self.assertEqual({ 367 | 'type': 'ip', 368 | }, field.to_dict()) 369 | 370 | 371 | class ListFieldTestCase(TestCase): 372 | def test_get_mapping(self): 373 | field = ListField(IntegerField(attr='foo.bar')) 374 | self.assertEqual({ 375 | 'type': 'integer', 376 | }, field.to_dict()) 377 | 378 | def test_get_value_from_instance(self): 379 | instance = NonCallableMock( 380 | foo=NonCallableMock(bar=["alpha", "beta", "gamma"]) 381 | ) 382 | field = ListField(TextField(attr='foo.bar')) 383 | self.assertEqual( 384 | field.get_value_from_instance(instance), instance.foo.bar) 385 | 386 | 387 | class ShortFieldTestCase(TestCase): 388 | def test_get_mapping(self): 389 | field = ShortField() 390 | 391 | self.assertEqual({ 392 | 'type': 'short', 393 | }, field.to_dict()) 394 | 395 | 396 | class FileFieldTestCase(TestCase): 397 | def test_get_mapping(self): 398 | field = FileField() 399 | 400 | expected_type = 'string' if ES_MAJOR_VERSION == 2 else 'text' 401 | 402 | self.assertEqual({ 403 | 'type': expected_type, 404 | }, field.to_dict()) 405 | 406 | def test_get_value_from_instance(self): 407 | field = FileField(attr='file') 408 | 409 | instance = NonCallableMock( 410 | file=NonCallableMock(spec=FieldFile, url='myfile.pdf'), 411 | ) 412 | self.assertEqual( 413 | field.get_value_from_instance(instance), 'myfile.pdf' 414 | ) 415 | 416 | field = FileField(attr='related.attr1') 417 | instance = NonCallableMock( 418 | attr1="foo", related=NonCallableMock(attr1="bar") 419 | ) 420 | self.assertEqual(field.get_value_from_instance(instance), 'bar') 421 | 422 | 423 | class TextFieldTestCase(TestCase): 424 | def test_get_mapping(self): 425 | field = TextField() 426 | 427 | expected_type = 'string' if ES_MAJOR_VERSION == 2 else 'text' 428 | 429 | self.assertEqual({ 430 | 'type': expected_type, 431 | }, field.to_dict()) 432 | 433 | 434 | class KeywordFieldTestCase(TestCase): 435 | def test_get_mapping(self): 436 | field = KeywordField() 437 | 438 | if ES_MAJOR_VERSION == 2: 439 | self.assertEqual({ 440 | 'type': 'string', 441 | 'index': 'not_analyzed', 442 | }, field.to_dict()) 443 | else: 444 | self.assertEqual({ 445 | 'type': 'keyword', 446 | }, field.to_dict()) 447 | -------------------------------------------------------------------------------- /tests/test_indices.py: -------------------------------------------------------------------------------- 1 | from unittest import TestCase 2 | from mock import patch 3 | 4 | from django.conf import settings 5 | 6 | from django_elasticsearch_dsl.indices import Index 7 | from django_elasticsearch_dsl.registries import DocumentRegistry 8 | 9 | from .fixtures import WithFixturesMixin 10 | 11 | 12 | class IndexTestCase(WithFixturesMixin, TestCase): 13 | def setUp(self): 14 | self.registry = DocumentRegistry() 15 | 16 | def test_documents_add_to_register(self): 17 | registry = self.registry 18 | with patch('django_elasticsearch_dsl.indices.registry', new=registry): 19 | index = Index('test') 20 | doc_a1 = self._generate_doc_mock(self.ModelA) 21 | doc_a2 = self._generate_doc_mock(self.ModelA) 22 | index.document(doc_a1) 23 | docs = list(registry.get_documents()) 24 | self.assertEqual(len(docs), 1) 25 | self.assertIs(docs[0], doc_a1) 26 | 27 | index.document(doc_a2) 28 | docs = registry.get_documents() 29 | self.assertEqual(docs, set([doc_a1, doc_a2])) 30 | 31 | def test__str__(self): 32 | index = Index('test') 33 | self.assertEqual(index.__str__(), 'test') 34 | 35 | def test__init__(self): 36 | settings.ELASTICSEARCH_DSL_INDEX_SETTINGS = { 37 | 'number_of_replicas': 0, 38 | 'number_of_shards': 2, 39 | } 40 | 41 | index = Index('test') 42 | self.assertEqual(index._settings, { 43 | 'number_of_replicas': 0, 44 | 'number_of_shards': 2, 45 | }) 46 | 47 | settings.ELASTICSEARCH_DSL_INDEX_SETTINGS = {} 48 | -------------------------------------------------------------------------------- /tests/test_integration.py: -------------------------------------------------------------------------------- 1 | from datetime import datetime 2 | import unittest 3 | 4 | import django 5 | from django.core.management import call_command 6 | from django.test import TestCase, TransactionTestCase 7 | if django.VERSION < (4, 0): 8 | from django.utils.translation import ugettext_lazy as _ 9 | else: 10 | from django.utils.translation import gettext_lazy as _ 11 | from six import StringIO 12 | 13 | from elasticsearch.exceptions import NotFoundError 14 | from elasticsearch_dsl import Index as DSLIndex 15 | from django_elasticsearch_dsl.test import ESTestCase, is_es_online 16 | from tests import ES_MAJOR_VERSION 17 | 18 | from .documents import ( 19 | ad_index, 20 | AdDocument, 21 | car_index, 22 | CarDocument, 23 | CarWithPrepareDocument, 24 | ArticleDocument, 25 | ArticleWithSlugAsIdDocument, 26 | index_settings 27 | ) 28 | from .models import Car, Manufacturer, Ad, Category, Article, COUNTRIES 29 | 30 | 31 | @unittest.skipUnless(is_es_online(), 'Elasticsearch is offline') 32 | class IntegrationTestCase(ESTestCase, TransactionTestCase): 33 | def setUp(self): 34 | super(IntegrationTestCase, self).setUp() 35 | self.manufacturer = Manufacturer( 36 | name="Peugeot", created=datetime(1900, 10, 9, 0, 0), 37 | country_code="FR", logo='logo.jpg' 38 | ) 39 | self.manufacturer.save() 40 | self.car1 = Car( 41 | name="508", launched=datetime(2010, 9, 9, 0, 0), 42 | manufacturer=self.manufacturer 43 | ) 44 | 45 | self.car1.save() 46 | self.car2 = Car( 47 | name="208", launched=datetime(2010, 10, 9, 0, 0), 48 | manufacturer=self.manufacturer 49 | ) 50 | self.car2.save() 51 | self.category1 = Category( 52 | title="Category 1", slug="category-1", icon="icon.jpeg" 53 | ) 54 | self.category1.save() 55 | self.car2.categories.add(self.category1) 56 | self.car2.save() 57 | 58 | self.car3 = Car(name="308", launched=datetime(2010, 11, 9, 0, 0)) 59 | self.car3.save() 60 | self.category2 = Category(title="Category 2", slug="category-2") 61 | self.category2.save() 62 | self.car3.categories.add(self.category1, self.category2) 63 | self.car3.save() 64 | 65 | self.ad1 = Ad( 66 | title=_("Ad number 1"), url="www.ad1.com", 67 | description="My super ad description 1", 68 | car=self.car1 69 | ) 70 | self.ad1.save() 71 | self.ad2 = Ad( 72 | title="Ad number 2", url="www.ad2.com", 73 | description="My super ad descriptio 2", 74 | car=self.car1 75 | ) 76 | self.ad2.save() 77 | self.car1.save() 78 | 79 | def test_get_doc_with_relationships(self): 80 | s = CarDocument.search().query("match", name=self.car2.name) 81 | result = s.execute() 82 | self.assertEqual(len(result), 1) 83 | car2_doc = result[0] 84 | self.assertEqual(car2_doc.ads, []) 85 | self.assertEqual(car2_doc.name, self.car2.name) 86 | self.assertEqual(int(car2_doc.meta.id), self.car2.pk) 87 | self.assertEqual(car2_doc.launched, self.car2.launched) 88 | self.assertEqual(car2_doc.manufacturer.name, 89 | self.car2.manufacturer.name) 90 | self.assertEqual(car2_doc.manufacturer.country, 91 | COUNTRIES[self.manufacturer.country_code]) 92 | 93 | s = CarDocument.search().query("match", name=self.car3.name) 94 | result = s.execute() 95 | car3_doc = result[0] 96 | self.assertEqual(car3_doc.manufacturer, {}) 97 | self.assertEqual(car3_doc.name, self.car3.name) 98 | self.assertEqual(int(car3_doc.meta.id), self.car3.pk) 99 | 100 | def test_get_doc_with_reverse_relationships(self): 101 | s = CarDocument.search().query("match", name=self.car1.name) 102 | result = s.execute() 103 | self.assertEqual(len(result), 1) 104 | car1_doc = result[0] 105 | self.assertEqual(car1_doc.ads, [ 106 | { 107 | 'title': self.ad1.title, 108 | 'description': self.ad1.description, 109 | 'pk': self.ad1.pk, 110 | }, 111 | { 112 | 'title': self.ad2.title, 113 | 'description': self.ad2.description, 114 | 'pk': self.ad2.pk, 115 | }, 116 | ]) 117 | self.assertEqual(car1_doc.name, self.car1.name) 118 | self.assertEqual(int(car1_doc.meta.id), self.car1.pk) 119 | 120 | def test_get_doc_with_many_to_many_relationships(self): 121 | s = CarDocument.search().query("match", name=self.car3.name) 122 | result = s.execute() 123 | self.assertEqual(len(result), 1) 124 | car1_doc = result[0] 125 | self.assertEqual(car1_doc.categories, [ 126 | { 127 | 'title': self.category1.title, 128 | 'slug': self.category1.slug, 129 | 'icon': self.category1.icon.url, 130 | }, 131 | { 132 | 'title': self.category2.title, 133 | 'slug': self.category2.slug, 134 | 'icon': '', 135 | } 136 | ]) 137 | 138 | def test_doc_to_dict(self): 139 | s = CarDocument.search().query("match", name=self.car2.name) 140 | result = s.execute() 141 | self.assertEqual(len(result), 1) 142 | car2_doc = result[0] 143 | self.assertEqual(car2_doc.to_dict(), { 144 | 'type': self.car2.type, 145 | 'launched': self.car2.launched, 146 | 'name': self.car2.name, 147 | 'manufacturer': { 148 | 'name': self.manufacturer.name, 149 | 'country': COUNTRIES[self.manufacturer.country_code], 150 | }, 151 | 'categories': [{ 152 | 'title': self.category1.title, 153 | 'slug': self.category1.slug, 154 | 'icon': self.category1.icon.url, 155 | }] 156 | }) 157 | 158 | s = CarDocument.search().query("match", name=self.car3.name) 159 | result = s.execute() 160 | self.assertEqual(len(result), 1) 161 | car3_doc = result[0] 162 | self.assertEqual(car3_doc.to_dict(), { 163 | 'type': self.car3.type, 164 | 'launched': self.car3.launched, 165 | 'name': self.car3.name, 166 | 'categories': [ 167 | { 168 | 'title': self.category1.title, 169 | 'slug': self.category1.slug, 170 | 'icon': self.category1.icon.url, 171 | }, 172 | { 173 | 'title': self.category2.title, 174 | 'slug': self.category2.slug, 175 | 'icon': '', 176 | } 177 | ] 178 | }) 179 | 180 | def test_index_to_dict(self): 181 | self.maxDiff = None 182 | index_dict = car_index.to_dict() 183 | text_type = 'string' if ES_MAJOR_VERSION == 2 else 'text' 184 | 185 | test_index = DSLIndex('test_index').settings(**index_settings) 186 | test_index.document(CarDocument) 187 | 188 | index_dict = test_index.to_dict() 189 | 190 | self.assertEqual(index_dict['settings'], { 191 | 'number_of_shards': 1, 192 | 'number_of_replicas': 0, 193 | 'analysis': { 194 | 'analyzer': { 195 | 'html_strip': { 196 | 'tokenizer': 'standard', 197 | 'filter': ['lowercase', 198 | 'stop', 'snowball'], 199 | 'type': 'custom', 200 | 'char_filter': ['html_strip'] 201 | } 202 | } 203 | } 204 | }) 205 | self.assertEqual(index_dict['mappings'], { 206 | 'properties': { 207 | 'ads': { 208 | 'type': 'nested', 209 | 'properties': { 210 | 'description': { 211 | 'type': text_type, 'analyzer': 212 | 'html_strip' 213 | }, 214 | 'pk': {'type': 'integer'}, 215 | 'title': {'type': text_type} 216 | }, 217 | }, 218 | 'categories': { 219 | 'type': 'nested', 220 | 'properties': { 221 | 'title': {'type': text_type}, 222 | 'slug': {'type': text_type}, 223 | 'icon': {'type': text_type} 224 | }, 225 | }, 226 | 'manufacturer': { 227 | 'type': 'object', 228 | 'properties': { 229 | 'country': {'type': text_type}, 230 | 'name': {'type': text_type} 231 | }, 232 | }, 233 | 'name': {'type': text_type}, 234 | 'launched': {'type': 'date'}, 235 | 'type': {'type': text_type} 236 | } 237 | }) 238 | 239 | def test_related_docs_are_updated(self): 240 | # test foreignkey relation 241 | self.manufacturer.name = 'Citroen' 242 | self.manufacturer.save() 243 | 244 | s = CarDocument.search().query("match", name=self.car2.name) 245 | car2_doc = s.execute()[0] 246 | self.assertEqual(car2_doc.manufacturer.name, 'Citroen') 247 | self.assertEqual(len(car2_doc.ads), 0) 248 | 249 | ad3 = Ad.objects.create( 250 | title=_("Ad number 3"), url="www.ad3.com", 251 | description="My super ad description 3", 252 | car=self.car2 253 | ) 254 | s = CarDocument.search().query("match", name=self.car2.name) 255 | car2_doc = s.execute()[0] 256 | self.assertEqual(len(car2_doc.ads), 1) 257 | ad3.delete() 258 | s = CarDocument.search().query("match", name=self.car2.name) 259 | car2_doc = s.execute()[0] 260 | self.assertEqual(len(car2_doc.ads), 0) 261 | 262 | self.manufacturer.delete() 263 | s = CarDocument.search().query("match", name=self.car2.name) 264 | car2_doc = s.execute()[0] 265 | self.assertEqual(car2_doc.manufacturer, {}) 266 | 267 | def test_m2m_related_docs_are_updated(self): 268 | # test m2m add 269 | category = Category( 270 | title="Category", slug="category", icon="icon.jpeg" 271 | ) 272 | category.save() 273 | self.car2.categories.add(category) 274 | s = CarDocument.search().query("match", name=self.car2.name) 275 | car2_doc = s.execute()[0] 276 | self.assertEqual(len(car2_doc.categories), 2) 277 | 278 | # test m2m deletion 279 | self.car2.categories.remove(category) 280 | s = CarDocument.search().query("match", name=self.car2.name) 281 | car2_doc = s.execute()[0] 282 | self.assertEqual(len(car2_doc.categories), 1) 283 | 284 | self.category1.car_set.clear() 285 | s = CarDocument.search().query("match", name=self.car2.name) 286 | car2_doc = s.execute()[0] 287 | self.assertEqual(len(car2_doc.categories), 0) 288 | 289 | def test_related_docs_with_prepare_are_updated(self): 290 | s = CarWithPrepareDocument.search().query("match", name=self.car2.name) 291 | self.assertEqual(s.execute()[0].manufacturer.name, 'Peugeot') 292 | self.assertEqual(s.execute()[0].manufacturer_short.name, 'Peugeot') 293 | 294 | self.manufacturer.name = 'Citroen' 295 | self.manufacturer.save() 296 | s = CarWithPrepareDocument.search().query("match", name=self.car2.name) 297 | self.assertEqual(s.execute()[0].manufacturer.name, 'Citroen') 298 | self.assertEqual(s.execute()[0].manufacturer_short.name, 'Citroen') 299 | 300 | self.manufacturer.delete() 301 | s = CarWithPrepareDocument.search().query("match", name=self.car2.name) 302 | self.assertEqual(s.execute()[0].manufacturer, {}) 303 | 304 | def test_delete_create_populate_commands(self): 305 | out = StringIO() 306 | self.assertTrue(ad_index.exists()) 307 | self.assertTrue(car_index.exists()) 308 | 309 | call_command('search_index', action='delete', 310 | force=True, stdout=out, models=['tests.ad']) 311 | self.assertFalse(ad_index.exists()) 312 | self.assertTrue(car_index.exists()) 313 | 314 | call_command('search_index', action='create', 315 | models=['tests.ad'], stdout=out) 316 | self.assertTrue(ad_index.exists()) 317 | call_command('search_index', action='populate', 318 | models=['tests.ad'], stdout=out) 319 | result = AdDocument().search().execute() 320 | self.assertEqual(len(result), 2) 321 | 322 | def test_rebuild_command(self): 323 | out = StringIO() 324 | result = AdDocument().search().execute() 325 | self.assertEqual(len(result), 2) 326 | 327 | Ad(title="Ad title 3").save() 328 | 329 | call_command('search_index', action='populate', 330 | force=True, stdout=out, models=['tests.ad']) 331 | result = AdDocument().search().execute() 332 | self.assertEqual(len(result), 3) 333 | 334 | def test_filter_queryset(self): 335 | Ad(title="Nothing that match", car=self.car1).save() 336 | 337 | qs = AdDocument().search().query( 338 | 'match', title="Ad number 2").filter_queryset(Ad.objects) 339 | self.assertEqual(qs.count(), 2) 340 | self.assertEqual(list(qs), [self.ad2, self.ad1]) 341 | 342 | qs = AdDocument().search().query( 343 | 'match', title="Ad number 2" 344 | ).filter_queryset(Ad.objects.filter(url="www.ad2.com")) 345 | self.assertEqual(qs.count(), 1) 346 | self.assertEqual(list(qs), [self.ad2]) 347 | 348 | with self.assertRaisesMessage(TypeError, 'Unexpected queryset model'): 349 | AdDocument().search().query( 350 | 'match', title="Ad number 2").filter_queryset(Category.objects) 351 | 352 | def test_to_queryset(self): 353 | Ad(title="Nothing that match", car=self.car1).save() 354 | qs = AdDocument().search().query( 355 | 'match', title="Ad number 2").to_queryset() 356 | self.assertEqual(qs.count(), 2) 357 | self.assertEqual(list(qs), [self.ad2, self.ad1]) 358 | 359 | def test_queryset_iterator_queries(self): 360 | ad3 = Ad(title="Ad 3", car=self.car1) 361 | ad3.save() 362 | with self.assertNumQueries(1): 363 | AdDocument().update(Ad.objects.all()) 364 | 365 | doc = AdDocument() 366 | 367 | with self.assertNumQueries(1): 368 | doc.update(Ad.objects.all().order_by('-id')) 369 | self.assertEqual( 370 | set(int(instance.meta.id) for instance in 371 | doc.search().query('match', title="Ad")), 372 | set([ad3.pk, self.ad1.pk, self.ad2.pk]) 373 | ) 374 | 375 | def test_default_document_id(self): 376 | obj_id = 12458 377 | article_slug = "some-article" 378 | article = Article( 379 | id=obj_id, 380 | slug=article_slug, 381 | ) 382 | 383 | # saving should create two documents (in the two indices): one with the 384 | # Django object's id as the ES doc _id, and the other with the slug 385 | # as the ES _id 386 | article.save() 387 | 388 | # assert that the document's id is the id of the Django object 389 | try: 390 | es_obj = ArticleDocument.get(id=obj_id) 391 | except NotFoundError: 392 | self.fail("document with _id {} not found").format(obj_id) 393 | self.assertEqual(es_obj.slug, article.slug) 394 | 395 | def test_custom_document_id(self): 396 | article_slug = "my-very-first-article" 397 | article = Article( 398 | slug=article_slug, 399 | ) 400 | 401 | # saving should create two documents (in the two indices): one with the 402 | # Django object's id as the ES doc _id, and the other with the slug 403 | # as the ES _id 404 | article.save() 405 | 406 | # assert that the document's id is its the slug 407 | try: 408 | es_obj = ArticleWithSlugAsIdDocument.get(id=article_slug) 409 | except NotFoundError: 410 | self.fail( 411 | "document with _id '{}' not found: " 412 | "using a custom id is broken".format(article_slug) 413 | ) 414 | self.assertEqual(es_obj.slug, article.slug) 415 | -------------------------------------------------------------------------------- /tests/test_registries.py: -------------------------------------------------------------------------------- 1 | from mock import Mock 2 | from unittest import TestCase 3 | 4 | from django.conf import settings 5 | 6 | from django_elasticsearch_dsl import Index 7 | from django_elasticsearch_dsl.registries import DocumentRegistry 8 | 9 | from .fixtures import WithFixturesMixin 10 | 11 | 12 | class DocumentRegistryTestCase(WithFixturesMixin, TestCase): 13 | def setUp(self): 14 | self.registry = DocumentRegistry() 15 | self.index_1 = Index(name='index_1') 16 | self.index_2 = Index(name='index_2') 17 | 18 | self.doc_a1 = self._generate_doc_mock(self.ModelA, self.index_1) 19 | self.doc_a2 = self._generate_doc_mock(self.ModelA, self.index_1) 20 | self.doc_b1 = self._generate_doc_mock(self.ModelB, self.index_2) 21 | self.doc_c1 = self._generate_doc_mock(self.ModelC, self.index_1) 22 | 23 | def test_empty_registry(self): 24 | registry = DocumentRegistry() 25 | self.assertEqual(registry._indices, {}) 26 | self.assertEqual(registry._models, {}) 27 | 28 | def test_register(self): 29 | self.assertEqual(self.registry._models[self.ModelA], 30 | set([self.doc_a1, self.doc_a2])) 31 | self.assertEqual(self.registry._models[self.ModelB], 32 | set([self.doc_b1])) 33 | 34 | self.assertEqual(self.registry._indices[self.index_1], 35 | set([self.doc_a1, self.doc_a2, self.doc_c1])) 36 | self.assertEqual(self.registry._indices[self.index_2], 37 | set([self.doc_b1])) 38 | 39 | def test_get_models(self): 40 | self.assertEqual(self.registry.get_models(), 41 | set([self.ModelA, self.ModelB, self.ModelC])) 42 | 43 | def test_get_documents(self): 44 | self.assertEqual(self.registry.get_documents(), 45 | set([self.doc_a1, self.doc_a2, 46 | self.doc_b1, self.doc_c1])) 47 | 48 | def test_get_documents_by_model(self): 49 | self.assertEqual(self.registry.get_documents([self.ModelA]), 50 | set([self.doc_a1, self.doc_a2])) 51 | 52 | def test_get_documents_by_unregister_model(self): 53 | ModelC = Mock() 54 | self.assertFalse(self.registry.get_documents([ModelC])) 55 | 56 | def test_get_indices(self): 57 | self.assertEqual(self.registry.get_indices(), 58 | set([self.index_1, self.index_2])) 59 | 60 | def test_get_indices_by_model(self): 61 | self.assertEqual(self.registry.get_indices([self.ModelA]), 62 | set([self.index_1])) 63 | 64 | def test_get_indices_by_unregister_model(self): 65 | ModelC = Mock() 66 | self.assertFalse(self.registry.get_indices([ModelC])) 67 | 68 | def test_update_instance(self): 69 | doc_a3 = self._generate_doc_mock( 70 | self.ModelA, self.index_1, _ignore_signals=True 71 | ) 72 | 73 | instance = self.ModelA() 74 | self.registry.update(instance) 75 | 76 | self.assertFalse(doc_a3.update.called) 77 | self.assertFalse(self.doc_b1.update.called) 78 | self.doc_a1.update.assert_called_once_with(instance) 79 | self.doc_a2.update.assert_called_once_with(instance) 80 | 81 | def test_update_related_instances(self): 82 | doc_d1 = self._generate_doc_mock( 83 | self.ModelD, self.index_1, 84 | _related_models=[self.ModelE, self.ModelB] 85 | ) 86 | doc_d2 = self._generate_doc_mock( 87 | self.ModelD, self.index_1, _related_models=[self.ModelE] 88 | ) 89 | 90 | instance_e = self.ModelE() 91 | instance_b = self.ModelB() 92 | related_instance = self.ModelD() 93 | 94 | doc_d2.get_instances_from_related.return_value = related_instance 95 | doc_d1.get_instances_from_related.return_value = related_instance 96 | self.registry.update_related(instance_e) 97 | 98 | doc_d1.get_instances_from_related.assert_called_once_with(instance_e) 99 | doc_d1.update.assert_called_once_with(related_instance) 100 | doc_d2.get_instances_from_related.assert_called_once_with(instance_e) 101 | doc_d2.update.assert_called_once_with(related_instance) 102 | 103 | doc_d1.get_instances_from_related.reset_mock() 104 | doc_d1.update.reset_mock() 105 | doc_d2.get_instances_from_related.reset_mock() 106 | doc_d2.update.reset_mock() 107 | 108 | self.registry.update_related(instance_b) 109 | doc_d1.get_instances_from_related.assert_called_once_with(instance_b) 110 | doc_d1.update.assert_called_once_with(related_instance) 111 | doc_d2.get_instances_from_related.assert_not_called() 112 | doc_d2.update.assert_not_called() 113 | 114 | def test_update_related_instances_not_defined(self): 115 | doc_d1 = self._generate_doc_mock(_model=self.ModelD, index=self.index_1, 116 | _related_models=[self.ModelE]) 117 | 118 | instance = self.ModelE() 119 | 120 | doc_d1.get_instances_from_related.return_value = None 121 | self.registry.update_related(instance) 122 | 123 | doc_d1.get_instances_from_related.assert_called_once_with(instance) 124 | doc_d1.update.assert_not_called() 125 | 126 | def test_delete_instance(self): 127 | doc_a3 = self._generate_doc_mock( 128 | self.ModelA, self.index_1, _ignore_signals=True 129 | ) 130 | 131 | instance = self.ModelA() 132 | self.registry.delete(instance) 133 | 134 | self.assertFalse(doc_a3.update.called) 135 | self.assertFalse(self.doc_b1.update.called) 136 | self.doc_a1.update.assert_called_once_with(instance, action='delete') 137 | self.doc_a2.update.assert_called_once_with(instance, action='delete') 138 | 139 | def test_autosync(self): 140 | settings.ELASTICSEARCH_DSL_AUTOSYNC = False 141 | 142 | instance = self.ModelA() 143 | self.registry.update(instance) 144 | self.assertFalse(self.doc_a1.update.called) 145 | 146 | settings.ELASTICSEARCH_DSL_AUTOSYNC = True 147 | -------------------------------------------------------------------------------- /tests/test_signals.py: -------------------------------------------------------------------------------- 1 | from unittest import TestCase 2 | 3 | from mock import Mock, patch 4 | 5 | from django_elasticsearch_dsl.documents import DocType 6 | from django_elasticsearch_dsl.registries import registry 7 | from django_elasticsearch_dsl.signals import post_index 8 | 9 | from .models import Car 10 | 11 | 12 | class PostIndexSignalTestCase(TestCase): 13 | 14 | @patch('django_elasticsearch_dsl.documents.DocType._get_actions') 15 | @patch('django_elasticsearch_dsl.documents.bulk') 16 | def test_post_index_signal_sent(self, bulk, get_actions): 17 | 18 | @registry.register_document 19 | class CarDocument(DocType): 20 | class Django: 21 | fields = ['name'] 22 | model = Car 23 | 24 | bulk.return_value = (1, []) 25 | 26 | # register a mock signal receiver 27 | mock_receiver = Mock() 28 | post_index.connect(mock_receiver) 29 | 30 | doc = CarDocument() 31 | car = Car( 32 | pk=51, 33 | name="Type 57" 34 | ) 35 | doc.update(car) 36 | 37 | bulk.assert_called_once() 38 | 39 | mock_receiver.assert_called_once_with( 40 | signal=post_index, 41 | sender=CarDocument, 42 | instance=doc, 43 | actions=get_actions(), 44 | response=(1, []) 45 | ) 46 | -------------------------------------------------------------------------------- /tox.ini: -------------------------------------------------------------------------------- 1 | [tox] 2 | envlist = 3 | py{38,39,310}-django-{32,41,42}-{es64,es74} 4 | py{311}-django-{41,42}-{es64,es74} 5 | 6 | 7 | [testenv] 8 | setenv = 9 | PYTHONPATH = {toxinidir}:{toxinidir}/django_elasticsearch_dsl 10 | commands = coverage run --source django_elasticsearch_dsl runtests.py {posargs} 11 | 12 | deps = 13 | django-32: Django>=3.2,<3.3 14 | django-41: Django>=4.1,<4.2 15 | django-42: Django>=4.2,<4.3 16 | es64: elasticsearch-dsl>=6.4.0,<7.0.0 17 | es74: elasticsearch-dsl>=7.4.0,<8 18 | -r{toxinidir}/requirements_test.txt 19 | 20 | basepython = 21 | py38: python3.8 22 | py39: python3.9 23 | py310: python3.10 24 | py311: python3.11 25 | --------------------------------------------------------------------------------