├── .bumpversion.cfg ├── .coveragerc ├── .editorconfig ├── .flake8 ├── .github └── ISSUE_TEMPLATE.md ├── .gitignore ├── .isort.cfg ├── .prospector.yml ├── .travis.yml ├── AUTHORS.rst ├── CHANGELOG.rst ├── CONTRIBUTING.rst ├── LICENSE ├── MANIFEST.in ├── Makefile ├── README.rst ├── SECURITY.rst ├── django_zombodb ├── __init__.py ├── admin_mixins.py ├── apps.py ├── base_indexes.py ├── exceptions.py ├── helpers.py ├── indexes.py ├── operations.py ├── querysets.py ├── serializers.py ├── static │ └── django_zombodb │ │ └── js │ │ └── hide_show_score.js └── templates │ └── django_zombodb │ └── base.html ├── docs ├── Makefile ├── authors.rst ├── changelog.rst ├── conf.py ├── contributing.rst ├── django_zombodb.rst ├── index.rst ├── installation.rst ├── integrating.rst ├── make.bat ├── modules.rst └── searching.rst ├── example ├── README.md ├── example │ ├── __init__.py │ ├── settings.py │ ├── urls.py │ └── wsgi.py ├── manage.py ├── requirements.txt └── restaurants │ ├── __init__.py │ ├── admin.py │ ├── apps.py │ ├── data │ └── yellowpages_com-restaurant_sample.csv │ ├── management │ ├── __init__.py │ └── commands │ │ ├── __init__.py │ │ └── filldata.py │ ├── migrations │ ├── 0001_squashed_0004_auto_20190718_2025.py │ └── __init__.py │ └── models.py ├── requirements.txt ├── requirements ├── base.in ├── dev.in ├── dev.txt ├── doc.in ├── doc.txt ├── prod.in ├── quality.in ├── quality.txt ├── test.in ├── test.txt ├── travis.in └── travis.txt ├── runtests.py ├── setup.cfg ├── setup.py ├── test_manage.py ├── tests ├── __init__.py ├── migrations │ ├── 0001_setup_extensions.py │ ├── 0002_datetimearraymodel_integerarraymodel.py │ └── __init__.py ├── models.py ├── restaurants │ ├── __init__.py │ ├── admin.py │ ├── apps.py │ ├── migrations │ │ ├── 0001_initial.py │ │ └── __init__.py │ └── models.py ├── settings.py ├── test_admin_mixins.py ├── test_apps.py ├── test_index.py ├── test_querysets.py └── urls.py └── tox.ini /.bumpversion.cfg: -------------------------------------------------------------------------------- 1 | [bumpversion] 2 | current_version = 0.3.0 3 | commit = True 4 | tag = True 5 | 6 | [bumpversion:file:django_zombodb/__init__.py] 7 | 8 | -------------------------------------------------------------------------------- /.coveragerc: -------------------------------------------------------------------------------- 1 | [run] 2 | branch = true 3 | 4 | [report] 5 | omit = 6 | *site-packages* 7 | *tests* 8 | *.tox* 9 | django_zombodb/apps.py 10 | show_missing = True 11 | exclude_lines = 12 | raise NotImplementedError 13 | pragma: no cover 14 | -------------------------------------------------------------------------------- /.editorconfig: -------------------------------------------------------------------------------- 1 | # http://editorconfig.org 2 | 3 | root = true 4 | 5 | [*] 6 | charset = utf-8 7 | end_of_line = lf 8 | insert_final_newline = true 9 | trim_trailing_whitespace = true 10 | 11 | [*.{py,rst,ini}] 12 | indent_style = space 13 | indent_size = 4 14 | 15 | [*.{html,css,scss,json,yml}] 16 | indent_style = space 17 | indent_size = 2 18 | 19 | [*.md] 20 | trim_trailing_whitespace = false 21 | 22 | [Makefile] 23 | indent_style = tab 24 | -------------------------------------------------------------------------------- /.flake8: -------------------------------------------------------------------------------- 1 | [flake8] 2 | max-line-length = 100 3 | ignore = D100,D101,D102,D103,D105,D205,D400 4 | exclude = */migrations/*,*/settings/*,./venv/*,./env/* 5 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE.md: -------------------------------------------------------------------------------- 1 | * django_zombodb version: 2 | * Postgres version: 3 | * ZomboDB version: 4 | * Django version: 5 | * Python version: 6 | * Operating System: 7 | 8 | ### Description 9 | 10 | Describe what you were trying to get done. 11 | Tell us what happened, what went wrong, and what you expected to happen. 12 | 13 | ### What I Did 14 | 15 | ``` 16 | Paste the command(s) you ran and the output. 17 | If there was a crash, please include the traceback here. 18 | ``` 19 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.py[cod] 2 | __pycache__ 3 | 4 | # C extensions 5 | *.so 6 | 7 | # Packages 8 | *.egg 9 | *.egg-info 10 | dist 11 | build 12 | eggs 13 | parts 14 | bin 15 | var 16 | sdist 17 | develop-eggs 18 | .installed.cfg 19 | lib 20 | lib64 21 | 22 | # Installer logs 23 | pip-log.txt 24 | 25 | # Unit test / coverage reports 26 | .coverage 27 | .tox 28 | nosetests.xml 29 | htmlcov 30 | 31 | # Translations 32 | *.mo 33 | 34 | # Mr Developer 35 | .mr.developer.cfg 36 | .project 37 | .pydevproject 38 | 39 | # Pycharm/Intellij 40 | .idea 41 | 42 | # Complexity 43 | output/*.html 44 | output/*/index.html 45 | 46 | # Sphinx 47 | docs/_build 48 | 49 | # Private deps 50 | requirement/private.in 51 | requirement/private.txt 52 | 53 | # Visual Studio Code 54 | .vscode 55 | 56 | # Local decouple env vars 57 | .env 58 | 59 | # Virtualenvs 60 | venv/ 61 | env/ 62 | 63 | # Pyenv 64 | .python-version 65 | -------------------------------------------------------------------------------- /.isort.cfg: -------------------------------------------------------------------------------- 1 | [settings] 2 | line_length=100 3 | multi_line_output=5 4 | known_django=django 5 | sections=FUTURE,STDLIB,DJANGO,THIRDPARTY,FIRSTPARTY,LOCALFOLDER 6 | lines_after_imports=2 7 | atomic=True 8 | combine_star=True 9 | skip=venv,env,node_modules,migrations,_scripts,_personal,mediafiles,wsgi.py,.tox 10 | -------------------------------------------------------------------------------- /.prospector.yml: -------------------------------------------------------------------------------- 1 | output-format: text 2 | 3 | strictness: veryhigh 4 | test-warnings: true 5 | doc-warnings: false 6 | member-warnings: true 7 | 8 | uses: 9 | - django 10 | 11 | pep8: 12 | full: true 13 | disable: 14 | - D100 15 | - D101 16 | - D102 17 | - D103 18 | - D105 19 | - D205 20 | - D400 21 | - N811 # constant imported as non constant, breaks for Django's `Q` 22 | - N802 # function name should be lowercase, breaks for some Django test methods 23 | options: 24 | max-line-length: 100 25 | 26 | pyflakes: 27 | disable: 28 | - F999 29 | 30 | pylint: 31 | disable: 32 | - invalid-name 33 | - no-else-return 34 | - no-member 35 | - no-self-use 36 | - protected-access 37 | - super-init-not-called 38 | - too-few-public-methods 39 | - too-many-public-methods 40 | - too-many-arguments 41 | - too-many-instance-attributes 42 | options: 43 | max-line-length: 100 44 | max-parents: 15 45 | 46 | dodgy: 47 | run: true 48 | 49 | ignore-paths: 50 | - node_modules 51 | - venv 52 | - env 53 | - docs 54 | 55 | ignore-patterns: 56 | - .+/migrations/.+ 57 | - tests/settings.py 58 | - setup.py 59 | - runtests.py 60 | - manage.py 61 | - .tox/.+ 62 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | # Config file for automatic testing at travis-ci.org 2 | 3 | language: python 4 | 5 | sudo: required 6 | dist: xenial 7 | 8 | python: 9 | - "3.5" 10 | - "3.6" 11 | - "3.7" 12 | 13 | env: 14 | - DJANGO=master 15 | - DJANGO=2.2 16 | - DJANGO=2.1 17 | - DJANGO=2.0 18 | 19 | matrix: 20 | include: 21 | - python: "3.7" 22 | env: TOXENV=quality 23 | - python: "3.7" 24 | env: TOXENV=docs 25 | allow_failures: 26 | - env: DJANGO=master 27 | 28 | addons: 29 | postgresql: "10" 30 | 31 | before_install: 32 | - ES=6.2.4 33 | - sudo apt-get purge -y elasticsearch 34 | - sudo apt-get update -y -qq 35 | - sudo apt-get install -y wget 36 | - sudo apt-get purge -y postgresql-9.1 37 | - sudo apt-get purge -y postgresql-9.2 38 | - sudo apt-get purge -y postgresql-9.3 39 | - sudo apt-get purge -y postgresql-9.4 40 | - sudo apt-get purge -y postgresql-9.5 41 | - sudo apt-get purge -y postgresql-9.6 42 | - sudo apt-get autoremove -y 43 | - chmod -R a+rwx ~/ 44 | - wget --no-check-certificate https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-${ES}.tar.gz 45 | - tar xzf ./elasticsearch-${ES}.tar.gz 46 | - wget --no-check-certificate https://www.zombodb.com/releases/v10-1.0.3/zombodb_xenial_pg10-10-1.0.3_amd64.deb 47 | - sudo dpkg -i zombodb_xenial_pg10-10-1.0.3_amd64.deb 48 | 49 | install: 50 | - pip install -r requirements/travis.txt 51 | 52 | script: 53 | - elasticsearch-${ES}/bin/elasticsearch -d 54 | - sudo /etc/init.d/postgresql restart 10 55 | - sudo su - postgres -c "psql -c \"CREATE USER django_zombodb WITH PASSWORD 'password' SUPERUSER;\"" 56 | - sleep 10 57 | - tox 58 | 59 | after_success: 60 | - codecov 61 | -------------------------------------------------------------------------------- /AUTHORS.rst: -------------------------------------------------------------------------------- 1 | ======= 2 | Credits 3 | ======= 4 | 5 | Development Lead 6 | ---------------- 7 | 8 | * Flávio Juvenal 9 | 10 | Contributors 11 | ------------ 12 | 13 | None yet. Why not be the first? 14 | -------------------------------------------------------------------------------- /CHANGELOG.rst: -------------------------------------------------------------------------------- 1 | .. :changelog: 2 | 3 | Change Log 4 | ---------- 5 | 6 | 0.3.0 (2019-07-18) 7 | ++++++++++++++++++ 8 | 9 | * Support for custom Elasticsearch mappings through ``field_mapping`` parameter on ``ZomboDBIndex``. 10 | * Support to ``limit`` parameter on search methods. 11 | 12 | 0.2.1 (2019-06-13) 13 | ++++++++++++++++++ 14 | 15 | * Dropped support for Python 3.4. 16 | * Added missing imports to docs. 17 | 18 | 19 | 0.2.0 (2019-03-01) 20 | ++++++++++++++++++ 21 | 22 | * Removed parameter ``url`` from ``ZomboDBIndex``. This simplifies the support of multiple deployment environments (local, staging, production), because the ElasticSearch URL isn't copied to inside migrations code (see `Issue #17 `_). 23 | 24 | 25 | 0.1.0 (2019-02-01) 26 | ++++++++++++++++++ 27 | 28 | * First release on PyPI. 29 | -------------------------------------------------------------------------------- /CONTRIBUTING.rst: -------------------------------------------------------------------------------- 1 | ============ 2 | Contributing 3 | ============ 4 | 5 | Contributions are welcome, and they are greatly appreciated! Every 6 | little bit helps, and credit will always be given. 7 | 8 | You can contribute in many ways: 9 | 10 | Types of Contributions 11 | ---------------------- 12 | 13 | Report Bugs 14 | ~~~~~~~~~~~ 15 | 16 | Report bugs at https://github.com/vintasoftware/django-zombodb/issues. 17 | Please fill the fields of the issue template. 18 | 19 | Fix Bugs 20 | ~~~~~~~~ 21 | 22 | Look through the GitHub issues for bugs. Anything tagged with "bug" 23 | is open to whoever wants to implement it. 24 | 25 | Implement Features 26 | ~~~~~~~~~~~~~~~~~~ 27 | 28 | Look through the GitHub issues for features. Anything tagged with "feature" 29 | is open to whoever wants to implement it. 30 | 31 | Write Documentation 32 | ~~~~~~~~~~~~~~~~~~~ 33 | 34 | django-zombodb could always use more documentation, whether as part of the 35 | official django-zombodb docs, in docstrings, or even on the web in blog posts, 36 | articles, and such. 37 | 38 | Submit Feedback 39 | ~~~~~~~~~~~~~~~ 40 | 41 | The best way to send feedback is to file an issue at https://github.com/vintasoftware/django-zombodb/issues. 42 | 43 | If you are proposing a feature: 44 | 45 | * Explain in detail how it would work. 46 | * Keep the scope as narrow as possible, to make it easier to implement. 47 | * Remember that this is a volunteer-driven project, and that contributions 48 | are welcome :) 49 | 50 | Get Started! 51 | ------------ 52 | 53 | Ready to contribute? Here's how to set up `django-zombodb` for local development. 54 | 55 | 1. Fork the `django-zombodb` repo on GitHub. 56 | 2. Clone your fork locally:: 57 | 58 | $ git clone git@github.com:your_name_here/django-zombodb.git 59 | 60 | 3. Install your local copy into a virtualenv. Assuming you have virtualenvwrapper installed, this is how you set up your fork for local development:: 61 | 62 | $ mkvirtualenv django-zombodb 63 | $ cd django-zombodb/ 64 | $ pip install -e . 65 | $ make install_requirements 66 | 67 | 4. Create a branch for local development:: 68 | 69 | $ git checkout -b name-of-your-bugfix-or-feature 70 | 71 | Now you can make your changes locally. 72 | 73 | 5. When you're done making changes, check that your changes pass the linters and the 74 | tests, including testing other Python versions with tox:: 75 | 76 | $ make lint 77 | $ make test 78 | $ make test-all 79 | 80 | 6. Commit your changes and push your branch to GitHub:: 81 | 82 | $ git add . 83 | $ git commit -m "Your detailed description of your changes." 84 | $ git push origin name-of-your-bugfix-or-feature 85 | 86 | 7. Submit a pull request through the GitHub website. 87 | 88 | Pull Request Guidelines 89 | ----------------------- 90 | 91 | Before you submit a pull request, check that it meets these guidelines: 92 | 93 | 1. The pull request should include tests. 94 | 2. If the pull request adds functionality, the docs should be updated. 95 | 3. The pull request should pass CI. Check 96 | https://travis-ci.org/vintasoftware/django-zombodb/pull_requests 97 | and make sure that the tests pass for all supported Python versions. 98 | 99 | Tips 100 | ---- 101 | 102 | To run a subset of tests:: 103 | 104 | $ python runtests.py tests.test_apps 105 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2019 Vinta Serviços e Soluções Tecnológicas Ltda - vintasoftware.com 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include AUTHORS.rst 2 | include CONTRIBUTING.rst 3 | include CHANGELOG.rst 4 | include LICENSE 5 | include README.rst 6 | recursive-include django_zombodb *.html *.png *.gif *js *.css *jpg *jpeg *svg *py 7 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | .PHONY: clean-pyc clean-build docs help 2 | .DEFAULT_GOAL := help 3 | define BROWSER_PYSCRIPT 4 | import os, webbrowser, sys 5 | try: 6 | from urllib import pathname2url 7 | except: 8 | from urllib.request import pathname2url 9 | 10 | webbrowser.open("file://" + pathname2url(os.path.abspath(sys.argv[1]))) 11 | endef 12 | export BROWSER_PYSCRIPT 13 | BROWSER := python -c "$$BROWSER_PYSCRIPT" 14 | 15 | help: 16 | @grep '^[a-zA-Z]' $(MAKEFILE_LIST) | sort | awk -F ':.*?## ' 'NF==2 {printf "\033[36m %-25s\033[0m %s\n", $$1, $$2}' 17 | 18 | clean: clean-build clean-pyc 19 | 20 | clean-build: ## remove build artifacts 21 | rm -fr build/ 22 | rm -fr dist/ 23 | rm -fr *.egg-info 24 | 25 | clean-pyc: ## remove Python file artifacts 26 | find . -name '*.pyc' -exec rm -f {} + 27 | find . -name '*.pyo' -exec rm -f {} + 28 | find . -name '*~' -exec rm -f {} + 29 | 30 | lint: ## check linters 31 | isort -rc . --check-only 32 | prospector --messages-only 33 | 34 | test: ## run tests quickly with the default Python 35 | python runtests.py 36 | 37 | test-all: ## run tests on every Python version with tox 38 | tox 39 | 40 | coverage: ## check code coverage quickly with the default Python 41 | coverage run --source django_zombodb runtests.py 42 | coverage report -m 43 | coverage html 44 | open htmlcov/index.html 45 | 46 | docs: ## generate Sphinx HTML documentation, including API docs 47 | rm -f docs/django_zombodb.rst 48 | rm -f docs/modules.rst 49 | sphinx-apidoc -o docs/ django_zombodb 50 | $(MAKE) -C docs clean 51 | DJANGO_SETTINGS_MODULE=example.example.settings $(MAKE) -C docs html 52 | $(BROWSER) docs/_build/html/index.html 53 | 54 | upgrade: ## update the requirements/*.txt files with the latest packages satisfying requirements/*.in 55 | pip install -U -q pip-tools 56 | pip-compile --upgrade -o requirements/dev.txt requirements/base.in requirements/dev.in requirements/quality.in 57 | pip-compile --upgrade -o requirements.txt requirements/base.in requirements/prod.in 58 | # Remove Django from requirements.txt 59 | sed '/django==/d' requirements.txt > requirements.tmp; mv requirements.tmp requirements.txt 60 | sed '/# via django$$/d' requirements.txt > requirements.tmp; mv requirements.tmp requirements.txt 61 | # Make everything =>, not == 62 | sed 's/==/>=/g' requirements.txt > requirements.tmp; mv requirements.tmp requirements.txt 63 | pip-compile --upgrade -o requirements/doc.txt requirements/base.in requirements/doc.in 64 | pip-compile --upgrade -o requirements/quality.txt requirements/quality.in 65 | pip-compile --upgrade -o requirements/test.txt requirements/base.in requirements/test.in 66 | pip-compile --upgrade -o requirements/travis.txt requirements/travis.in 67 | # Let tox control the Django version for tests 68 | sed '/django==/d' requirements/test.txt > requirements/test.tmp; mv requirements/test.tmp requirements/test.txt 69 | sed '/# via django$$/d' requirements/test.txt > requirements/test.tmp; mv requirements/test.tmp requirements/test.txt 70 | 71 | install_requirements: ## install development environment requirements 72 | pip install -qr requirements/dev.txt -qr requirements/test.txt -qr requirements/quality.txt --exists-action w 73 | pip-sync requirements/dev.txt requirements/doc.txt requirements/quality.txt requirements/test.txt 74 | 75 | release: clean ## package and upload a release 76 | python setup.py sdist upload 77 | python setup.py bdist_wheel upload 78 | 79 | sdist: clean ## package 80 | python setup.py sdist 81 | ls -l dist 82 | 83 | selfcheck: ## check that the Makefile is well-formed 84 | @echo "The Makefile is well-formed." 85 | -------------------------------------------------------------------------------- /README.rst: -------------------------------------------------------------------------------- 1 | ============== 2 | django-zombodb 3 | ============== 4 | 5 | .. image:: https://badge.fury.io/py/django-zombodb.svg 6 | :target: https://badge.fury.io/py/django-zombodb 7 | :alt: PyPI Status 8 | 9 | .. image:: https://travis-ci.org/vintasoftware/django-zombodb.svg?branch=master 10 | :target: https://travis-ci.org/vintasoftware/django-zombodb 11 | :alt: Build Status 12 | 13 | .. image:: https://readthedocs.org/projects/django-zombodb/badge/?version=latest 14 | :target: https://django-zombodb.readthedocs.io/en/latest/?badge=latest 15 | :alt: Documentation Status 16 | 17 | .. image:: https://codecov.io/gh/vintasoftware/django-zombodb/branch/master/graph/badge.svg 18 | :target: https://codecov.io/gh/vintasoftware/django-zombodb 19 | :alt: Coverage Status 20 | 21 | .. image:: https://img.shields.io/github/license/vintasoftware/django-zombodb.svg 22 | :alt: GitHub 23 | 24 | Easy Django integration with Elasticsearch through `ZomboDB `_ Postgres Extension. 25 | Thanks to ZomboDB, **your Django models are synced with Elasticsearch at transaction time**! Searching is also very simple: you can make 26 | Elasticsearch queries by just calling one of the search methods on your querysets. Couldn't be easier! 27 | 28 | Documentation 29 | ------------- 30 | 31 | The full documentation is at ``_. 32 | 33 | 34 | Requirements 35 | ------------ 36 | 37 | * **Python**: 3.5, 3.6, 3.7 38 | * **Django**: 2.0, 2.1, 2.2 39 | * **Postgres** and **Elasticsearch**: same as `ZomboDB current requirements `_ 40 | 41 | 42 | Quickstart 43 | ---------- 44 | 45 | 1. Install ZomboDB (instructions `here `_) 46 | 47 | 2. Install django-zombodb: 48 | 49 | :: 50 | 51 | pip install django-zombodb 52 | 53 | 3. Add the ``SearchQuerySet`` and the ``ZomboDBIndex`` to your model: 54 | 55 | .. code-block:: python 56 | 57 | from django_zombodb.indexes import ZomboDBIndex 58 | from django_zombodb.querysets import SearchQuerySet 59 | 60 | class Restaurant(models.Model): 61 | name = models.TextField() 62 | 63 | objects = models.Manager.from_queryset(SearchQuerySet)() 64 | 65 | class Meta: 66 | indexes = [ 67 | ZomboDBIndex(fields=[ 68 | 'name', 69 | ]), 70 | ] 71 | 72 | 4. Make the migrations: 73 | 74 | :: 75 | 76 | python manage.py makemigrations 77 | 78 | 5. Add ``django_zombodb.operations.ZomboDBExtension()`` as the first operation of the migration you've just created: 79 | 80 | .. code-block:: python 81 | 82 | import django_zombodb.operations 83 | 84 | class Migration(migrations.Migration): 85 | 86 | dependencies = [ 87 | ('restaurants', '0001_initial'), 88 | ] 89 | 90 | operations = [ 91 | django_zombodb.operations.ZomboDBExtension(), # <<< here 92 | ] 93 | 94 | 6. Run the migrations (Postgres user must be SUPERUSER to create the ZomboDB extension): 95 | 96 | :: 97 | 98 | python manage.py migrate 99 | 100 | 7. Done! Now you can make Elasticsearch queries directly from your model: 101 | 102 | .. code-block:: python 103 | 104 | Restaurant.objects.filter(is_open=True).query_string_search("brazil* AND coffee~") 105 | 106 | Full Example 107 | ------------ 108 | 109 | Check ``_ 110 | 111 | Running Tests 112 | ------------- 113 | 114 | You need to have Elasticsearch and Postgres instances running on default ports. Also, you need ZomboDB installed. Then just do: 115 | 116 | :: 117 | 118 | python runtests.py 119 | 120 | Security 121 | -------- 122 | 123 | Please check `SECURITY.rst `_. 124 | If you found or if you think you found a vulnerability please get in touch via admin *AT* vinta.com.br 125 | 126 | Please avoid disclosing any security issue on GitHub or any other public website. We'll work to swiftly address any possible vulnerability and give credit to reporters (if wanted). 127 | 128 | 129 | Commercial Support 130 | ------------------ 131 | This project is maintained by `Vinta Software `_ and other contributors. We are always looking for exciting work, so if you need any commercial support, feel free to get in touch: contact@vinta.com.br 132 | -------------------------------------------------------------------------------- /SECURITY.rst: -------------------------------------------------------------------------------- 1 | =============== 2 | Security Policy 3 | =============== 4 | 5 | Supported Versions 6 | ------------------ 7 | 8 | +------------+--------------------+ 9 | | Version | Supported | 10 | +============+====================+ 11 | | >= 0.2.x | ✅ | 12 | +------------+--------------------+ 13 | | < 0.2 | ❌ | 14 | +------------+--------------------+ 15 | 16 | Reporting a Vulnerability 17 | ------------------------- 18 | 19 | If you found or if you think you found a vulnerability please get in touch via admin *AT* vinta.com.br 20 | 21 | Please avoid disclosing any security issue on GitHub or any other public website. We'll work to swiftly address any possible vulnerability and give credit to reporters (if wanted). 22 | -------------------------------------------------------------------------------- /django_zombodb/__init__.py: -------------------------------------------------------------------------------- 1 | __version__ = '0.3.0' 2 | -------------------------------------------------------------------------------- /django_zombodb/admin_mixins.py: -------------------------------------------------------------------------------- 1 | 2 | from django.contrib.admin.views.main import SEARCH_VAR 3 | from django.db.models import FloatField 4 | from django.db.models.expressions import Value 5 | from django.utils.translation import gettext as _ 6 | 7 | from django_zombodb.helpers import validate_query_string 8 | 9 | 10 | class ZomboDBAdminMixin: 11 | max_search_results = None 12 | 13 | def get_search_fields(self, request): 14 | """ 15 | get_search_fields is unnecessary if ZomboDBAdminMixin is used. 16 | But since search_form.html uses this, we'll return a placeholder tuple 17 | """ 18 | return ('-placeholder-',) 19 | 20 | def _check_if_valid_search(self, request): 21 | search_term = request.GET.get(SEARCH_VAR, '') 22 | if not search_term: 23 | return False 24 | 25 | return validate_query_string(self.model, search_term) 26 | 27 | def get_list_display(self, request): 28 | request._has_valid_search = self._check_if_valid_search(request) 29 | return super().get_list_display(request) 30 | 31 | def get_queryset(self, request): 32 | queryset = super().get_queryset(request) 33 | 34 | if not getattr(request, '_has_valid_search', False): 35 | queryset = queryset.annotate(zombodb_score=Value(0.0, FloatField())) 36 | 37 | return queryset 38 | 39 | def get_search_results(self, request, queryset, search_term): 40 | if search_term: 41 | if request._has_valid_search: 42 | queryset = queryset.query_string_search( 43 | search_term, 44 | validate=False, 45 | sort=False, 46 | score_attr='zombodb_score', 47 | limit=self.max_search_results 48 | ).annotate_score() 49 | else: 50 | self.message_user( 51 | request, 52 | _("Invalid search query. Not filtering by search."), 53 | level='ERROR') 54 | use_distinct = False 55 | return queryset, use_distinct 56 | 57 | def get_ordering(self, request): 58 | ordering = super().get_ordering(request) 59 | 60 | if getattr(request, '_has_valid_search', False): 61 | ordering = ('-zombodb_score', 'pk') 62 | 63 | return ordering 64 | 65 | def _zombodb_score(self, instance): 66 | return instance.zombodb_score 67 | _zombodb_score.short_description = "Search score" 68 | _zombodb_score.admin_order_field = 'zombodb_score' 69 | -------------------------------------------------------------------------------- /django_zombodb/apps.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 2 | from django.apps import AppConfig 3 | 4 | 5 | class DjangoZomboDBConfig(AppConfig): 6 | name = 'django_zombodb' 7 | -------------------------------------------------------------------------------- /django_zombodb/base_indexes.py: -------------------------------------------------------------------------------- 1 | # From Django 2.1, for Django < 2.1 2 | from django.db.models import Index 3 | from django.utils.functional import cached_property 4 | 5 | 6 | class PostgresIndex(Index): 7 | 8 | @cached_property 9 | def max_name_length(self): 10 | # Allow an index name longer than 30 characters when the suffix is 11 | # longer than the usual 3 character limit. The 30 character limit for 12 | # cross-database compatibility isn't applicable to PostgreSQL-specific 13 | # indexes. 14 | return Index.max_name_length - len(Index.suffix) + len(self.suffix) 15 | 16 | def create_sql(self, model, schema_editor, using=''): 17 | statement = super().create_sql(model, schema_editor, using=' USING %s' % self.suffix) 18 | with_params = self.get_with_params() 19 | if with_params: # pragma: no cover 20 | statement.parts['extra'] = 'WITH (%s) %s' % ( 21 | ', '.join(with_params), 22 | statement.parts['extra'], 23 | ) 24 | return statement 25 | 26 | def get_with_params(self): 27 | return [] # pragma: no cover 28 | -------------------------------------------------------------------------------- /django_zombodb/exceptions.py: -------------------------------------------------------------------------------- 1 | class InvalidElasticsearchQuery(Exception): 2 | pass 3 | -------------------------------------------------------------------------------- /django_zombodb/helpers.py: -------------------------------------------------------------------------------- 1 | from django.core.exceptions import ImproperlyConfigured 2 | from django.db import connection 3 | 4 | from django_zombodb.indexes import ZomboDBIndex 5 | from django_zombodb.serializers import ES_JSON_SERIALIZER 6 | 7 | 8 | def get_zombodb_index_from_model(model): 9 | for index in model._meta.indexes: 10 | if isinstance(index, ZomboDBIndex): 11 | return index 12 | 13 | raise ImproperlyConfigured( 14 | "Can't find a ZomboDBIndex at model {model}. " 15 | "Did you forget it? ".format(model=model)) 16 | 17 | 18 | def _validate_query(index, post_data): 19 | with connection.cursor() as cursor: 20 | cursor.execute(''' 21 | SELECT zdb.request(%(index_name)s, %(endpoint)s, 'POST', %(post_data)s); 22 | ''', { 23 | 'index_name': index.name, 24 | 'endpoint': '_validate/query', 25 | 'post_data': post_data 26 | }) 27 | validation_result = ES_JSON_SERIALIZER.loads(cursor.fetchone()[0]) 28 | if 'error' in validation_result: 29 | raise ImproperlyConfigured( 30 | "Unexpected Elasticsearch error. " 31 | "You may need to recreate your index={index}. " 32 | "Details:\n" 33 | "{error}".format(index=index, error=validation_result)) 34 | return validation_result['valid'] 35 | 36 | 37 | def validate_query_string(model, query): 38 | post_data = ES_JSON_SERIALIZER.dumps( 39 | {'query': {'query_string': {'query': query}}}) 40 | index = get_zombodb_index_from_model(model) 41 | 42 | return _validate_query(index, post_data) 43 | 44 | 45 | def validate_query_dict(model, query): 46 | post_data = ES_JSON_SERIALIZER.dumps({'query': query}) 47 | index = get_zombodb_index_from_model(model) 48 | 49 | return _validate_query(index, post_data) 50 | -------------------------------------------------------------------------------- /django_zombodb/indexes.py: -------------------------------------------------------------------------------- 1 | import django 2 | from django.conf import settings 3 | from django.core.exceptions import ImproperlyConfigured 4 | 5 | from django_zombodb.serializers import ES_JSON_SERIALIZER 6 | 7 | 8 | try: 9 | from django.contrib.postgres.indexes import PostgresIndex 10 | except ImportError: 11 | # Django < 2.1 12 | from django_zombodb.base_indexes import PostgresIndex 13 | 14 | 15 | class ZomboDBIndexCreateStatementAdapter: 16 | template = "CREATE INDEX %(name)s ON %(table)s USING zombodb ((ROW(%(columns)s)::%(row_type)s)) %(extra)s" # noqa: E501 17 | 18 | def __init__(self, statement, model, schema_editor, fields, field_mapping, row_type): 19 | self.statement = statement 20 | self.parts = self.statement.parts 21 | 22 | self.model = model 23 | self.schema_editor = schema_editor 24 | self.fields = fields 25 | self.field_mapping = field_mapping 26 | self.row_type = row_type 27 | 28 | def references_table(self, *args, **kwargs): 29 | return self.statement.references_table(*args, **kwargs) 30 | 31 | def references_column(self, *args, **kwargs): 32 | return self.statement.references_column(*args, **kwargs) 33 | 34 | def rename_table_references(self, *args, **kwargs): 35 | return self.statement.rename_table_references(*args, **kwargs) 36 | 37 | def rename_column_references(self, *args, **kwargs): 38 | return self.statement.rename_column_references(*args, **kwargs) 39 | 40 | def _get_field_mapping(self): 41 | define_field_mapping = 'SELECT zdb.define_field_mapping(\'%s\', \'%s\', \'%s\');' 42 | s = '' 43 | if self.field_mapping: 44 | for field in self.field_mapping: 45 | mapping = ES_JSON_SERIALIZER.dumps(self.field_mapping[field]) 46 | s += define_field_mapping % (self.parts['table'], field, mapping) 47 | return s 48 | 49 | def _get_create_type(self): 50 | create_type = 'CREATE TYPE %s AS (' % self.row_type 51 | for field in self.fields: 52 | field_db_type = self.model._meta.get_field(field).db_type( 53 | connection=self.schema_editor.connection) 54 | create_type += field + ' ' + field_db_type + ', ' 55 | create_type = create_type[:-len(', ')] 56 | create_type += '); ' 57 | return create_type 58 | 59 | def __repr__(self): 60 | return '<%s %r>' % (self.__class__.__name__, str(self)) 61 | 62 | def __str__(self): 63 | parts = dict(self.parts) # copy 64 | parts['row_type'] = self.row_type 65 | create_index = self.template % parts 66 | 67 | s = self._get_create_type() + self._get_field_mapping() + create_index 68 | return s 69 | 70 | 71 | class ZomboDBIndexRemoveStatementAdapter: 72 | 73 | def __init__(self, statement, row_type): 74 | self.statement = statement 75 | self.template = self.statement.template 76 | self.parts = self.statement.parts 77 | 78 | self.row_type = row_type 79 | 80 | def references_table(self, *args, **kwargs): 81 | return self.statement.references_table(*args, **kwargs) 82 | 83 | def references_column(self, *args, **kwargs): 84 | return self.statement.references_column(*args, **kwargs) 85 | 86 | def rename_table_references(self, *args, **kwargs): 87 | return self.statement.rename_table_references(*args, **kwargs) 88 | 89 | def rename_column_references(self, *args, **kwargs): 90 | return self.statement.rename_column_references(*args, **kwargs) 91 | 92 | def __repr__(self): 93 | return '<%s %r>' % (self.__class__.__name__, str(self)) 94 | 95 | def __str__(self): 96 | s = str(self.statement) 97 | s += ('; DROP TYPE IF EXISTS %s;' % self.row_type) 98 | return s 99 | 100 | 101 | class ZomboDBIndex(PostgresIndex): 102 | suffix = 'zombodb' 103 | 104 | def __init__( 105 | self, *, 106 | shards=None, 107 | replicas=None, 108 | alias=None, 109 | refresh_interval=None, 110 | type_name=None, 111 | bulk_concurrency=None, 112 | batch_size=None, 113 | compression_level=None, 114 | llapi=None, 115 | field_mapping=None, 116 | **kwargs): 117 | url = kwargs.pop('url', None) 118 | if url: 119 | raise ImproperlyConfigured( 120 | "The `url` param is not supported anymore. " 121 | "Instead, please remove it and set ZOMBODB_ELASTICSEARCH_URL on settings.") 122 | 123 | try: 124 | url = settings.ZOMBODB_ELASTICSEARCH_URL 125 | except AttributeError: 126 | raise ImproperlyConfigured("Please set ZOMBODB_ELASTICSEARCH_URL on settings.") 127 | 128 | self.url = url 129 | self.shards = shards 130 | self.replicas = replicas 131 | self.alias = alias 132 | self.refresh_interval = refresh_interval 133 | self.type_name = type_name 134 | self.bulk_concurrency = bulk_concurrency 135 | self.batch_size = batch_size 136 | self.compression_level = compression_level 137 | self.llapi = llapi 138 | self.field_mapping = field_mapping 139 | super().__init__(**kwargs) 140 | 141 | def _get_row_type_name(self): 142 | # should be less than 63 (DatabaseOperations.max_name_length), 143 | # since Index.max_name_length is 30 144 | return self.name + '_row_type' 145 | 146 | def create_sql(self, model, schema_editor, using=''): # pylint: disable=unused-argument 147 | statement = super().create_sql(model, schema_editor) 148 | row_type = schema_editor.quote_name(self._get_row_type_name()) 149 | return ZomboDBIndexCreateStatementAdapter( 150 | statement, model, schema_editor, self.fields, self.field_mapping, row_type) 151 | 152 | def remove_sql(self, model, schema_editor): 153 | statement = super().remove_sql(model, schema_editor) 154 | row_type = schema_editor.quote_name(self._get_row_type_name()) 155 | if django.VERSION >= (2, 2, 0): 156 | return ZomboDBIndexRemoveStatementAdapter(statement, row_type) 157 | else: 158 | return statement + ('; DROP TYPE IF EXISTS %s;' % row_type) 159 | 160 | def _get_params(self): 161 | return [ 162 | ('url', self.url, str), 163 | ('shards', self.shards, int), 164 | ('replicas', self.replicas, int), 165 | ('alias', self.alias, str), 166 | ('refresh_interval', self.refresh_interval, str), 167 | ('type_name', self.type_name, str), 168 | ('bulk_concurrency', self.bulk_concurrency, int), 169 | ('batch_size', self.batch_size, int), 170 | ('compression_level', self.compression_level, int), 171 | ('llapi', self.llapi, bool), 172 | ('field_mapping', self.field_mapping, dict), 173 | ] 174 | 175 | def _format_param_value(self, value, param_type): 176 | if param_type == str: 177 | value_formatted = "'" + value + "'" 178 | elif param_type == int: 179 | value_formatted = str(value) 180 | else: # param_type == bool 181 | value_formatted = 'true' if value else 'false' 182 | return value_formatted 183 | 184 | def deconstruct(self): 185 | path, args, kwargs = super().deconstruct() 186 | params = self._get_params() 187 | params = params[1:] # don't add URL to migrations 188 | for param, value, __ in params: 189 | if value is not None: 190 | kwargs[param] = value 191 | return path, args, kwargs 192 | 193 | def get_with_params(self): 194 | with_params = [] 195 | for param, value, param_type in self._get_params(): 196 | if param == 'field_mapping': 197 | continue 198 | if value is not None: 199 | value_formatted = self._format_param_value(value, param_type) 200 | with_params.append('%s = %s' % (param, value_formatted)) 201 | 202 | return with_params 203 | -------------------------------------------------------------------------------- /django_zombodb/operations.py: -------------------------------------------------------------------------------- 1 | from django.contrib.postgres.operations import CreateExtension 2 | 3 | 4 | class ZomboDBExtension(CreateExtension): 5 | 6 | def __init__(self): 7 | self.name = 'zombodb' 8 | -------------------------------------------------------------------------------- /django_zombodb/querysets.py: -------------------------------------------------------------------------------- 1 | from django.db import connection, models 2 | from django.db.models.expressions import RawSQL 3 | 4 | from elasticsearch_dsl import Search 5 | 6 | from django_zombodb.exceptions import InvalidElasticsearchQuery 7 | from django_zombodb.helpers import validate_query_dict, validate_query_string 8 | from django_zombodb.serializers import ES_JSON_SERIALIZER 9 | 10 | 11 | class SearchQuerySetMixin: 12 | 13 | def annotate_score(self, attr='zombodb_score'): 14 | db_table = connection.ops.quote_name(self.model._meta.db_table) 15 | return self.annotate(**{ 16 | attr: RawSQL('zdb.score(' + db_table + '."ctid")', []) 17 | }) 18 | 19 | def order_by_score(self, score_attr='zombodb_score'): 20 | return self.annotate_score(score_attr).order_by('-' + score_attr, 'pk') 21 | 22 | def _search(self, query, query_str, validate, validate_fn, sort, score_attr, limit): 23 | if validate: 24 | is_valid = validate_fn(self.model, query) 25 | if not is_valid: 26 | raise InvalidElasticsearchQuery( 27 | "Invalid Elasticsearch query: {}".format(query_str)) 28 | 29 | if limit is not None: 30 | queryset = self.extra( 31 | where=[ 32 | connection.ops.quote_name(self.model._meta.db_table) + ' ==> dsl.limit(%s, %s)' 33 | ], 34 | params=[limit, query_str], 35 | ) 36 | else: 37 | queryset = self.extra( 38 | where=[connection.ops.quote_name(self.model._meta.db_table) + ' ==> %s'], 39 | params=[query_str], 40 | ) 41 | if sort: 42 | queryset = queryset.order_by_score(score_attr=score_attr) 43 | 44 | return queryset 45 | 46 | def query_string_search( 47 | self, query, validate=False, sort=False, score_attr='zombodb_score', limit=None): 48 | query_str = query 49 | 50 | return self._search( 51 | query=query, 52 | query_str=query_str, 53 | validate=validate, 54 | validate_fn=validate_query_string, 55 | sort=sort, 56 | score_attr=score_attr, 57 | limit=limit) 58 | 59 | def dict_search( 60 | self, query, validate=False, sort=False, score_attr='zombodb_score', limit=None): 61 | query_str = ES_JSON_SERIALIZER.dumps(query) 62 | 63 | return self._search( 64 | query=query, 65 | query_str=query_str, 66 | validate=validate, 67 | validate_fn=validate_query_dict, 68 | sort=sort, 69 | score_attr=score_attr, 70 | limit=limit) 71 | 72 | def dsl_search( 73 | self, query, validate=False, sort=False, score_attr='zombodb_score', limit=None): 74 | if isinstance(query, Search): 75 | raise InvalidElasticsearchQuery( 76 | "Do not use the `Search` class. " 77 | "`query` must be an instance of a class inheriting from `DslBase`.") 78 | 79 | query_dict = query.to_dict() 80 | 81 | return self.dict_search( 82 | query=query_dict, 83 | validate=validate, 84 | sort=sort, 85 | score_attr=score_attr, 86 | limit=limit) 87 | 88 | 89 | class SearchQuerySet(SearchQuerySetMixin, models.QuerySet): 90 | pass 91 | -------------------------------------------------------------------------------- /django_zombodb/serializers.py: -------------------------------------------------------------------------------- 1 | from elasticsearch.serializer import JSONSerializer 2 | 3 | 4 | ES_JSON_SERIALIZER = JSONSerializer() 5 | -------------------------------------------------------------------------------- /django_zombodb/static/django_zombodb/js/hide_show_score.js: -------------------------------------------------------------------------------- 1 | (function($) { 2 | $(document).ready(function () { 3 | var scoreAllZero = ( 4 | $('td.field-_zombodb_score') 5 | .toArray() 6 | .every(function (x) { return x.textContent == '0.0' })); 7 | if (scoreAllZero) { 8 | $('th.column-_zombodb_score').remove(); 9 | $('td.field-_zombodb_score').remove(); 10 | } 11 | }); 12 | })(django.jQuery || jQuery); 13 | -------------------------------------------------------------------------------- /django_zombodb/templates/django_zombodb/base.html: -------------------------------------------------------------------------------- 1 | 2 | {% comment %} 3 | As the developer of this package, don't place anything here if you can help it 4 | since this allows developers to have interoperability between your template 5 | structure and their own. 6 | 7 | Example: Developer melding the 2SoD pattern to fit inside with another pattern:: 8 | 9 | {% extends "base.html" %} 10 | {% load static %} 11 | 12 | 13 | {% block extra_js %} 14 | 15 | 16 | {% block javascript %} 17 | 18 | {% endblock javascript %} 19 | 20 | {% endblock extra_js %} 21 | {% endcomment %} 22 | -------------------------------------------------------------------------------- /docs/Makefile: -------------------------------------------------------------------------------- 1 | # Makefile for Sphinx documentation 2 | # 3 | 4 | # You can set these variables from the command line. 5 | SPHINXOPTS = 6 | SPHINXBUILD = sphinx-build 7 | PAPER = 8 | BUILDDIR = _build 9 | 10 | # User-friendly check for sphinx-build 11 | ifeq ($(shell which $(SPHINXBUILD) >/dev/null 2>&1; echo $$?), 1) 12 | $(error The '$(SPHINXBUILD)' command was not found. Make sure you have Sphinx installed, then set the SPHINXBUILD environment variable to point to the full path of the '$(SPHINXBUILD)' executable. Alternatively you can add the directory with the executable to your PATH. If you don't have Sphinx installed, grab it from http://sphinx-doc.org/) 13 | endif 14 | 15 | # Internal variables. 16 | PAPEROPT_a4 = -D latex_paper_size=a4 17 | PAPEROPT_letter = -D latex_paper_size=letter 18 | ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . 19 | # the i18n builder cannot share the environment and doctrees with the others 20 | I18NSPHINXOPTS = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . 21 | 22 | .PHONY: help clean html dirhtml singlehtml pickle json htmlhelp qthelp devhelp epub latex latexpdf text man changes linkcheck doctest gettext 23 | 24 | help: 25 | @echo "Please use \`make ' where is one of" 26 | @echo " html to make standalone HTML files" 27 | @echo " dirhtml to make HTML files named index.html in directories" 28 | @echo " singlehtml to make a single large HTML file" 29 | @echo " pickle to make pickle files" 30 | @echo " json to make JSON files" 31 | @echo " htmlhelp to make HTML files and a HTML help project" 32 | @echo " qthelp to make HTML files and a qthelp project" 33 | @echo " devhelp to make HTML files and a Devhelp project" 34 | @echo " epub to make an epub" 35 | @echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter" 36 | @echo " latexpdf to make LaTeX files and run them through pdflatex" 37 | @echo " latexpdfja to make LaTeX files and run them through platex/dvipdfmx" 38 | @echo " text to make text files" 39 | @echo " man to make manual pages" 40 | @echo " texinfo to make Texinfo files" 41 | @echo " info to make Texinfo files and run them through makeinfo" 42 | @echo " gettext to make PO message catalogs" 43 | @echo " changes to make an overview of all changed/added/deprecated items" 44 | @echo " xml to make Docutils-native XML files" 45 | @echo " pseudoxml to make pseudoxml-XML files for display purposes" 46 | @echo " linkcheck to check all external links for integrity" 47 | @echo " doctest to run all doctests embedded in the documentation (if enabled)" 48 | 49 | clean: 50 | rm -rf $(BUILDDIR)/* 51 | 52 | html: 53 | $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html 54 | @echo 55 | @echo "Build finished. The HTML pages are in $(BUILDDIR)/html." 56 | 57 | dirhtml: 58 | $(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml 59 | @echo 60 | @echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml." 61 | 62 | singlehtml: 63 | $(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) $(BUILDDIR)/singlehtml 64 | @echo 65 | @echo "Build finished. The HTML page is in $(BUILDDIR)/singlehtml." 66 | 67 | pickle: 68 | $(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle 69 | @echo 70 | @echo "Build finished; now you can process the pickle files." 71 | 72 | json: 73 | $(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json 74 | @echo 75 | @echo "Build finished; now you can process the JSON files." 76 | 77 | htmlhelp: 78 | $(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp 79 | @echo 80 | @echo "Build finished; now you can run HTML Help Workshop with the" \ 81 | ".hhp project file in $(BUILDDIR)/htmlhelp." 82 | 83 | qthelp: 84 | $(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp 85 | @echo 86 | @echo "Build finished; now you can run "qcollectiongenerator" with the" \ 87 | ".qhcp project file in $(BUILDDIR)/qthelp, like this:" 88 | @echo "# qcollectiongenerator $(BUILDDIR)/qthelp/complexity.qhcp" 89 | @echo "To view the help file:" 90 | @echo "# assistant -collectionFile $(BUILDDIR)/qthelp/complexity.qhc" 91 | 92 | devhelp: 93 | $(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILDDIR)/devhelp 94 | @echo 95 | @echo "Build finished." 96 | @echo "To view the help file:" 97 | @echo "# mkdir -p $$HOME/.local/share/devhelp/complexity" 98 | @echo "# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/complexity" 99 | @echo "# devhelp" 100 | 101 | epub: 102 | $(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILDDIR)/epub 103 | @echo 104 | @echo "Build finished. The epub file is in $(BUILDDIR)/epub." 105 | 106 | latex: 107 | $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex 108 | @echo 109 | @echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex." 110 | @echo "Run \`make' in that directory to run these through (pdf)latex" \ 111 | "(use \`make latexpdf' here to do that automatically)." 112 | 113 | latexpdf: 114 | $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex 115 | @echo "Running LaTeX files through pdflatex..." 116 | $(MAKE) -C $(BUILDDIR)/latex all-pdf 117 | @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." 118 | 119 | latexpdfja: 120 | $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex 121 | @echo "Running LaTeX files through platex and dvipdfmx..." 122 | $(MAKE) -C $(BUILDDIR)/latex all-pdf-ja 123 | @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." 124 | 125 | text: 126 | $(SPHINXBUILD) -b text $(ALLSPHINXOPTS) $(BUILDDIR)/text 127 | @echo 128 | @echo "Build finished. The text files are in $(BUILDDIR)/text." 129 | 130 | man: 131 | $(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man 132 | @echo 133 | @echo "Build finished. The manual pages are in $(BUILDDIR)/man." 134 | 135 | texinfo: 136 | $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo 137 | @echo 138 | @echo "Build finished. The Texinfo files are in $(BUILDDIR)/texinfo." 139 | @echo "Run \`make' in that directory to run these through makeinfo" \ 140 | "(use \`make info' here to do that automatically)." 141 | 142 | info: 143 | $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo 144 | @echo "Running Texinfo files through makeinfo..." 145 | make -C $(BUILDDIR)/texinfo info 146 | @echo "makeinfo finished; the Info files are in $(BUILDDIR)/texinfo." 147 | 148 | gettext: 149 | $(SPHINXBUILD) -b gettext $(I18NSPHINXOPTS) $(BUILDDIR)/locale 150 | @echo 151 | @echo "Build finished. The message catalogs are in $(BUILDDIR)/locale." 152 | 153 | changes: 154 | $(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes 155 | @echo 156 | @echo "The overview file is in $(BUILDDIR)/changes." 157 | 158 | linkcheck: 159 | $(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck 160 | @echo 161 | @echo "Link check complete; look for any errors in the above output " \ 162 | "or in $(BUILDDIR)/linkcheck/output.txt." 163 | 164 | doctest: 165 | $(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest 166 | @echo "Testing of doctests in the sources finished, look at the " \ 167 | "results in $(BUILDDIR)/doctest/output.txt." 168 | 169 | xml: 170 | $(SPHINXBUILD) -b xml $(ALLSPHINXOPTS) $(BUILDDIR)/xml 171 | @echo 172 | @echo "Build finished. The XML files are in $(BUILDDIR)/xml." 173 | 174 | pseudoxml: 175 | $(SPHINXBUILD) -b pseudoxml $(ALLSPHINXOPTS) $(BUILDDIR)/pseudoxml 176 | @echo 177 | @echo "Build finished. The pseudo-XML files are in $(BUILDDIR)/pseudoxml." 178 | -------------------------------------------------------------------------------- /docs/authors.rst: -------------------------------------------------------------------------------- 1 | .. include:: ../AUTHORS.rst 2 | -------------------------------------------------------------------------------- /docs/changelog.rst: -------------------------------------------------------------------------------- 1 | .. include:: ../CHANGELOG.rst 2 | -------------------------------------------------------------------------------- /docs/conf.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # 3 | # complexity documentation build configuration file, created by 4 | # sphinx-quickstart on Tue Jul 9 22:26:36 2013. 5 | # 6 | # This file is execfile()d with the current directory set to its containing dir. 7 | # 8 | # Note that not all possible configuration values are present in this 9 | # autogenerated file. 10 | # 11 | # All configuration values have a default; values that are commented out 12 | # serve to show the default. 13 | 14 | import os 15 | import sys 16 | 17 | import django_zombodb 18 | 19 | 20 | # If extensions (or modules to document with autodoc) are in another directory, 21 | # add these directories to sys.path here. If the directory is relative to the 22 | # documentation root, use os.path.abspath to make it absolute, like shown here. 23 | #sys.path.insert(0, os.path.abspath('.')) 24 | 25 | cwd = os.getcwd() 26 | parent = os.path.dirname(cwd) 27 | sys.path.append(parent) 28 | 29 | 30 | # -- General configuration ----------------------------------------------------- 31 | 32 | # If your documentation needs a minimal Sphinx version, state it here. 33 | #needs_sphinx = '1.0' 34 | 35 | # Add any Sphinx extension module names here, as strings. They can be extensions 36 | # coming with Sphinx (named 'sphinx.ext.*') or your custom ones. 37 | extensions = ['sphinx.ext.autodoc', 'sphinx.ext.viewcode'] 38 | 39 | # Add any paths that contain templates here, relative to this directory. 40 | templates_path = ['_templates'] 41 | 42 | # The suffix of source filenames. 43 | source_suffix = '.rst' 44 | 45 | # The encoding of source files. 46 | #source_encoding = 'utf-8-sig' 47 | 48 | # The master toctree document. 49 | master_doc = 'index' 50 | 51 | # General information about the project. 52 | project = u'django-zombodb' 53 | copyright = u'2019, vintasoftware.com' 54 | 55 | # The version info for the project you're documenting, acts as replacement for 56 | # |version| and |release|, also used in various other places throughout the 57 | # built documents. 58 | # 59 | # The short X.Y version. 60 | version = django_zombodb.__version__ 61 | # The full version, including alpha/beta/rc tags. 62 | release = django_zombodb.__version__ 63 | 64 | # The language for content autogenerated by Sphinx. Refer to documentation 65 | # for a list of supported languages. 66 | #language = None 67 | 68 | # There are two options for replacing |today|: either, you set today to some 69 | # non-false value, then it is used: 70 | #today = '' 71 | # Else, today_fmt is used as the format for a strftime call. 72 | #today_fmt = '%B %d, %Y' 73 | 74 | # List of patterns, relative to source directory, that match files and 75 | # directories to ignore when looking for source files. 76 | exclude_patterns = ['_build'] 77 | 78 | # The reST default role (used for this markup: `text`) to use for all documents. 79 | #default_role = None 80 | 81 | # If true, '()' will be appended to :func: etc. cross-reference text. 82 | #add_function_parentheses = True 83 | 84 | # If true, the current module name will be prepended to all description 85 | # unit titles (such as .. function::). 86 | #add_module_names = True 87 | 88 | # If true, sectionauthor and moduleauthor directives will be shown in the 89 | # output. They are ignored by default. 90 | #show_authors = False 91 | 92 | # The name of the Pygments (syntax highlighting) style to use. 93 | pygments_style = 'sphinx' 94 | 95 | # A list of ignored prefixes for module index sorting. 96 | #modindex_common_prefix = [] 97 | 98 | # If true, keep warnings as "system message" paragraphs in the built documents. 99 | #keep_warnings = False 100 | 101 | 102 | # -- Options for HTML output --------------------------------------------------- 103 | 104 | # The theme to use for HTML and HTML Help pages. See the documentation for 105 | # a list of builtin themes. 106 | html_theme = 'sphinx_rtd_theme' 107 | 108 | # Theme options are theme-specific and customize the look and feel of a theme 109 | # further. For a list of options available for each theme, see the 110 | # documentation. 111 | #html_theme_options = {} 112 | 113 | # Add any paths that contain custom themes here, relative to this directory. 114 | #html_theme_path = [] 115 | 116 | # The name for this set of Sphinx documents. If None, it defaults to 117 | # " v documentation". 118 | #html_title = None 119 | 120 | # A shorter title for the navigation bar. Default is the same as html_title. 121 | #html_short_title = None 122 | 123 | # The name of an image file (relative to this directory) to place at the top 124 | # of the sidebar. 125 | #html_logo = None 126 | 127 | # The name of an image file (within the static path) to use as favicon of the 128 | # docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 129 | # pixels large. 130 | #html_favicon = None 131 | 132 | # Add any paths that contain custom static files (such as style sheets) here, 133 | # relative to this directory. They are copied after the builtin static files, 134 | # so a file named "default.css" will overwrite the builtin "default.css". 135 | html_static_path = ['_static'] 136 | 137 | # If not '', a 'Last updated on:' timestamp is inserted at every page bottom, 138 | # using the given strftime format. 139 | #html_last_updated_fmt = '%b %d, %Y' 140 | 141 | # If true, SmartyPants will be used to convert quotes and dashes to 142 | # typographically correct entities. 143 | #html_use_smartypants = True 144 | 145 | # Custom sidebar templates, maps document names to template names. 146 | #html_sidebars = {} 147 | 148 | # Additional templates that should be rendered to pages, maps page names to 149 | # template names. 150 | #html_additional_pages = {} 151 | 152 | # If false, no module index is generated. 153 | #html_domain_indices = True 154 | 155 | # If false, no index is generated. 156 | #html_use_index = True 157 | 158 | # If true, the index is split into individual pages for each letter. 159 | #html_split_index = False 160 | 161 | # If true, links to the reST sources are added to the pages. 162 | #html_show_sourcelink = True 163 | 164 | # If true, "Created using Sphinx" is shown in the HTML footer. Default is True. 165 | #html_show_sphinx = True 166 | 167 | # If true, "(C) Copyright ..." is shown in the HTML footer. Default is True. 168 | #html_show_copyright = True 169 | 170 | # If true, an OpenSearch description file will be output, and all pages will 171 | # contain a tag referring to it. The value of this option must be the 172 | # base URL from which the finished HTML is served. 173 | #html_use_opensearch = '' 174 | 175 | # This is the file name suffix for HTML files (e.g. ".xhtml"). 176 | #html_file_suffix = None 177 | 178 | # Output file base name for HTML help builder. 179 | htmlhelp_basename = 'django-zombodbdoc' 180 | 181 | 182 | # -- Options for LaTeX output -------------------------------------------------- 183 | 184 | latex_elements = { 185 | # The paper size ('letterpaper' or 'a4paper'). 186 | #'papersize': 'letterpaper', 187 | 188 | # The font size ('10pt', '11pt' or '12pt'). 189 | #'pointsize': '10pt', 190 | 191 | # Additional stuff for the LaTeX preamble. 192 | #'preamble': '', 193 | } 194 | 195 | # Grouping the document tree into LaTeX files. List of tuples 196 | # (source start file, target name, title, author, documentclass [howto/manual]). 197 | latex_documents = [ 198 | ('index', 'django-zombodb.tex', u'django-zombodb Documentation', 199 | u'Flávio Juvenal', 'manual'), 200 | ] 201 | 202 | # The name of an image file (relative to this directory) to place at the top of 203 | # the title page. 204 | #latex_logo = None 205 | 206 | # For "manual" documents, if this is true, then toplevel headings are parts, 207 | # not chapters. 208 | #latex_use_parts = False 209 | 210 | # If true, show page references after internal links. 211 | #latex_show_pagerefs = False 212 | 213 | # If true, show URL addresses after external links. 214 | #latex_show_urls = False 215 | 216 | # Documents to append as an appendix to all manuals. 217 | #latex_appendices = [] 218 | 219 | # If false, no module index is generated. 220 | #latex_domain_indices = True 221 | 222 | 223 | # -- Options for manual page output -------------------------------------------- 224 | 225 | # One entry per manual page. List of tuples 226 | # (source start file, name, description, authors, manual section). 227 | man_pages = [ 228 | ('index', 'django-zombodb', u'django-zombodb Documentation', 229 | [u'Flávio Juvenal'], 1) 230 | ] 231 | 232 | # If true, show URL addresses after external links. 233 | #man_show_urls = False 234 | 235 | 236 | # -- Options for Texinfo output ------------------------------------------------ 237 | 238 | # Grouping the document tree into Texinfo files. List of tuples 239 | # (source start file, target name, title, author, 240 | # dir menu entry, description, category) 241 | texinfo_documents = [ 242 | ('index', 'django-zombodb', u'django-zombodb Documentation', 243 | u'Flávio Juvenal', 'django-zombodb', 244 | 'Django + ZomboDB: best way to integrate Django with Elasticsearch', 245 | 'Miscellaneous'), 246 | ] 247 | 248 | # Documents to append as an appendix to all manuals. 249 | #texinfo_appendices = [] 250 | 251 | # If false, no module index is generated. 252 | #texinfo_domain_indices = True 253 | 254 | # How to display URL addresses: 'footnote', 'no', or 'inline'. 255 | #texinfo_show_urls = 'footnote' 256 | 257 | # If true, do not generate a @detailmenu in the "Top" node's menu. 258 | #texinfo_no_detailmenu = False 259 | -------------------------------------------------------------------------------- /docs/contributing.rst: -------------------------------------------------------------------------------- 1 | .. include:: ../CONTRIBUTING.rst 2 | -------------------------------------------------------------------------------- /docs/django_zombodb.rst: -------------------------------------------------------------------------------- 1 | django\_zombodb package 2 | ======================= 3 | 4 | Submodules 5 | ---------- 6 | 7 | django\_zombodb.admin\_mixins module 8 | ------------------------------------ 9 | 10 | .. automodule:: django_zombodb.admin_mixins 11 | :members: 12 | :undoc-members: 13 | :show-inheritance: 14 | 15 | django\_zombodb.apps module 16 | --------------------------- 17 | 18 | .. automodule:: django_zombodb.apps 19 | :members: 20 | :undoc-members: 21 | :show-inheritance: 22 | 23 | django\_zombodb.base\_indexes module 24 | ------------------------------------ 25 | 26 | .. automodule:: django_zombodb.base_indexes 27 | :members: 28 | :undoc-members: 29 | :show-inheritance: 30 | 31 | django\_zombodb.exceptions module 32 | --------------------------------- 33 | 34 | .. automodule:: django_zombodb.exceptions 35 | :members: 36 | :undoc-members: 37 | :show-inheritance: 38 | 39 | django\_zombodb.helpers module 40 | ------------------------------ 41 | 42 | .. automodule:: django_zombodb.helpers 43 | :members: 44 | :undoc-members: 45 | :show-inheritance: 46 | 47 | django\_zombodb.indexes module 48 | ------------------------------ 49 | 50 | .. automodule:: django_zombodb.indexes 51 | :members: 52 | :undoc-members: 53 | :show-inheritance: 54 | 55 | django\_zombodb.operations module 56 | --------------------------------- 57 | 58 | .. automodule:: django_zombodb.operations 59 | :members: 60 | :undoc-members: 61 | :show-inheritance: 62 | 63 | django\_zombodb.querysets module 64 | -------------------------------- 65 | 66 | .. automodule:: django_zombodb.querysets 67 | :members: 68 | :undoc-members: 69 | :show-inheritance: 70 | 71 | django\_zombodb.serializers module 72 | ---------------------------------- 73 | 74 | .. automodule:: django_zombodb.serializers 75 | :members: 76 | :undoc-members: 77 | :show-inheritance: 78 | 79 | 80 | Module contents 81 | --------------- 82 | 83 | .. automodule:: django_zombodb 84 | :members: 85 | :undoc-members: 86 | :show-inheritance: 87 | -------------------------------------------------------------------------------- /docs/index.rst: -------------------------------------------------------------------------------- 1 | .. complexity documentation master file, created by 2 | sphinx-quickstart on Tue Jul 9 22:26:36 2013. 3 | You can adapt this file completely to your liking, but it should at least 4 | contain the root `toctree` directive. 5 | 6 | django-zombodb documentation 7 | ============================ 8 | 9 | Easy Django integration with Elasticsearch through `ZomboDB `_ Postgres Extension. 10 | Thanks to ZomboDB, **your Django models are synced with Elasticsearch at transaction time**! Searching is also very simple: you can make 11 | Elasticsearch queries by just calling one of the search methods on your querysets. Couldn't be easier! 12 | 13 | .. toctree:: 14 | :maxdepth: 2 15 | :caption: User Guide 16 | 17 | installation 18 | integrating 19 | searching 20 | 21 | .. toctree:: 22 | :maxdepth: 2 23 | :caption: Reference 24 | 25 | django_zombodb 26 | 27 | .. toctree:: 28 | :maxdepth: 2 29 | :caption: Releases 30 | 31 | changelog 32 | 33 | .. toctree:: 34 | :maxdepth: 2 35 | :caption: Development 36 | 37 | contributing 38 | authors 39 | -------------------------------------------------------------------------------- /docs/installation.rst: -------------------------------------------------------------------------------- 1 | ============================== 2 | Installation and Configuration 3 | ============================== 4 | 5 | Example 6 | ------- 7 | You can check a fully configured Django project with django-zombodb at ``_ 8 | 9 | Requirements 10 | ------------ 11 | 12 | * **Python**: 3.5, 3.6, 3.7 13 | * **Django**: 2.0, 2.1 14 | 15 | Installation 16 | ------------ 17 | 18 | Install django-zombodb: :: 19 | 20 | pip install django-zombodb 21 | 22 | Settings 23 | -------- 24 | 25 | Set ``ZOMBODB_ELASTICSEARCH_URL`` on your settings.py. That is the URL of the ElasticSearch cluster used by ZomboDB. 26 | 27 | .. code-block:: python 28 | 29 | ZOMBODB_ELASTICSEARCH_URL = 'http://localhost:9200/' 30 | 31 | Move forward to learn how to integrate your models with Elasticsearch. 32 | -------------------------------------------------------------------------------- /docs/integrating.rst: -------------------------------------------------------------------------------- 1 | ============================== 2 | Integrating with Elasticsearch 3 | ============================== 4 | 5 | ZomboDB integrates Postgres with Elasticsearch through Postgres indexes. If you don't know much about ZomboDB, please read its `tutorial `_ before proceeding. 6 | 7 | Installing ZomboDB extension 8 | ---------------------------- 9 | 10 | Since ZomboDB is a Postgres extension, you must install and activate it. Follow the official ZomboDB installation `instructions `_. 11 | 12 | Activating ZomboDB extension 13 | ---------------------------- 14 | 15 | django-zombodb provides a Django migration operation to activate ZomboDB extension on your database. To run it, please make sure your database user is a superuser: :: 16 | 17 | psql -d your_database -c "ALTER USER your_database_user SUPERUSER" 18 | 19 | Then create an empty migration on your "main" app (usually called "core" or "common"): :: 20 | 21 | python manage.py makemigrations core --empty 22 | 23 | Add the :py:class:`django_zombodb.operations.ZomboDBExtension` operation to the migration you've just created: 24 | 25 | .. code-block:: python 26 | 27 | import django_zombodb.operations 28 | 29 | class Migration(migrations.Migration): 30 | 31 | dependencies = [ 32 | ('restaurants', '0001_initial'), 33 | ] 34 | 35 | operations = [ 36 | django_zombodb.operations.ZomboDBExtension(), 37 | ... 38 | ] 39 | 40 | Alternatively, you can activate the extension manually with a command. But you should **avoid** this because you'll need to remember to run this on production, on tests, and on the machines of all your co-workers: :: 41 | 42 | psql -d django_zombodb -c "CREATE EXTENSION zombodb" 43 | 44 | Creating an index 45 | ----------------- 46 | 47 | Imagine you have the following model: 48 | 49 | .. code-block:: python 50 | 51 | class Restaurant(models.Model): 52 | name = models.TextField() 53 | street = models.TextField() 54 | 55 | To integrate it with Elasticsearch, we need to add a :py:class:`~django_zombodb.indexes.ZomboDBIndex` to it: 56 | 57 | .. code-block:: python 58 | 59 | from django_zombodb.indexes import ZomboDBIndex 60 | 61 | class Restaurant(models.Model): 62 | name = models.TextField() 63 | street = models.TextField() 64 | 65 | class Meta: 66 | indexes = [ 67 | ZomboDBIndex(fields=[ 68 | 'name', 69 | 'street', 70 | ]), 71 | ] 72 | 73 | After that, create and run the migrations: :: 74 | 75 | python manage.py makemigrations 76 | python manage.py migrate 77 | 78 | .. warning:: 79 | 80 | During the migration, :py:class:`~django_zombodb.indexes.ZomboDBIndex` reads the value at ``settings.ZOMBODB_ELASTICSEARCH_URL``. That means if ``settings.ZOMBODB_ELASTICSEARCH_URL`` changes after the :py:class:`~django_zombodb.indexes.ZomboDBIndex` migration, **the internal index stored at Postgres will still point to the old URL**. If you wish to change the URL of an existing :py:class:`~django_zombodb.indexes.ZomboDBIndex`, change both ``settings.ZOMBODB_ELASTICSEARCH_URL`` and issue a ``ALTER INDEX index_name SET (url='http://some.new.url');`` (preferably inside a ``migrations.RunSQL`` in a new migration). 81 | 82 | Now the ``Restaurant`` model will support Elasticsearch queries for both ``name`` and ``street`` fields. But to perform those searches, we need it to use the custom queryset :py:class:`~django_zombodb.querysets.SearchQuerySet`: 83 | 84 | .. code-block:: python 85 | 86 | from django_zombodb.indexes import ZomboDBIndex 87 | from django_zombodb.querysets import SearchQuerySet 88 | 89 | class Restaurant(models.Model): 90 | name = models.TextField() 91 | street = models.TextField() 92 | 93 | objects = models.Manager.from_queryset(SearchQuerySet)() 94 | 95 | class Meta: 96 | indexes = [ 97 | ZomboDBIndex(fields=[ 98 | 'name', 99 | 'street', 100 | ]), 101 | ] 102 | 103 | .. note:: 104 | 105 | If you already have a custom queryset on your model, make it inherit from :py:class:`~django_zombodb.querysets.SearchQuerySetMixin`. 106 | 107 | Field mapping 108 | ------------- 109 | 110 | From `Elasticsearch documentation `_: 111 | 112 | "Mapping is the process of defining how a document, and the fields it contains, are stored and indexed. For instance, use mappings to define: 113 | 114 | - which string fields should be treated as full text fields. 115 | - which fields contain numbers, dates, or geolocations. 116 | - whether the values of all fields in the document should be indexed into the catch-all _all field. 117 | - the format of date values. 118 | - custom rules to control the mapping for dynamically added fields." 119 | 120 | If you don't specify a mapping for your :py:class:`~django_zombodb.indexes.ZomboDBIndex`, django-zombodb uses `ZomboDB's default mappings `_, which are based on the Postgres type of your model fields. 121 | 122 | To customize mapping, specify a ``field_mapping`` parameter to your :py:class:`~django_zombodb.indexes.ZomboDBIndex` like below: 123 | 124 | .. code-block:: python 125 | 126 | from django_zombodb.indexes import ZomboDBIndex 127 | from django_zombodb.querysets import SearchQuerySet 128 | 129 | class Restaurant(models.Model): 130 | name = models.TextField() 131 | street = models.TextField() 132 | 133 | objects = models.Manager.from_queryset(SearchQuerySet)() 134 | 135 | class Meta: 136 | indexes = [ 137 | ZomboDBIndex( 138 | fields=[ 139 | 'name', 140 | 'street', 141 | ], 142 | field_mapping={ 143 | 'name': {"type": "text", 144 | "copy_to": "zdb_all", 145 | "analyzer": "fulltext_with_shingles", 146 | "search_analyzer": "fulltext_with_shingles_search"}, 147 | 'street': {"type": "text", 148 | "copy_to": "zdb_all", 149 | "analyzer": "brazilian"}, 150 | } 151 | ) 152 | ] 153 | 154 | .. note:: 155 | 156 | You probably wish to have ``"copy_to": "zdb_all"`` on your textual fields to match ZomboDB default behavior. From ZomboDB docs: "``zdb_all`` is ZomboDB's version of Elasticsearch's "_all" field, except ``zdb_all`` is enabled for all versions of Elasticsearch. It is also configured as the default search field for every ZomboDB index". For more info, read `Elasticsearch docs take on the "_all" field `_. 157 | 158 | Move forward to learn how to perform Elasticsearch queries through your model. 159 | -------------------------------------------------------------------------------- /docs/make.bat: -------------------------------------------------------------------------------- 1 | @ECHO OFF 2 | 3 | REM Command file for Sphinx documentation 4 | 5 | if "%SPHINXBUILD%" == "" ( 6 | set SPHINXBUILD=sphinx-build 7 | ) 8 | set BUILDDIR=_build 9 | set ALLSPHINXOPTS=-d %BUILDDIR%/doctrees %SPHINXOPTS% . 10 | set I18NSPHINXOPTS=%SPHINXOPTS% . 11 | if NOT "%PAPER%" == "" ( 12 | set ALLSPHINXOPTS=-D latex_paper_size=%PAPER% %ALLSPHINXOPTS% 13 | set I18NSPHINXOPTS=-D latex_paper_size=%PAPER% %I18NSPHINXOPTS% 14 | ) 15 | 16 | if "%1" == "" goto help 17 | 18 | if "%1" == "help" ( 19 | :help 20 | echo.Please use `make ^` where ^ is one of 21 | echo. html to make standalone HTML files 22 | echo. dirhtml to make HTML files named index.html in directories 23 | echo. singlehtml to make a single large HTML file 24 | echo. pickle to make pickle files 25 | echo. json to make JSON files 26 | echo. htmlhelp to make HTML files and a HTML help project 27 | echo. qthelp to make HTML files and a qthelp project 28 | echo. devhelp to make HTML files and a Devhelp project 29 | echo. epub to make an epub 30 | echo. latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter 31 | echo. text to make text files 32 | echo. man to make manual pages 33 | echo. texinfo to make Texinfo files 34 | echo. gettext to make PO message catalogs 35 | echo. changes to make an overview over all changed/added/deprecated items 36 | echo. xml to make Docutils-native XML files 37 | echo. pseudoxml to make pseudoxml-XML files for display purposes 38 | echo. linkcheck to check all external links for integrity 39 | echo. doctest to run all doctests embedded in the documentation if enabled 40 | goto end 41 | ) 42 | 43 | if "%1" == "clean" ( 44 | for /d %%i in (%BUILDDIR%\*) do rmdir /q /s %%i 45 | del /q /s %BUILDDIR%\* 46 | goto end 47 | ) 48 | 49 | 50 | %SPHINXBUILD% 2> nul 51 | if errorlevel 9009 ( 52 | echo. 53 | echo.The 'sphinx-build' command was not found. Make sure you have Sphinx 54 | echo.installed, then set the SPHINXBUILD environment variable to point 55 | echo.to the full path of the 'sphinx-build' executable. Alternatively you 56 | echo.may add the Sphinx directory to PATH. 57 | echo. 58 | echo.If you don't have Sphinx installed, grab it from 59 | echo.http://sphinx-doc.org/ 60 | exit /b 1 61 | ) 62 | 63 | if "%1" == "html" ( 64 | %SPHINXBUILD% -b html %ALLSPHINXOPTS% %BUILDDIR%/html 65 | if errorlevel 1 exit /b 1 66 | echo. 67 | echo.Build finished. The HTML pages are in %BUILDDIR%/html. 68 | goto end 69 | ) 70 | 71 | if "%1" == "dirhtml" ( 72 | %SPHINXBUILD% -b dirhtml %ALLSPHINXOPTS% %BUILDDIR%/dirhtml 73 | if errorlevel 1 exit /b 1 74 | echo. 75 | echo.Build finished. The HTML pages are in %BUILDDIR%/dirhtml. 76 | goto end 77 | ) 78 | 79 | if "%1" == "singlehtml" ( 80 | %SPHINXBUILD% -b singlehtml %ALLSPHINXOPTS% %BUILDDIR%/singlehtml 81 | if errorlevel 1 exit /b 1 82 | echo. 83 | echo.Build finished. The HTML pages are in %BUILDDIR%/singlehtml. 84 | goto end 85 | ) 86 | 87 | if "%1" == "pickle" ( 88 | %SPHINXBUILD% -b pickle %ALLSPHINXOPTS% %BUILDDIR%/pickle 89 | if errorlevel 1 exit /b 1 90 | echo. 91 | echo.Build finished; now you can process the pickle files. 92 | goto end 93 | ) 94 | 95 | if "%1" == "json" ( 96 | %SPHINXBUILD% -b json %ALLSPHINXOPTS% %BUILDDIR%/json 97 | if errorlevel 1 exit /b 1 98 | echo. 99 | echo.Build finished; now you can process the JSON files. 100 | goto end 101 | ) 102 | 103 | if "%1" == "htmlhelp" ( 104 | %SPHINXBUILD% -b htmlhelp %ALLSPHINXOPTS% %BUILDDIR%/htmlhelp 105 | if errorlevel 1 exit /b 1 106 | echo. 107 | echo.Build finished; now you can run HTML Help Workshop with the ^ 108 | .hhp project file in %BUILDDIR%/htmlhelp. 109 | goto end 110 | ) 111 | 112 | if "%1" == "qthelp" ( 113 | %SPHINXBUILD% -b qthelp %ALLSPHINXOPTS% %BUILDDIR%/qthelp 114 | if errorlevel 1 exit /b 1 115 | echo. 116 | echo.Build finished; now you can run "qcollectiongenerator" with the ^ 117 | .qhcp project file in %BUILDDIR%/qthelp, like this: 118 | echo.^> qcollectiongenerator %BUILDDIR%\qthelp\complexity.qhcp 119 | echo.To view the help file: 120 | echo.^> assistant -collectionFile %BUILDDIR%\qthelp\complexity.ghc 121 | goto end 122 | ) 123 | 124 | if "%1" == "devhelp" ( 125 | %SPHINXBUILD% -b devhelp %ALLSPHINXOPTS% %BUILDDIR%/devhelp 126 | if errorlevel 1 exit /b 1 127 | echo. 128 | echo.Build finished. 129 | goto end 130 | ) 131 | 132 | if "%1" == "epub" ( 133 | %SPHINXBUILD% -b epub %ALLSPHINXOPTS% %BUILDDIR%/epub 134 | if errorlevel 1 exit /b 1 135 | echo. 136 | echo.Build finished. The epub file is in %BUILDDIR%/epub. 137 | goto end 138 | ) 139 | 140 | if "%1" == "latex" ( 141 | %SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex 142 | if errorlevel 1 exit /b 1 143 | echo. 144 | echo.Build finished; the LaTeX files are in %BUILDDIR%/latex. 145 | goto end 146 | ) 147 | 148 | if "%1" == "latexpdf" ( 149 | %SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex 150 | cd %BUILDDIR%/latex 151 | make all-pdf 152 | cd %BUILDDIR%/.. 153 | echo. 154 | echo.Build finished; the PDF files are in %BUILDDIR%/latex. 155 | goto end 156 | ) 157 | 158 | if "%1" == "latexpdfja" ( 159 | %SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex 160 | cd %BUILDDIR%/latex 161 | make all-pdf-ja 162 | cd %BUILDDIR%/.. 163 | echo. 164 | echo.Build finished; the PDF files are in %BUILDDIR%/latex. 165 | goto end 166 | ) 167 | 168 | if "%1" == "text" ( 169 | %SPHINXBUILD% -b text %ALLSPHINXOPTS% %BUILDDIR%/text 170 | if errorlevel 1 exit /b 1 171 | echo. 172 | echo.Build finished. The text files are in %BUILDDIR%/text. 173 | goto end 174 | ) 175 | 176 | if "%1" == "man" ( 177 | %SPHINXBUILD% -b man %ALLSPHINXOPTS% %BUILDDIR%/man 178 | if errorlevel 1 exit /b 1 179 | echo. 180 | echo.Build finished. The manual pages are in %BUILDDIR%/man. 181 | goto end 182 | ) 183 | 184 | if "%1" == "texinfo" ( 185 | %SPHINXBUILD% -b texinfo %ALLSPHINXOPTS% %BUILDDIR%/texinfo 186 | if errorlevel 1 exit /b 1 187 | echo. 188 | echo.Build finished. The Texinfo files are in %BUILDDIR%/texinfo. 189 | goto end 190 | ) 191 | 192 | if "%1" == "gettext" ( 193 | %SPHINXBUILD% -b gettext %I18NSPHINXOPTS% %BUILDDIR%/locale 194 | if errorlevel 1 exit /b 1 195 | echo. 196 | echo.Build finished. The message catalogs are in %BUILDDIR%/locale. 197 | goto end 198 | ) 199 | 200 | if "%1" == "changes" ( 201 | %SPHINXBUILD% -b changes %ALLSPHINXOPTS% %BUILDDIR%/changes 202 | if errorlevel 1 exit /b 1 203 | echo. 204 | echo.The overview file is in %BUILDDIR%/changes. 205 | goto end 206 | ) 207 | 208 | if "%1" == "linkcheck" ( 209 | %SPHINXBUILD% -b linkcheck %ALLSPHINXOPTS% %BUILDDIR%/linkcheck 210 | if errorlevel 1 exit /b 1 211 | echo. 212 | echo.Link check complete; look for any errors in the above output ^ 213 | or in %BUILDDIR%/linkcheck/output.txt. 214 | goto end 215 | ) 216 | 217 | if "%1" == "doctest" ( 218 | %SPHINXBUILD% -b doctest %ALLSPHINXOPTS% %BUILDDIR%/doctest 219 | if errorlevel 1 exit /b 1 220 | echo. 221 | echo.Testing of doctests in the sources finished, look at the ^ 222 | results in %BUILDDIR%/doctest/output.txt. 223 | goto end 224 | ) 225 | 226 | if "%1" == "xml" ( 227 | %SPHINXBUILD% -b xml %ALLSPHINXOPTS% %BUILDDIR%/xml 228 | if errorlevel 1 exit /b 1 229 | echo. 230 | echo.Build finished. The XML files are in %BUILDDIR%/xml. 231 | goto end 232 | ) 233 | 234 | if "%1" == "pseudoxml" ( 235 | %SPHINXBUILD% -b pseudoxml %ALLSPHINXOPTS% %BUILDDIR%/pseudoxml 236 | if errorlevel 1 exit /b 1 237 | echo. 238 | echo.Build finished. The pseudo-XML files are in %BUILDDIR%/pseudoxml. 239 | goto end 240 | ) 241 | 242 | :end 243 | -------------------------------------------------------------------------------- /docs/modules.rst: -------------------------------------------------------------------------------- 1 | django_zombodb 2 | ============== 3 | 4 | .. toctree:: 5 | :maxdepth: 4 6 | 7 | django_zombodb 8 | -------------------------------------------------------------------------------- /docs/searching.rst: -------------------------------------------------------------------------------- 1 | ========= 2 | Searching 3 | ========= 4 | 5 | On models with :py:class:`~django_zombodb.indexes.ZomboDBIndex`, use methods from :py:class:`~django_zombodb.querysets.SearchQuerySet`/:py:class:`~django_zombodb.querysets.SearchQuerySetMixin` to perform various kinds of Elasticsearch queries: 6 | 7 | query_string_search 8 | ------------------- 9 | 10 | The :py:meth:`~django_zombodb.querysets.SearchQuerySetMixin.query_string_search` method implements the simplest type of Elasticsearch queries: the ones with the `query string syntax `_. To use it, just pass as an argument a string that follows the query string syntax. 11 | 12 | .. code-block:: python 13 | 14 | Restaurant.objects.query_string_search("brasil~ AND steak*") 15 | 16 | dsl_search 17 | ---------- 18 | 19 | The query string syntax is user-friendly, but it's limited. For supporting all kinds of Elasticsearch queries, the recommended way is to use the :py:meth:`~django_zombodb.querysets.SearchQuerySetMixin.dsl_search` method. It accepts arguments of `elasticsearch-dsl-py `_ ``Query`` objects. Those objects have the same representation power of the `Elasticsearch JSON Query DSL `_. You can do "match", "term", and even compound queries like "bool". 20 | 21 | Here we're using the elasticsearch-dsl-py ``Q`` shortcut to create ``Query`` objects: 22 | 23 | .. code-block:: python 24 | 25 | from elasticsearch_dsl import Q as ElasticsearchQ 26 | 27 | query = ElasticsearchQ( 28 | 'bool', 29 | must=[ 30 | ElasticsearchQ('match', name='pizza'), 31 | ElasticsearchQ('match', street='school') 32 | ] 33 | ) 34 | Restaurant.objects.dsl_search(query) 35 | 36 | dict_search 37 | ----------- 38 | 39 | If you already have a Elasticsearch JSON query mounted as a ``dict``, use the :py:meth:`~django_zombodb.querysets.SearchQuerySetMixin.dict_search` method. The ``dict`` will be serialized using the ``JSONSerializer`` of `elasticsearch-py `_, the official Python Elasticsearch client. This means dict values of ``date``, ``datetime``, ``Decimal``, and ``UUID`` types will be correctly serialized. 40 | 41 | 42 | Validation 43 | ---------- 44 | 45 | If you're receiving queries from the end-user, particularly query string queries, you should call the search methods with ``validate=True``. This will perform Elasticsearch-side validation through the `Validate API `_. When doing that, :py:class:`~django_zombodb.exceptions.InvalidElasticsearchQuery` may be raised. 46 | 47 | .. code-block:: python 48 | 49 | from django_zombodb.exceptions import InvalidElasticsearchQuery 50 | 51 | queryset = Restaurant.objects.all() 52 | try: 53 | queryset = queryset.query_string_search("AND steak*", validate=True) 54 | except InvalidElasticsearchQuery: 55 | messages.error(request, "Invalid search query. Not filtering by search.") 56 | 57 | Sorting by score 58 | ---------------- 59 | 60 | By default, the resulting queryset from the search methods is unordered. You can get results ordered by Elasticsearch's score passing ``sort=True``. 61 | 62 | .. code-block:: python 63 | 64 | Restaurant.objects.query_string_search("brasil~ AND steak*", sort=True) 65 | 66 | Alternatively, if you want to combine with your own ``order_by``, you can use the method :py:meth:`~django_zombodb.querysets.SearchQuerySetMixin.annotate_score`: 67 | 68 | .. code-block:: python 69 | 70 | Restaurant.objects.query_string_search( 71 | "brazil* AND steak*" 72 | ).annotate_score( 73 | attr='zombodb_score' 74 | ).order_by('-zombodb_score', 'name', 'pk') 75 | 76 | Limiting 77 | -------- 78 | 79 | **It's a good practice to set a hard limit to the number of search results.** For most search use cases, you shouldn't need more than a certain number of results, either because users will only consume some of the high scoring results, or because documents with lower scores aren't relevant to your process. To limit the results, use the ``limit`` parameter on search methods: 80 | 81 | .. code-block:: python 82 | 83 | Restaurant.objects.query_string_search("brasil~ AND steak*", limit=1000) 84 | 85 | Lazy and Chainable 86 | ------------------ 87 | 88 | The search methods are like the traditional ``filter`` method: they return a regular Django ``QuerySet`` that supports all operations, and that's lazy and chainable. Therefore, you can do things like: 89 | 90 | .. code-block:: python 91 | 92 | Restaurant.objects.filter( 93 | name__startswith='Pizza' 94 | ).query_string_search( 95 | 'name:Hut' 96 | ).filter( 97 | street__contains='Road' 98 | ) 99 | 100 | .. warning:: 101 | 102 | It's fine to call ``filter``/``exclude``/etc. before and after search. If possible, the best would be using only a Elasticsearch query. However, it's definitely **slow** to call search methods multiple times on the same queryset! **Please avoid this**: 103 | 104 | .. code-block:: python 105 | 106 | Restaurant.objects.query_string_search( 107 | 'name:Pizza' 108 | ).query_string_search( 109 | 'name:Hut' 110 | ) 111 | 112 | While that may work as expected, it's `extremely inneficient `_. Instead, use compound queries like `"bool" `_. They'll be much faster. Note that "bool" queries might be quite confusing to implement. Check tutorials about them, like `this one `_. 113 | 114 | 115 | Missing features 116 | ---------------- 117 | 118 | Currently django-zombodb doesn't support ZomboDB's `offset and sort functions `_ that work on the Elasticsearch side. Regular SQL LIMIT/OFFSET/ORDER BY works fine, therefore traditional ``QuerySet`` operations work, but aren't as performant as doing the same on ES side. 119 | -------------------------------------------------------------------------------- /example/README.md: -------------------------------------------------------------------------------- 1 | ## Example Project for django_zombodb 2 | 3 | This example is provided as a convenience feature to allow potential users to try the app straight from the app repo without having to create a Django project. 4 | 5 | It can also be used to develop the app in place. 6 | 7 | To run this example, follow these instructions: 8 | 9 | 1. Navigate to the `example` directory 10 | 2. Install the requirements for the package: 11 | 12 | pip install -r requirements.txt 13 | 14 | 3. Create the Postgres user and DB: 15 | 16 | sudo su - postgres -c "psql -c \"CREATE USER django_zombodb WITH PASSWORD 'password' SUPERUSER;\"" 17 | createdb django_zombodb 18 | 19 | 4. Run a Elasticsearch cluster: 20 | 21 | elasticsearch 22 | 23 | 5. Apply migrations: 24 | 25 | python manage.py migrate 26 | 27 | 6. Load sample data from PromptCloud's Restaurants on Yellowpages.com [dataset](https://www.kaggle.com/PromptCloudHQ/restaurants-on-yellowpagescom): 28 | 29 | python manage.py filldata 30 | 31 | 7. Create a Django admin user: 32 | 33 | python manage.py createsuperuser 34 | 35 | 8. Run the server: 36 | 37 | python manage.py runserver 38 | 39 | 9. Log in into admin and run searches at `http://127.0.0.1:8000/admin/restaurants/restaurant/`: 40 | 41 | ![Django admin screenshot](https://user-images.githubusercontent.com/397989/52665839-63ea4300-2eeb-11e9-9039-7d05bff0ac3a.png) 42 | 43 | 44 | ### Notes: 45 | - `django_zombodb` Postgres user needs to be a `SUPERUSER` for activating the `zombodb` extension on the newly created database. This is handled by the operation `django_zombodb.operations.ZomboDBExtension()` on the `0002` migration. If you wish you can `ALTER ROLE django_zombodb NOSUPERUSER` after running the migrations. 46 | - `ZOMBODB_ELASTICSEARCH_URL` on settings.py defines your Elasticsearch URL. It's set to `http://localhost:9200/`. Change it if necessary. 47 | -------------------------------------------------------------------------------- /example/example/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vintasoftware/django-zombodb/2391f27709819bd135e3d0dd571ef42e6996a201/example/example/__init__.py -------------------------------------------------------------------------------- /example/example/settings.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | from decouple import config 4 | 5 | 6 | # django-zombodb settings 7 | 8 | ZOMBODB_ELASTICSEARCH_URL = 'http://localhost:9200/' 9 | 10 | # Django settings 11 | 12 | BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) 13 | 14 | # Quick-start development settings - unsuitable for production 15 | # See https://docs.djangoproject.com/en/dev/howto/deployment/checklist/ 16 | 17 | SECRET_KEY = 'secret' # noqa 18 | 19 | DEBUG = True 20 | 21 | ALLOWED_HOSTS = [] 22 | 23 | INSTALLED_APPS = [ 24 | 'django.contrib.admin', 25 | 'django.contrib.auth', 26 | 'django.contrib.contenttypes', 27 | 'django.contrib.sessions', 28 | 'django.contrib.messages', 29 | 'django.contrib.staticfiles', 30 | 31 | 'django_zombodb', 32 | 33 | 'restaurants', 34 | ] 35 | 36 | MIDDLEWARE = [ 37 | 'django.middleware.security.SecurityMiddleware', 38 | 'django.contrib.sessions.middleware.SessionMiddleware', 39 | 'django.middleware.common.CommonMiddleware', 40 | 'django.middleware.csrf.CsrfViewMiddleware', 41 | 'django.contrib.auth.middleware.AuthenticationMiddleware', 42 | 'django.contrib.messages.middleware.MessageMiddleware', 43 | 'django.middleware.clickjacking.XFrameOptionsMiddleware', 44 | ] 45 | 46 | ROOT_URLCONF = 'example.urls' 47 | 48 | TEMPLATES = [ 49 | { 50 | 'BACKEND': 'django.template.backends.django.DjangoTemplates', 51 | 'DIRS': [os.path.join(BASE_DIR, 'templates'), ], 52 | 'APP_DIRS': True, 53 | 'OPTIONS': { 54 | 'context_processors': [ 55 | 'django.template.context_processors.debug', 56 | 'django.template.context_processors.request', 57 | 'django.contrib.auth.context_processors.auth', 58 | 'django.contrib.messages.context_processors.messages', 59 | ], 60 | }, 61 | }, 62 | ] 63 | 64 | WSGI_APPLICATION = 'example.wsgi.application' 65 | 66 | DATABASES = { 67 | 'default': { 68 | 'ENGINE': 'django.db.backends.postgresql_psycopg2', 69 | 'USER': 'django_zombodb', 70 | 'NAME': 'django_zombodb', 71 | 'PASSWORD': 'password', 72 | 'HOST': '127.0.0.1', 73 | 'PORT': config('POSTGRES_PORT', default='5432'), 74 | 'ATOMIC_REQUESTS': False, # False gives better stacktraces 75 | } 76 | } 77 | 78 | AUTH_PASSWORD_VALIDATORS = [ 79 | { 80 | 'NAME': 'django.contrib.auth.password_validation.UserAttributeSimilarityValidator', 81 | }, 82 | { 83 | 'NAME': 'django.contrib.auth.password_validation.MinimumLengthValidator', 84 | }, 85 | { 86 | 'NAME': 'django.contrib.auth.password_validation.CommonPasswordValidator', 87 | }, 88 | { 89 | 'NAME': 'django.contrib.auth.password_validation.NumericPasswordValidator', 90 | }, 91 | ] 92 | 93 | LANGUAGE_CODE = 'en-us' 94 | 95 | TIME_ZONE = 'UTC' 96 | 97 | USE_I18N = True 98 | 99 | USE_L10N = True 100 | 101 | USE_TZ = True 102 | 103 | STATIC_URL = '/static/' 104 | -------------------------------------------------------------------------------- /example/example/urls.py: -------------------------------------------------------------------------------- 1 | from django.conf.urls import url 2 | from django.contrib import admin 3 | 4 | 5 | urlpatterns = [ 6 | url(r'^admin/', admin.site.urls), 7 | ] 8 | -------------------------------------------------------------------------------- /example/example/wsgi.py: -------------------------------------------------------------------------------- 1 | """ 2 | WSGI config for example project. 3 | 4 | It exposes the WSGI callable as a module-level variable named ``application``. 5 | 6 | For more information on this file, see 7 | https://docs.djangoproject.com/en/1.9/howto/deployment/wsgi/ 8 | """ 9 | 10 | import os 11 | 12 | from django.core.wsgi import get_wsgi_application 13 | 14 | os.environ.setdefault("DJANGO_SETTINGS_MODULE", "example.settings") 15 | 16 | application = get_wsgi_application() 17 | -------------------------------------------------------------------------------- /example/manage.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | import os 3 | import sys 4 | 5 | 6 | if __name__ == "__main__": 7 | os.environ.setdefault("DJANGO_SETTINGS_MODULE", "example.settings") 8 | 9 | from django.core.management import execute_from_command_line 10 | 11 | execute_from_command_line(sys.argv) 12 | -------------------------------------------------------------------------------- /example/requirements.txt: -------------------------------------------------------------------------------- 1 | # Your app requirements. 2 | -r ../requirements/base.in 3 | -r ../requirements/dev.in 4 | 5 | # Your app in editable mode. 6 | -e ../ 7 | -------------------------------------------------------------------------------- /example/restaurants/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vintasoftware/django-zombodb/2391f27709819bd135e3d0dd571ef42e6996a201/example/restaurants/__init__.py -------------------------------------------------------------------------------- /example/restaurants/admin.py: -------------------------------------------------------------------------------- 1 | from django.contrib import admin 2 | 3 | from django_zombodb.admin_mixins import ZomboDBAdminMixin 4 | from restaurants.models import Restaurant 5 | 6 | 7 | @admin.register(Restaurant) 8 | class RestaurantAdmin(ZomboDBAdminMixin, admin.ModelAdmin): 9 | model = Restaurant 10 | list_display = ( 11 | 'name', 12 | '_zombodb_score', 13 | 'street', 14 | 'zip_code', 15 | 'city', 16 | 'state', 17 | 'phone', 18 | 'categories', 19 | ) 20 | max_search_results = 100 21 | 22 | class Media: 23 | js = ('django_zombodb/js/hide_show_score.js',) 24 | -------------------------------------------------------------------------------- /example/restaurants/apps.py: -------------------------------------------------------------------------------- 1 | from django.apps import AppConfig 2 | 3 | 4 | class RestaurantsConfig(AppConfig): 5 | name = 'restaurants' 6 | -------------------------------------------------------------------------------- /example/restaurants/management/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vintasoftware/django-zombodb/2391f27709819bd135e3d0dd571ef42e6996a201/example/restaurants/management/__init__.py -------------------------------------------------------------------------------- /example/restaurants/management/commands/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vintasoftware/django-zombodb/2391f27709819bd135e3d0dd571ef42e6996a201/example/restaurants/management/commands/__init__.py -------------------------------------------------------------------------------- /example/restaurants/management/commands/filldata.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | from django.conf import settings 4 | from django.core.management.base import BaseCommand 5 | from django.db import connection 6 | 7 | from restaurants.models import Restaurant 8 | 9 | 10 | class Command(BaseCommand): 11 | help = 'Closes the specified poll for voting' 12 | 13 | def handle(self, *args, **options): 14 | csv_file_path = os.path.join( 15 | settings.BASE_DIR, 16 | 'restaurants', 17 | 'data', 18 | 'yellowpages_com-restaurant_sample.csv') 19 | 20 | with connection.cursor() as cursor: 21 | cursor.execute(''' 22 | CREATE TEMPORARY TABLE t ( 23 | id uuid, 24 | url text, 25 | name text, 26 | street text, 27 | zip_code text, 28 | city text, 29 | state text, 30 | phone text, 31 | email text, 32 | website text, 33 | categories text 34 | ); 35 | ''') 36 | 37 | with open(csv_file_path) as csv_file: 38 | cursor.copy_expert( 39 | "COPY t FROM stdin DELIMITER ',' CSV HEADER", csv_file) 40 | 41 | cursor.execute(''' 42 | INSERT INTO restaurants_restaurant( 43 | id, 44 | url, 45 | name, 46 | street, 47 | zip_code, 48 | city, 49 | state, 50 | phone, 51 | email, 52 | website, 53 | categories 54 | ) 55 | SELECT 56 | id, 57 | url, 58 | name, 59 | street, 60 | zip_code, 61 | city, 62 | state, 63 | phone, 64 | email, 65 | COALESCE(website, ''), 66 | regexp_split_to_array(categories, '\\s*,\\s*') 67 | FROM t; 68 | ''') 69 | 70 | count = Restaurant.objects.count() 71 | self.stdout.write( 72 | self.style.SUCCESS('Successfully created %s Restaurants' % count)) 73 | -------------------------------------------------------------------------------- /example/restaurants/migrations/0001_squashed_0004_auto_20190718_2025.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 2.2.3 on 2019-10-02 20:08 2 | 3 | import django.contrib.postgres.fields 4 | from django.db import migrations, models 5 | import django_zombodb.indexes 6 | import django_zombodb.operations 7 | import uuid 8 | 9 | 10 | class Migration(migrations.Migration): 11 | initial = True 12 | 13 | dependencies = [ 14 | ] 15 | 16 | operations = [ 17 | migrations.CreateModel( 18 | name='Restaurant', 19 | fields=[ 20 | ('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)), 21 | ('url', models.URLField()), 22 | ('name', models.TextField()), 23 | ('street', models.TextField()), 24 | ('zip_code', models.TextField()), 25 | ('city', models.TextField()), 26 | ('state', models.TextField()), 27 | ('phone', models.TextField()), 28 | ('email', models.EmailField(max_length=254)), 29 | ('website', models.URLField(blank=True)), 30 | ('categories', django.contrib.postgres.fields.ArrayField(base_field=models.TextField(), size=None)), 31 | ], 32 | ), 33 | django_zombodb.operations.ZomboDBExtension( 34 | ), 35 | migrations.AddIndex( 36 | model_name='restaurant', 37 | index=django_zombodb.indexes.ZomboDBIndex(field_mapping={'name': {'analyzer': 'fulltext_with_shingles', 'copy_to': 'zdb_all', 'search_analyzer': 'fulltext_with_shingles_search', 'type': 'text'}, 'street': {'analyzer': 'fulltext_with_shingles', 'copy_to': 'zdb_all', 'search_analyzer': 'fulltext_with_shingles_search', 'type': 'text'}}, fields=['name', 'street', 'zip_code', 'city', 'state', 'phone', 'categories'], name='restaurants_name_72ba02_zombodb'), 38 | ), 39 | ] 40 | -------------------------------------------------------------------------------- /example/restaurants/migrations/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vintasoftware/django-zombodb/2391f27709819bd135e3d0dd571ef42e6996a201/example/restaurants/migrations/__init__.py -------------------------------------------------------------------------------- /example/restaurants/models.py: -------------------------------------------------------------------------------- 1 | import uuid 2 | 3 | from django.contrib.postgres.fields import ArrayField 4 | from django.db import models 5 | 6 | from django_zombodb.indexes import ZomboDBIndex 7 | from django_zombodb.querysets import SearchQuerySet 8 | 9 | 10 | class Restaurant(models.Model): 11 | id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) 12 | url = models.URLField() 13 | name = models.TextField() 14 | street = models.TextField() 15 | zip_code = models.TextField() 16 | city = models.TextField() 17 | state = models.TextField() 18 | phone = models.TextField() 19 | email = models.EmailField() 20 | website = models.URLField(blank=True) 21 | categories = ArrayField(models.TextField()) 22 | 23 | objects = models.Manager.from_queryset(SearchQuerySet)() 24 | 25 | class Meta: 26 | indexes = [ 27 | ZomboDBIndex( 28 | fields=[ 29 | 'name', 30 | 'street', 31 | 'zip_code', 32 | 'city', 33 | 'state', 34 | 'phone', 35 | 'categories', 36 | ], 37 | field_mapping={ 38 | 'name': {"type": "text", 39 | "copy_to": "zdb_all", 40 | "analyzer": "fulltext_with_shingles", 41 | "search_analyzer": "fulltext_with_shingles_search"}, 42 | 'street': {"type": "text", 43 | "copy_to": "zdb_all", 44 | "analyzer": "fulltext_with_shingles", 45 | "search_analyzer": "fulltext_with_shingles_search"}, 46 | } 47 | ) 48 | ] 49 | 50 | def __str__(self): 51 | return self.name 52 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | # 2 | # This file is autogenerated by pip-compile 3 | # To update, run: 4 | # 5 | # pip-compile --output-file=requirements.txt requirements/base.in requirements/prod.in 6 | # 7 | elasticsearch-dsl>=6.4.0 8 | elasticsearch>=6.4.0 9 | psycopg2>=2.8.3 10 | python-dateutil>=2.8.0 # via elasticsearch-dsl 11 | six>=1.12.0 # via elasticsearch-dsl, python-dateutil 12 | urllib3>=1.25.6 # via elasticsearch 13 | -------------------------------------------------------------------------------- /requirements/base.in: -------------------------------------------------------------------------------- 1 | # Core requirements for using this application 2 | 3 | Django 4 | elasticsearch-dsl<7 5 | elasticsearch<7 6 | psycopg2 7 | -------------------------------------------------------------------------------- /requirements/dev.in: -------------------------------------------------------------------------------- 1 | # Additional requirements for development of this application 2 | 3 | bumpversion==0.5.3 # Update version command line 4 | pip-tools # Requirements file management 5 | tox # virtualenv management for tests 6 | tox-battery # Makes tox aware of requirements file changes 7 | wheel==0.38.1 # For generation of wheels for PyPI 8 | python-decouple # For settings.py 9 | -------------------------------------------------------------------------------- /requirements/dev.txt: -------------------------------------------------------------------------------- 1 | # 2 | # This file is autogenerated by pip-compile 3 | # To update, run: 4 | # 5 | # pip-compile --output-file=requirements/dev.txt requirements/base.in requirements/dev.in requirements/quality.in 6 | # 7 | bumpversion==0.5.3 8 | # via -r dev.in 9 | click==7.0 10 | # via pip-tools 11 | filelock==3.0.12 12 | # via tox 13 | packaging==19.2 14 | # via tox 15 | pip-tools==4.1.0 16 | # via -r dev.in 17 | pluggy==0.13.0 18 | # via tox 19 | py==1.8.0 20 | # via tox 21 | pyparsing==2.4.2 22 | # via packaging 23 | python-decouple==3.1 24 | # via -r dev.in 25 | six==1.12.0 26 | # via 27 | # packaging 28 | # pip-tools 29 | # tox 30 | toml==0.10.0 31 | # via tox 32 | tox==3.14.0 33 | # via 34 | # -r dev.in 35 | # tox-battery 36 | tox-battery==0.5.1 37 | # via -r dev.in 38 | virtualenv==16.7.5 39 | # via tox 40 | wheel==0.38.1 41 | # via -r dev.in 42 | -------------------------------------------------------------------------------- /requirements/doc.in: -------------------------------------------------------------------------------- 1 | # Requirements for documentation validation 2 | 3 | -r dev.in 4 | doc8 # reStructuredText style checker 5 | readme_renderer # Validates README.rst for usage on PyPI 6 | Sphinx # Documentation builder 7 | sphinx-rtd-theme # Sphinx Read the Docs theme 8 | -------------------------------------------------------------------------------- /requirements/doc.txt: -------------------------------------------------------------------------------- 1 | # 2 | # This file is autogenerated by pip-compile 3 | # To update, run: 4 | # 5 | # pip-compile --output-file=requirements/doc.txt requirements/base.in requirements/doc.in 6 | # 7 | alabaster==0.7.12 8 | # via sphinx 9 | babel==2.7.0 10 | # via sphinx 11 | bleach==3.1.0 12 | # via readme-renderer 13 | bumpversion==0.5.3 14 | # via -r dev.in 15 | certifi==2019.9.11 16 | # via requests 17 | chardet==3.0.4 18 | # via 19 | # doc8 20 | # requests 21 | click==7.0 22 | # via pip-tools 23 | doc8==0.8.0 24 | # via -r doc.in 25 | docutils==0.15.2 26 | # via 27 | # doc8 28 | # readme-renderer 29 | # restructuredtext-lint 30 | # sphinx 31 | filelock==3.0.12 32 | # via tox 33 | idna==2.8 34 | # via requests 35 | imagesize==1.1.0 36 | # via sphinx 37 | jinja2==2.10.1 38 | # via sphinx 39 | markupsafe==1.1.1 40 | # via jinja2 41 | packaging==19.2 42 | # via 43 | # sphinx 44 | # tox 45 | pbr==5.4.3 46 | # via stevedore 47 | pip-tools==4.1.0 48 | # via -r dev.in 49 | pluggy==0.13.0 50 | # via tox 51 | py==1.8.0 52 | # via tox 53 | pygments==2.4.2 54 | # via 55 | # readme-renderer 56 | # sphinx 57 | pyparsing==2.4.2 58 | # via packaging 59 | python-decouple==3.1 60 | # via -r dev.in 61 | pytz==2019.2 62 | # via babel 63 | readme-renderer==24.0 64 | # via -r doc.in 65 | requests==2.22.0 66 | # via sphinx 67 | restructuredtext-lint==1.3.0 68 | # via doc8 69 | six==1.12.0 70 | # via 71 | # bleach 72 | # doc8 73 | # packaging 74 | # pip-tools 75 | # readme-renderer 76 | # stevedore 77 | # tox 78 | snowballstemmer==1.9.1 79 | # via sphinx 80 | sphinx==2.2.0 81 | # via 82 | # -r doc.in 83 | # sphinx-rtd-theme 84 | sphinx-rtd-theme==0.4.3 85 | # via -r doc.in 86 | sphinxcontrib-applehelp==1.0.1 87 | # via sphinx 88 | sphinxcontrib-devhelp==1.0.1 89 | # via sphinx 90 | sphinxcontrib-htmlhelp==1.0.2 91 | # via sphinx 92 | sphinxcontrib-jsmath==1.0.1 93 | # via sphinx 94 | sphinxcontrib-qthelp==1.0.2 95 | # via sphinx 96 | sphinxcontrib-serializinghtml==1.1.3 97 | # via sphinx 98 | stevedore==1.31.0 99 | # via doc8 100 | toml==0.10.0 101 | # via tox 102 | tox==3.14.0 103 | # via 104 | # -r dev.in 105 | # tox-battery 106 | tox-battery==0.5.1 107 | # via -r dev.in 108 | urllib3==1.25.6 109 | # via requests 110 | virtualenv==16.7.5 111 | # via tox 112 | webencodings==0.5.1 113 | # via bleach 114 | wheel==0.38.1 115 | # via -r dev.in 116 | 117 | # The following packages are considered to be unsafe in a requirements file: 118 | # setuptools 119 | -------------------------------------------------------------------------------- /requirements/prod.in: -------------------------------------------------------------------------------- 1 | # Additional requirements go here 2 | -------------------------------------------------------------------------------- /requirements/quality.in: -------------------------------------------------------------------------------- 1 | # Requirements for code quality checks 2 | 3 | isort # imports sorter for python 4 | pre-commit # pre-commit hooks for git 5 | prospector==1.1.6.2 # linter wrapper for python 6 | pyyaml>=4.2b1 # force safer pyyaml version 7 | -------------------------------------------------------------------------------- /requirements/quality.txt: -------------------------------------------------------------------------------- 1 | # 2 | # This file is autogenerated by pip-compile 3 | # To update, run: 4 | # 5 | # pip-compile --output-file=requirements/quality.txt requirements/quality.in 6 | # 7 | aspy.yaml==1.3.0 # via pre-commit 8 | astroid==2.0.4 # via prospector, pylint, pylint-celery, pylint-flask, requirements-detector 9 | cfgv==2.0.1 # via pre-commit 10 | dodgy==0.1.9 # via prospector 11 | identify==1.4.7 # via pre-commit 12 | importlib-metadata==0.23 # via pre-commit 13 | isort==4.3.21 14 | lazy-object-proxy==1.4.2 # via astroid 15 | mccabe==0.6.1 # via prospector, pylint 16 | more-itertools==7.2.0 # via zipp 17 | nodeenv==1.3.3 # via pre-commit 18 | pep8-naming==0.4.1 # via prospector 19 | pre-commit==1.18.3 20 | prospector==1.1.6.2 21 | pycodestyle==2.4.0 # via prospector 22 | pydocstyle==4.0.1 # via prospector 23 | pyflakes==1.6.0 # via prospector 24 | pylint-celery==0.3 # via prospector 25 | pylint-django==2.0.2 # via prospector 26 | pylint-flask==0.5 # via prospector 27 | pylint-plugin-utils==0.6 # via prospector, pylint-celery, pylint-django, pylint-flask 28 | pylint==2.1.1 # via prospector, pylint-celery, pylint-django, pylint-flask, pylint-plugin-utils 29 | pyyaml==5.1.2 30 | requirements-detector==0.6 # via prospector 31 | setoptconf==0.2.0 # via prospector 32 | six==1.12.0 # via astroid, cfgv, pre-commit 33 | snowballstemmer==1.9.1 # via pydocstyle 34 | toml==0.10.0 # via pre-commit 35 | virtualenv==16.7.5 # via pre-commit 36 | wrapt==1.11.2 # via astroid 37 | zipp==0.6.0 # via importlib-metadata 38 | -------------------------------------------------------------------------------- /requirements/test.in: -------------------------------------------------------------------------------- 1 | # Requirements for test runs. 2 | 3 | coverage==4.4.1 # Analyzes test coverage 4 | codecov>=2.0.0 # Integration with codecov 5 | mock>=1.0.1 # Mocks for unittests 6 | python-decouple # For settings.py 7 | -------------------------------------------------------------------------------- /requirements/test.txt: -------------------------------------------------------------------------------- 1 | # 2 | # This file is autogenerated by pip-compile 3 | # To update, run: 4 | # 5 | # pip-compile --output-file=requirements/test.txt requirements/base.in requirements/test.in 6 | # 7 | certifi==2019.9.11 # via requests 8 | chardet==3.0.4 # via requests 9 | codecov==2.0.15 10 | coverage==4.4.1 11 | elasticsearch-dsl==6.4.0 12 | elasticsearch==6.4.0 13 | idna==2.8 # via requests 14 | mock==3.0.5 15 | psycopg2==2.8.3 16 | python-dateutil==2.8.0 # via elasticsearch-dsl 17 | python-decouple==3.1 18 | requests==2.22.0 # via codecov 19 | six==1.12.0 # via elasticsearch-dsl, mock, python-dateutil 20 | urllib3==1.25.6 # via elasticsearch, requests 21 | -------------------------------------------------------------------------------- /requirements/travis.in: -------------------------------------------------------------------------------- 1 | # Requirements for running tests in Travis 2 | 3 | codecov # Code coverage reporting 4 | tox # Virtualenv management for tests 5 | tox-travis # Makes tox aware of travis env vars 6 | -------------------------------------------------------------------------------- /requirements/travis.txt: -------------------------------------------------------------------------------- 1 | # 2 | # This file is autogenerated by pip-compile 3 | # To update, run: 4 | # 5 | # pip-compile --output-file=requirements/travis.txt requirements/travis.in 6 | # 7 | certifi==2019.9.11 # via requests 8 | chardet==3.0.4 # via requests 9 | codecov==2.0.15 10 | coverage==4.5.4 # via codecov 11 | filelock==3.0.12 # via tox 12 | idna==2.8 # via requests 13 | importlib-metadata==0.23 # via pluggy, tox 14 | more-itertools==7.2.0 # via zipp 15 | packaging==19.2 # via tox 16 | pluggy==0.13.0 # via tox 17 | py==1.8.0 # via tox 18 | pyparsing==2.4.2 # via packaging 19 | requests==2.22.0 # via codecov 20 | six==1.12.0 # via packaging, tox 21 | toml==0.10.0 # via tox 22 | tox-travis==0.12 23 | tox==3.14.0 24 | urllib3==1.25.6 # via requests 25 | virtualenv==16.7.5 # via tox 26 | zipp==0.6.0 # via importlib-metadata 27 | -------------------------------------------------------------------------------- /runtests.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 3 | from __future__ import absolute_import, unicode_literals 4 | 5 | import os 6 | import sys 7 | 8 | from django.core.management import execute_from_command_line 9 | 10 | 11 | def runtests(): 12 | os.environ.setdefault("DJANGO_SETTINGS_MODULE", "tests.settings") 13 | argv = sys.argv[:1] + ['test'] + sys.argv[1:] 14 | execute_from_command_line(argv) 15 | 16 | 17 | if __name__ == '__main__': 18 | runtests() 19 | -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [bumpversion] 2 | current_version = 0.1.0 3 | commit = True 4 | tag = True 5 | 6 | [bumpversion:file:setup.py] 7 | 8 | [bumpversion:file:django_zombodb/__init__.py] 9 | 10 | [wheel] 11 | universal = 1 12 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | import os 4 | import re 5 | import sys 6 | 7 | 8 | try: 9 | from setuptools import setup 10 | except ImportError: 11 | from distutils.core import setup 12 | 13 | 14 | def get_version(*file_paths): 15 | """Retrieves the version from django_zombodb/__init__.py""" 16 | filename = os.path.join(os.path.dirname(__file__), *file_paths) 17 | version_file = open(filename).read() 18 | version_match = re.search(r"^__version__ = ['\"]([^'\"]*)['\"]", 19 | version_file, re.M) 20 | if version_match: 21 | return version_match.group(1) 22 | raise RuntimeError('Unable to find version string.') 23 | 24 | 25 | version = get_version("django_zombodb", "__init__.py") 26 | 27 | 28 | if sys.argv[-1] == 'publish': 29 | try: 30 | import wheel 31 | print("Wheel version: ", wheel.__version__) 32 | except ImportError: 33 | print('Wheel library missing. Please run "pip install wheel"') 34 | sys.exit() 35 | os.system('python setup.py sdist upload') 36 | os.system('python setup.py bdist_wheel upload') 37 | sys.exit() 38 | 39 | if sys.argv[-1] == 'tag': 40 | print("Tagging the version on git:") 41 | os.system("git tag -a %s -m 'version %s'" % (version, version)) 42 | os.system("git push --tags") 43 | sys.exit() 44 | 45 | readme = open('README.rst').read() 46 | history = open('CHANGELOG.rst').read().replace('.. :changelog:', '') 47 | 48 | setup( 49 | name='django-zombodb', 50 | version=version, 51 | description="""Easy Django integration with Elasticsearch through ZomboDB Postgres Extension""", 52 | long_description=readme + '\n\n' + history, 53 | author='Flávio Juvenal', 54 | author_email='flavio@vinta.com.br', 55 | url='https://github.com/vintasoftware/django-zombodb', 56 | packages=[ 57 | 'django_zombodb', 58 | ], 59 | include_package_data=True, 60 | install_requires=[ 61 | 'elasticsearch-dsl>=6.3.1', 62 | 'elasticsearch>=6.3.1', 63 | 'psycopg2>=2.7.7', 64 | ], 65 | zip_safe=False, 66 | keywords='django-zombodb', 67 | classifiers=[ 68 | 'Development Status :: 4 - Beta', 69 | 'Framework :: Django :: 2.0', 70 | 'Framework :: Django :: 2.1', 71 | 'Intended Audience :: Developers', 72 | 'License :: OSI Approved :: MIT License', 73 | 'Natural Language :: English', 74 | 'Programming Language :: Python :: 3', 75 | 'Programming Language :: Python :: 3.5', 76 | 'Programming Language :: Python :: 3.6', 77 | 'Programming Language :: Python :: 3.7', 78 | ], 79 | ) 80 | -------------------------------------------------------------------------------- /test_manage.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | from __future__ import absolute_import, unicode_literals 4 | 5 | import os 6 | import sys 7 | 8 | 9 | if __name__ == "__main__": 10 | os.environ.setdefault("DJANGO_SETTINGS_MODULE", "tests.settings") 11 | from django.core.management import execute_from_command_line 12 | 13 | execute_from_command_line(sys.argv) 14 | -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vintasoftware/django-zombodb/2391f27709819bd135e3d0dd571ef42e6996a201/tests/__init__.py -------------------------------------------------------------------------------- /tests/migrations/0001_setup_extensions.py: -------------------------------------------------------------------------------- 1 | from django.db import migrations 2 | 3 | from django_zombodb.operations import ZomboDBExtension 4 | 5 | 6 | class Migration(migrations.Migration): 7 | 8 | operations = [ 9 | ZomboDBExtension(), 10 | ] 11 | -------------------------------------------------------------------------------- /tests/migrations/0002_datetimearraymodel_integerarraymodel.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 2.1.5 on 2019-02-11 19:30 2 | 3 | import django.contrib.postgres.fields 4 | from django.db import migrations, models 5 | 6 | 7 | class Migration(migrations.Migration): 8 | 9 | initial = True 10 | 11 | dependencies = [ 12 | ('tests', '0001_setup_extensions'), 13 | ] 14 | 15 | operations = [ 16 | migrations.CreateModel( 17 | name='DateTimeArrayModel', 18 | fields=[ 19 | ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), 20 | ('datetimes', django.contrib.postgres.fields.ArrayField(base_field=models.DateTimeField(), size=None)), 21 | ('dates', django.contrib.postgres.fields.ArrayField(base_field=models.DateField(), size=None)), 22 | ('times', django.contrib.postgres.fields.ArrayField(base_field=models.TimeField(), size=None)), 23 | ], 24 | ), 25 | migrations.CreateModel( 26 | name='IntegerArrayModel', 27 | fields=[ 28 | ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), 29 | ('field', django.contrib.postgres.fields.ArrayField(base_field=models.IntegerField(), blank=True, default=list, size=None)), 30 | ], 31 | ), 32 | ] 33 | -------------------------------------------------------------------------------- /tests/migrations/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vintasoftware/django-zombodb/2391f27709819bd135e3d0dd571ef42e6996a201/tests/migrations/__init__.py -------------------------------------------------------------------------------- /tests/models.py: -------------------------------------------------------------------------------- 1 | from django.contrib.postgres.fields import ArrayField 2 | from django.db import models 3 | 4 | 5 | class IntegerArrayModel(models.Model): 6 | field = ArrayField(models.IntegerField(), default=list, blank=True) 7 | 8 | 9 | class DateTimeArrayModel(models.Model): 10 | datetimes = ArrayField(models.DateTimeField()) 11 | dates = ArrayField(models.DateField()) 12 | times = ArrayField(models.TimeField()) 13 | -------------------------------------------------------------------------------- /tests/restaurants/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vintasoftware/django-zombodb/2391f27709819bd135e3d0dd571ef42e6996a201/tests/restaurants/__init__.py -------------------------------------------------------------------------------- /tests/restaurants/admin.py: -------------------------------------------------------------------------------- 1 | from django.contrib import admin 2 | 3 | from django_zombodb.admin_mixins import ZomboDBAdminMixin 4 | 5 | from .models import Restaurant 6 | 7 | 8 | class RestaurantAdmin(ZomboDBAdminMixin, admin.ModelAdmin): 9 | model = Restaurant 10 | list_display = ( 11 | 'name', 12 | '_zombodb_score', 13 | 'street', 14 | 'zip_code', 15 | 'city', 16 | 'state', 17 | 'phone', 18 | 'categories', 19 | ) 20 | 21 | class Media: 22 | js = ('django_zombodb/js/hide_show_score.js',) 23 | 24 | 25 | admin.site.register(Restaurant, RestaurantAdmin) 26 | -------------------------------------------------------------------------------- /tests/restaurants/apps.py: -------------------------------------------------------------------------------- 1 | from django.apps import AppConfig 2 | 3 | 4 | class RestaurantsConfig(AppConfig): 5 | name = 'restaurants' 6 | -------------------------------------------------------------------------------- /tests/restaurants/migrations/0001_initial.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 2.0.13 on 2019-02-12 15:29 2 | 3 | import django.contrib.postgres.fields 4 | from django.db import migrations, models 5 | import django_zombodb.indexes 6 | import uuid 7 | 8 | 9 | class Migration(migrations.Migration): 10 | 11 | initial = True 12 | 13 | dependencies = [ 14 | ('tests', '0001_setup_extensions'), 15 | ] 16 | 17 | operations = [ 18 | migrations.CreateModel( 19 | name='Restaurant', 20 | fields=[ 21 | ('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)), 22 | ('url', models.URLField()), 23 | ('name', models.TextField()), 24 | ('street', models.TextField()), 25 | ('zip_code', models.TextField()), 26 | ('city', models.TextField()), 27 | ('state', models.TextField()), 28 | ('phone', models.TextField()), 29 | ('email', models.EmailField(max_length=254)), 30 | ('website', models.URLField(blank=True)), 31 | ('categories', django.contrib.postgres.fields.ArrayField(base_field=models.TextField(), size=None)), 32 | ], 33 | ), 34 | migrations.CreateModel( 35 | name='RestaurantNoIndex', 36 | fields=[ 37 | ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), 38 | ('name', models.TextField()), 39 | ], 40 | ), 41 | migrations.AddIndex( 42 | model_name='restaurant', 43 | index=models.Index(fields=['url'], name='other-index'), 44 | ), 45 | migrations.AddIndex( 46 | model_name='restaurant', 47 | index=django_zombodb.indexes.ZomboDBIndex(fields=['name', 'street', 'zip_code', 'city', 'state', 'phone', 'email', 'website', 'categories'], name='restaurants_name_f38813_zombodb'), 48 | ), 49 | ] 50 | -------------------------------------------------------------------------------- /tests/restaurants/migrations/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vintasoftware/django-zombodb/2391f27709819bd135e3d0dd571ef42e6996a201/tests/restaurants/migrations/__init__.py -------------------------------------------------------------------------------- /tests/restaurants/models.py: -------------------------------------------------------------------------------- 1 | import uuid 2 | 3 | from django.contrib.postgres.fields import ArrayField 4 | from django.db import models 5 | from django.db.models.indexes import Index 6 | 7 | from django_zombodb.indexes import ZomboDBIndex 8 | from django_zombodb.querysets import SearchQuerySet 9 | 10 | 11 | class Restaurant(models.Model): 12 | id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) 13 | url = models.URLField() 14 | name = models.TextField() 15 | street = models.TextField() 16 | zip_code = models.TextField() 17 | city = models.TextField() 18 | state = models.TextField() 19 | phone = models.TextField() 20 | email = models.EmailField() 21 | website = models.URLField(blank=True) 22 | categories = ArrayField(models.TextField()) 23 | 24 | objects = models.Manager.from_queryset(SearchQuerySet)() 25 | 26 | class Meta: 27 | indexes = [ 28 | Index(name='other-index', fields=['url']), 29 | ZomboDBIndex( 30 | fields=[ 31 | 'name', 32 | 'street', 33 | 'zip_code', 34 | 'city', 35 | 'state', 36 | 'phone', 37 | 'email', 38 | 'website', 39 | 'categories', 40 | ] 41 | ), 42 | ] 43 | 44 | def __str__(self): 45 | return self.name 46 | 47 | 48 | class RestaurantNoIndex(models.Model): 49 | name = models.TextField() 50 | 51 | objects = models.Manager.from_queryset(SearchQuerySet)() 52 | 53 | def __str__(self): 54 | return self.name 55 | -------------------------------------------------------------------------------- /tests/settings.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 2 | from __future__ import absolute_import, unicode_literals 3 | 4 | from decouple import config 5 | 6 | 7 | ZOMBODB_ELASTICSEARCH_URL = 'http://localhost:9200/' 8 | 9 | DEBUG = False 10 | USE_TZ = True 11 | 12 | SECRET_KEY = 'test' 13 | 14 | DATABASES = { 15 | 'default': { 16 | 'ENGINE': 'django.db.backends.postgresql_psycopg2', 17 | 'USER': 'django_zombodb', 18 | 'NAME': 'django_zombodb', 19 | 'PASSWORD': 'password', 20 | 'HOST': '127.0.0.1', 21 | 'PORT': config('POSTGRES_PORT', default='5432'), 22 | 'ATOMIC_REQUESTS': False, # False gives better stacktraces 23 | } 24 | } 25 | 26 | ROOT_URLCONF = "tests.urls" 27 | 28 | INSTALLED_APPS = [ 29 | 'django.contrib.admin', 30 | 'django.contrib.auth', 31 | 'django.contrib.contenttypes', 32 | 'django.contrib.sessions', 33 | 'django.contrib.messages', 34 | 'django.contrib.staticfiles', 35 | 'django_zombodb', 36 | 37 | 'tests', 38 | 'tests.restaurants', 39 | ] 40 | 41 | SITE_ID = 1 42 | 43 | MIDDLEWARE = [ 44 | 'django.contrib.sessions.middleware.SessionMiddleware', 45 | 'django.middleware.common.CommonMiddleware', 46 | 'django.middleware.csrf.CsrfViewMiddleware', 47 | 'django.contrib.auth.middleware.AuthenticationMiddleware', 48 | 'django.contrib.messages.middleware.MessageMiddleware', 49 | ] 50 | 51 | TEMPLATES = [{ 52 | 'BACKEND': 'django.template.backends.django.DjangoTemplates', 53 | 'APP_DIRS': True, 54 | 'OPTIONS': { 55 | 'context_processors': [ 56 | 'django.template.context_processors.debug', 57 | 'django.template.context_processors.request', 58 | 'django.contrib.auth.context_processors.auth', 59 | 'django.contrib.messages.context_processors.messages', 60 | ], 61 | }, 62 | }] 63 | 64 | STATIC_URL = '/static/' 65 | -------------------------------------------------------------------------------- /tests/test_admin_mixins.py: -------------------------------------------------------------------------------- 1 | from django.contrib import admin 2 | from django.contrib.auth.models import User 3 | from django.test import TransactionTestCase 4 | from django.test.client import RequestFactory 5 | from django.urls import reverse 6 | 7 | from django_zombodb.admin_mixins import ZomboDBAdminMixin 8 | 9 | from .restaurants.admin import RestaurantAdmin 10 | from .restaurants.models import Restaurant 11 | 12 | 13 | class AdminMixinsTests(TransactionTestCase): 14 | factory = RequestFactory() 15 | 16 | def setUp(self): 17 | self.superuser = User.objects.create_superuser( 18 | username='super', email='a@b.com', password='xxx') 19 | 20 | self.alcove = Restaurant.objects.create( 21 | url='http://example.org?thealcove', 22 | name='The Alcove', 23 | street='41-11 49th St', 24 | zip_code='11104', 25 | city='New York City', 26 | state='NY', 27 | phone='+1 347-813-4159', 28 | email='alcove@example.org', 29 | website='https://www.facebook.com/thealcoveny/', 30 | categories=['Gastropub', 'Tapas', 'Bar'], 31 | ) 32 | self.tj = Restaurant.objects.create( 33 | url='http://example.org?tjasianbistro', 34 | name='TJ Asian Bistro', 35 | street='50-19 Skillman Ave', 36 | zip_code='11377', 37 | city='New York City', 38 | state='NY', 39 | phone='+1 718-205-2088', 40 | email='tjasianbistro@example.org', 41 | website='http://www.tjsushi.com/', 42 | categories=['Sushi', 'Asian', 'Japanese'], 43 | ) 44 | self.soleil = Restaurant.objects.create( 45 | url='http://example.org?cotesoleil', 46 | name='Côté Soleil', 47 | street='50-12 Skillman Ave', 48 | zip_code='11377', 49 | city='New York City', 50 | state='NY', 51 | phone='+1 347-612-4333', 52 | email='cotesoleilnyc@example.org', 53 | website='https://cotesoleilnyc.com/', 54 | categories=['French', 'Coffee', 'European'], 55 | ) 56 | 57 | def _mocked_authenticated_request(self, url, params, user): 58 | request = self.factory.get(url, params) 59 | request.user = user 60 | return request 61 | 62 | def test_inherits_from_mixin(self): 63 | self.assertTrue(issubclass(RestaurantAdmin, ZomboDBAdminMixin)) 64 | 65 | def test_no_search(self): 66 | restaurant_admin = RestaurantAdmin(Restaurant, admin.site) 67 | request = self._mocked_authenticated_request( 68 | '/restaurant/', {}, self.superuser) 69 | cl = restaurant_admin.get_changelist_instance(request) 70 | queryset = cl.get_queryset(request) 71 | self.assertCountEqual(queryset, [self.alcove, self.tj, self.soleil]) 72 | 73 | def test_get_search_results(self): 74 | restaurant_admin = RestaurantAdmin(Restaurant, admin.site) 75 | search_query = { 76 | 'q': 'sushi asian japanese 11377' 77 | } 78 | request = self._mocked_authenticated_request( 79 | '/restaurant/', search_query, self.superuser) 80 | cl = restaurant_admin.get_changelist_instance(request) 81 | queryset = cl.get_queryset(request) 82 | self.assertEqual(list(queryset), [self.tj, self.soleil]) 83 | 84 | def test_no_search_annotates_zombodb_score_as_0(self): 85 | restaurant_admin = RestaurantAdmin(Restaurant, admin.site) 86 | request = self._mocked_authenticated_request( 87 | '/restaurant/', {}, self.superuser) 88 | cl = restaurant_admin.get_changelist_instance(request) 89 | queryset = cl.get_queryset(request) 90 | self.assertCountEqual(queryset, [self.alcove, self.tj, self.soleil]) 91 | for restaurant in queryset: 92 | self.assertTrue(hasattr(restaurant, 'zombodb_score')) 93 | self.assertEqual(restaurant.zombodb_score, 0) 94 | 95 | def test_search_annotates_zombodb_score(self): 96 | restaurant_admin = RestaurantAdmin(Restaurant, admin.site) 97 | search_query = { 98 | 'q': 'sushi asian japanese 11377' 99 | } 100 | request = self._mocked_authenticated_request( 101 | '/restaurant/', search_query, self.superuser) 102 | cl = restaurant_admin.get_changelist_instance(request) 103 | queryset = cl.get_queryset(request) 104 | self.assertEqual(list(queryset), [self.tj, self.soleil]) 105 | for restaurant in queryset: 106 | self.assertTrue(hasattr(restaurant, 'zombodb_score')) 107 | self.assertGreater(restaurant.zombodb_score, 0) 108 | 109 | def test_search_validates_query(self): 110 | search_query = { 111 | 'q': 'sushi AND AND' 112 | } 113 | self.client.force_login(self.superuser) 114 | response = self.client.get( 115 | reverse('admin:restaurants_restaurant_changelist'), 116 | search_query 117 | ) 118 | self.assertContains(response, self.alcove.name) 119 | self.assertContains(response, self.tj.name) 120 | self.assertContains(response, self.soleil.name) 121 | self.assertEqual( 122 | [m.message for m in response.context['messages']], 123 | ['Invalid search query. Not filtering by search.'] 124 | ) 125 | 126 | def test_search_displays_score(self): 127 | search_query = { 128 | 'q': 'sushi asian japanese 11377' 129 | } 130 | self.client.force_login(self.superuser) 131 | response = self.client.get( 132 | reverse('admin:restaurants_restaurant_changelist'), 133 | search_query 134 | ) 135 | self.assertContains(response, self.tj.name) 136 | self.assertContains(response, self.soleil.name) 137 | self.assertContains(response, '') 138 | -------------------------------------------------------------------------------- /tests/test_apps.py: -------------------------------------------------------------------------------- 1 | from django.apps import apps 2 | from django.test import TestCase, modify_settings 3 | 4 | 5 | @modify_settings(INSTALLED_APPS={'append': 'django_zombodb'}) 6 | class AppsTests(TestCase): 7 | 8 | def test_apps(self): 9 | app = apps.get_app_config('django_zombodb') 10 | self.assertEqual(app.name, 'django_zombodb') 11 | -------------------------------------------------------------------------------- /tests/test_index.py: -------------------------------------------------------------------------------- 1 | from collections import OrderedDict 2 | 3 | import django 4 | from django.conf import settings 5 | from django.core.exceptions import ImproperlyConfigured 6 | from django.db import connection 7 | from django.test import TestCase, override_settings 8 | 9 | from django_zombodb.indexes import ZomboDBIndex 10 | 11 | from .models import DateTimeArrayModel, IntegerArrayModel 12 | 13 | 14 | @override_settings(ZOMBODB_ELASTICSEARCH_URL='http://localhost:9999/') 15 | class ZomboDBIndexCreateStatementAdapterTests(TestCase): 16 | maxDiff = 10000 17 | 18 | def setUp(self): 19 | self.index_name = 'my_test_index' 20 | self.index = ZomboDBIndex( 21 | fields=['datetimes', 'dates', 'times'], 22 | field_mapping=OrderedDict([ # use OrderedDict to stabilize tests on Python < 3.6 23 | ('datetimes', OrderedDict( 24 | [('type', 'date'), ('format', 'HH:mm:ss.SSSSSS'), ('copy_to', 'zdb_all')])), 25 | ('dates', OrderedDict([('type', 'date'), ('copy_to', 'zdb_all')])), 26 | ('times', OrderedDict([('type', 'date'), ('copy_to', 'zdb_all')])), 27 | ]), 28 | name=self.index_name, 29 | shards=4, 30 | replicas=1, 31 | alias='my-test-index-alias', 32 | refresh_interval='1s', 33 | type_name='doc', 34 | bulk_concurrency=10, 35 | batch_size=8388608, 36 | compression_level=9, 37 | llapi=False, 38 | ) 39 | with connection.schema_editor() as editor: 40 | self.statement_adapter = self.index.create_sql( 41 | model=DateTimeArrayModel, 42 | schema_editor=editor, 43 | using='') 44 | self.repr = repr(self.statement_adapter) 45 | self.str = str(self.statement_adapter) 46 | 47 | def test_references_table(self): 48 | self.assertIs( 49 | self.statement_adapter.references_table(DateTimeArrayModel._meta.db_table), True) 50 | self.assertIs( 51 | self.statement_adapter.references_table(IntegerArrayModel._meta.db_table), False) 52 | 53 | def test_references_column(self): 54 | self.assertIs( 55 | self.statement_adapter.references_column( 56 | DateTimeArrayModel._meta.db_table, 57 | 'datetimes' 58 | ), True) 59 | self.assertIs( 60 | self.statement_adapter.references_column( 61 | DateTimeArrayModel._meta.db_table, 62 | 'other' 63 | ), False) 64 | 65 | def test_rename_table_references(self): 66 | self.statement_adapter.rename_table_references( 67 | DateTimeArrayModel._meta.db_table, 68 | 'other') 69 | self.assertEqual( 70 | repr(self.statement_adapter.parts['table']), 71 | '') 72 | 73 | def test_rename_column_references(self): 74 | self.statement_adapter.rename_column_references( 75 | DateTimeArrayModel._meta.db_table, 76 | 'dates', 77 | 'other') 78 | self.assertEqual( 79 | repr(self.statement_adapter.parts['columns']), 80 | '') 81 | 82 | def test_repr(self): 83 | self.assertEqual( 84 | self.repr, 85 | '' 98 | ) 99 | 100 | def test_str(self): 101 | self.assertEqual( 102 | self.str, 103 | 'CREATE TYPE "my_test_index_row_type" ' 104 | 'AS (datetimes timestamp with time zone[], dates date[], times time[]); ' 105 | 'SELECT zdb.define_field_mapping(\'"tests_datetimearraymodel"\', \'datetimes\', ' 106 | '\'{"type":"date","format":"HH:mm:ss.SSSSSS","copy_to":"zdb_all"}\');' 107 | 'SELECT zdb.define_field_mapping(\'"tests_datetimearraymodel"\', \'dates\', ' 108 | '\'{"type":"date","copy_to":"zdb_all"}\');' 109 | 'SELECT zdb.define_field_mapping(\'"tests_datetimearraymodel"\', \'times\', ' 110 | '\'{"type":"date","copy_to":"zdb_all"}\');' 111 | 'CREATE INDEX "my_test_index" ON "tests_datetimearraymodel" ' 112 | 'USING zombodb ((ROW("datetimes", "dates", "times")::"my_test_index_row_type")) ' 113 | 'WITH (url = \'http://localhost:9999/\', shards = 4, replicas = 1, ' 114 | 'alias = \'my-test-index-alias\', refresh_interval = \'1s\', type_name = \'doc\', ' 115 | 'bulk_concurrency = 10, batch_size = 8388608, compression_level = 9, llapi = false) ' 116 | ) 117 | 118 | 119 | @override_settings(ZOMBODB_ELASTICSEARCH_URL='http://localhost:9999') 120 | class ZomboDBIndexRemoveStatementAdapter(TestCase): 121 | 122 | def setUp(self): 123 | self.index_name = 'my_other_test_index' 124 | self.index = ZomboDBIndex( 125 | fields=['dates', 'times'], 126 | name=self.index_name, 127 | ) 128 | with connection.schema_editor() as editor: 129 | self.statement_adapter_or_str = self.index.remove_sql( 130 | model=DateTimeArrayModel, 131 | schema_editor=editor) 132 | self.str = str(self.statement_adapter_or_str) 133 | self.repr = repr(self.statement_adapter_or_str) 134 | 135 | def test_references_table(self): 136 | if django.VERSION >= (2, 2, 0): 137 | self.assertIs( 138 | self.statement_adapter_or_str.references_table( 139 | DateTimeArrayModel._meta.db_table), True) 140 | self.assertIs( 141 | self.statement_adapter_or_str.references_table( 142 | IntegerArrayModel._meta.db_table), False) 143 | 144 | def test_references_column(self): 145 | if django.VERSION >= (2, 2, 0): 146 | self.assertIs( 147 | self.statement_adapter_or_str.references_column( # does nothing 148 | DateTimeArrayModel._meta.db_table, 149 | 'dates' 150 | ), False) 151 | self.assertNotIn('columns', self.statement_adapter_or_str.parts) 152 | 153 | def test_rename_table_references(self): 154 | if django.VERSION >= (2, 2, 0): 155 | self.statement_adapter_or_str.rename_table_references( 156 | DateTimeArrayModel._meta.db_table, 157 | 'other') 158 | self.assertEqual( 159 | repr(self.statement_adapter_or_str.parts['table']), 160 | '
') 161 | 162 | def test_rename_column_references(self): 163 | if django.VERSION >= (2, 2, 0): 164 | self.statement_adapter_or_str.rename_column_references( # does nothing 165 | DateTimeArrayModel._meta.db_table, 166 | 'dates', 167 | 'other') 168 | self.assertNotIn('columns', self.statement_adapter_or_str.parts) 169 | 170 | def test_repr(self): 171 | if django.VERSION >= (2, 2, 0): 172 | self.assertEqual( 173 | self.repr, 174 | '' 177 | ) 178 | 179 | def test_str(self): 180 | self.assertEqual( 181 | self.str, 182 | 'DROP INDEX IF EXISTS "my_other_test_index"; ' 183 | 'DROP TYPE IF EXISTS "my_other_test_index_row_type";') 184 | 185 | 186 | # Based on django/tests/postgres_tests/test_indexes.py 187 | @override_settings(ZOMBODB_ELASTICSEARCH_URL='http://localhost:9999') 188 | class ZomboDBIndexTests(TestCase): 189 | 190 | def test_suffix(self): 191 | self.assertEqual(ZomboDBIndex.suffix, 'zombodb') 192 | 193 | def test_eq(self): 194 | index = ZomboDBIndex(fields=['title']) 195 | same_index = ZomboDBIndex(fields=['title']) 196 | another_index = ZomboDBIndex(fields=['author']) 197 | self.assertEqual(index, same_index) 198 | self.assertNotEqual(index, another_index) 199 | 200 | def test_name_auto_generation(self): 201 | index = ZomboDBIndex(fields=['datetimes', 'dates', 'times']) 202 | index.set_name_with_model(DateTimeArrayModel) 203 | self.assertEqual(index.name, 'tests_datet_datetim_22445c_zombodb') 204 | 205 | def test_deconstruction(self): 206 | index = ZomboDBIndex( 207 | fields=['title'], 208 | field_mapping=OrderedDict([('title', {'type': 'text'})]), 209 | name='test_title_zombodb', 210 | shards=2, 211 | replicas=2, 212 | alias='test-alias', 213 | refresh_interval='10s', 214 | type_name='test-doc', 215 | bulk_concurrency=20, 216 | batch_size=8388608 * 2, 217 | compression_level=9, 218 | llapi=True, 219 | ) 220 | path, args, kwargs = index.deconstruct() 221 | self.assertEqual(path, 'django_zombodb.indexes.ZomboDBIndex') 222 | self.assertEqual(args, ()) 223 | self.assertEqual( 224 | kwargs, 225 | { 226 | 'fields': ['title'], 227 | 'field_mapping': OrderedDict([('title', {'type': 'text'})]), 228 | 'name': 'test_title_zombodb', 229 | 'shards': 2, 230 | 'replicas': 2, 231 | 'alias': 'test-alias', 232 | 'refresh_interval': '10s', 233 | 'type_name': 'test-doc', 234 | 'bulk_concurrency': 20, 235 | 'batch_size': 8388608 * 2, 236 | 'compression_level': 9, 237 | 'llapi': True, 238 | } 239 | ) 240 | 241 | def test_deconstruct_no_args(self): 242 | index = ZomboDBIndex(fields=['title'], name='test_title_zombodb') 243 | path, args, kwargs = index.deconstruct() 244 | self.assertEqual(path, 'django_zombodb.indexes.ZomboDBIndex') 245 | self.assertEqual(args, ()) 246 | self.assertEqual( 247 | kwargs, 248 | { 249 | 'fields': ['title'], 250 | 'name': 'test_title_zombodb' 251 | } 252 | ) 253 | 254 | 255 | class ZomboDBIndexURLTests(TestCase): 256 | 257 | @override_settings() 258 | def test_exception_no_url(self): 259 | del settings.ZOMBODB_ELASTICSEARCH_URL 260 | 261 | with self.assertRaises(ImproperlyConfigured) as cm: 262 | ZomboDBIndex(fields=['title'], name='test_title_zombodb') 263 | 264 | self.assertEqual( 265 | str(cm.exception), 266 | "Please set ZOMBODB_ELASTICSEARCH_URL on settings.") 267 | 268 | def test_exception_old_url_param(self): 269 | with self.assertRaises(ImproperlyConfigured) as cm: 270 | ZomboDBIndex(url='http://localhost:9200/', fields=['title'], name='test_title_zombodb') 271 | 272 | self.assertEqual( 273 | str(cm.exception), 274 | "The `url` param is not supported anymore. " 275 | "Instead, please remove it and set ZOMBODB_ELASTICSEARCH_URL on settings.") 276 | 277 | 278 | # Based on django/tests/postgres_tests/test_indexes.py 279 | @override_settings(ZOMBODB_ELASTICSEARCH_URL='http://localhost:9200/') 280 | class ZomboDBIndexSchemaTests(TestCase): 281 | ''' 282 | This test needs a running Elasticsearch instance at http://localhost:9200/ 283 | ''' 284 | 285 | def get_constraints(self, table): 286 | """ 287 | Get the indexes on the table using a new cursor. 288 | """ 289 | with connection.cursor() as cursor: 290 | return connection.introspection.get_constraints(cursor, table) 291 | 292 | def test_zombodb_index(self): 293 | # Ensure the table is there and doesn't have an index. 294 | self.assertNotIn('field', self.get_constraints(IntegerArrayModel._meta.db_table)) 295 | # Add the index 296 | index_name = 'integer_array_model_field_zombodb' 297 | index = ZomboDBIndex(fields=['field'], name=index_name) 298 | with connection.schema_editor() as editor: 299 | editor.add_index(IntegerArrayModel, index) 300 | constraints = self.get_constraints(IntegerArrayModel._meta.db_table) 301 | # Check zombodb index was added 302 | self.assertEqual(constraints[index_name]['type'], ZomboDBIndex.suffix) 303 | # Drop the index 304 | with connection.schema_editor() as editor: 305 | editor.remove_index(IntegerArrayModel, index) 306 | self.assertNotIn(index_name, self.get_constraints(IntegerArrayModel._meta.db_table)) 307 | 308 | def test_zombodb_parameters(self): 309 | index_name = 'integer_array_zombodb_params' 310 | index = ZomboDBIndex( 311 | fields=['field'], 312 | field_mapping=OrderedDict([('title', {'type': 'text'})]), 313 | name=index_name, 314 | shards=2, 315 | replicas=2, 316 | alias='test-alias', 317 | refresh_interval='10s', 318 | type_name='test-doc', 319 | bulk_concurrency=20, 320 | batch_size=8388608 * 2, 321 | compression_level=9, 322 | llapi=True, 323 | ) 324 | with connection.schema_editor() as editor: 325 | editor.add_index(IntegerArrayModel, index) 326 | constraints = self.get_constraints(IntegerArrayModel._meta.db_table) 327 | self.assertEqual(constraints[index_name]['type'], ZomboDBIndex.suffix) 328 | actual_options = constraints[index_name]['options'] 329 | for expected_option in [ 330 | "url=http://localhost:9200/", 331 | "shards=2", 332 | "replicas=2", 333 | "alias=test-alias", 334 | "refresh_interval=10s", 335 | "type_name=test-doc", 336 | "bulk_concurrency=20", 337 | "batch_size=16777216", 338 | "compression_level=9", 339 | "llapi=true", 340 | ]: 341 | self.assertIn(expected_option, actual_options) 342 | with connection.schema_editor() as editor: 343 | editor.remove_index(IntegerArrayModel, index) 344 | self.assertNotIn(index_name, self.get_constraints(IntegerArrayModel._meta.db_table)) 345 | -------------------------------------------------------------------------------- /tests/test_querysets.py: -------------------------------------------------------------------------------- 1 | from django.core.exceptions import ImproperlyConfigured 2 | from django.test import TransactionTestCase, override_settings 3 | 4 | from elasticsearch_dsl import Q as ElasticsearchQ 5 | from elasticsearch_dsl import Search 6 | from elasticsearch_dsl.query import Term 7 | 8 | from django_zombodb.exceptions import InvalidElasticsearchQuery 9 | 10 | from .restaurants.models import Restaurant, RestaurantNoIndex 11 | 12 | 13 | @override_settings(ZOMBODB_ELASTICSEARCH_URL='http://localhost:9200/') 14 | class SearchQuerySetTests(TransactionTestCase): 15 | 16 | def setUp(self): 17 | self.alcove = Restaurant.objects.create( 18 | url='http://example.org?thealcove', 19 | name='The Alcove', 20 | street='41-11 49th St', 21 | zip_code='11104', 22 | city='New York City', 23 | state='NY', 24 | phone='+1 347-813-4159', 25 | email='alcove@example.org', 26 | website='https://www.facebook.com/thealcoveny/', 27 | categories=['Gastropub', 'Tapas', 'Bar'], 28 | ) 29 | self.tj = Restaurant.objects.create( 30 | url='http://example.org?tjasianbistro', 31 | name='TJ Asian Bistro', 32 | street='50-19 Skillman Ave', 33 | zip_code='11377', 34 | city='New York City', 35 | state='NY', 36 | phone='+1 718-205-2088', 37 | email='tjasianbistro@example.org', 38 | website='http://www.tjsushi.com/', 39 | categories=['Sushi', 'Asian', 'Japanese'], 40 | ) 41 | self.soleil = Restaurant.objects.create( 42 | url='http://example.org?cotesoleil', 43 | name='Côté Soleil', 44 | street='50-12 Skillman Ave', 45 | zip_code='11377', 46 | city='New York City', 47 | state='NY', 48 | phone='+1 347-612-4333', 49 | email='cotesoleilnyc@example.org', 50 | website='https://cotesoleilnyc.com/', 51 | categories=['French', 'Coffee', 'European'], 52 | ) 53 | 54 | def test_query_string_search(self): 55 | results = Restaurant.objects.query_string_search('coffee') 56 | self.assertCountEqual(results, [self.soleil]) 57 | 58 | results = Restaurant.objects.query_string_search('11377') 59 | self.assertCountEqual(results, [self.tj, self.soleil]) 60 | 61 | results = Restaurant.objects.query_string_search('email:alcove@example.org') 62 | self.assertCountEqual(results, [self.alcove]) 63 | 64 | def test_dict_search(self): 65 | results = Restaurant.objects.dict_search( 66 | {'bool': {'must': [{'match': {'street': 'Skillman Ave'}}, 67 | {'match': {'categories': 'French'}}]}} 68 | ) 69 | self.assertCountEqual(results, [self.soleil]) 70 | 71 | results = Restaurant.objects.dict_search( 72 | {'bool': {'must': [{'match': {'street': 'Skillman Ave'}}, 73 | {'match': {'zip_code': '11377'}}]}} 74 | ) 75 | self.assertCountEqual(results, [self.tj, self.soleil]) 76 | 77 | results = Restaurant.objects.dict_search( 78 | {'term': {'email': 'alcove@example.org'}} 79 | ) 80 | self.assertCountEqual(results, [self.alcove]) 81 | 82 | def test_dsl_search(self): 83 | results = Restaurant.objects.dsl_search(ElasticsearchQ( 84 | 'bool', 85 | must=[ 86 | ElasticsearchQ('match', street='Skillman Ave'), 87 | ElasticsearchQ('match', categories='French') 88 | ] 89 | )) 90 | self.assertCountEqual(results, [self.soleil]) 91 | 92 | results = Restaurant.objects.dsl_search(ElasticsearchQ( 93 | 'bool', 94 | must=[ 95 | ElasticsearchQ('match', street='Skillman Ave'), 96 | ElasticsearchQ('match', zip_code='11377') 97 | ] 98 | )) 99 | self.assertCountEqual(results, [self.tj, self.soleil]) 100 | 101 | results = Restaurant.objects.dsl_search(Term(email='alcove@example.org')) 102 | self.assertCountEqual(results, [self.alcove]) 103 | 104 | def test_query_string_search_sort(self): 105 | results = Restaurant.objects.query_string_search( 106 | 'sushi OR asian OR japanese OR french', 107 | validate=True, 108 | sort=True) 109 | self.assertEqual(list(results), [self.tj, self.soleil]) 110 | 111 | results = Restaurant.objects.query_string_search( 112 | 'french OR coffee OR european OR sushi', 113 | validate=True, 114 | sort=True) 115 | self.assertEqual(list(results), [self.soleil, self.tj]) 116 | 117 | def test_dict_search_sort(self): 118 | results = Restaurant.objects.dict_search( 119 | {'bool': {'minimum_should_match': 1, 120 | 'should': [{'match': {'categories': 'sushi'}}, 121 | {'match': {'categories': 'asian'}}, 122 | {'match': {'categories': 'japanese'}}, 123 | {'match': {'categories': 'french'}}]}}, 124 | validate=True, 125 | sort=True) 126 | self.assertEqual(list(results), [self.tj, self.soleil]) 127 | 128 | results = Restaurant.objects.dict_search( 129 | {'bool': {'minimum_should_match': 1, 130 | 'should': [{'match': {'categories': 'french'}}, 131 | {'match': {'categories': 'coffee'}}, 132 | {'match': {'categories': 'european'}}, 133 | {'match': {'categories': 'sushi'}}]}}, 134 | validate=True, 135 | sort=True) 136 | self.assertEqual(list(results), [self.soleil, self.tj]) 137 | 138 | def test_dsl_search_sort(self): 139 | results = Restaurant.objects.dsl_search( 140 | ElasticsearchQ( 141 | 'bool', 142 | should=[ 143 | ElasticsearchQ('match', categories='sushi'), 144 | ElasticsearchQ('match', categories='asian'), 145 | ElasticsearchQ('match', categories='japanese'), 146 | ElasticsearchQ('match', categories='french'), 147 | ], 148 | minimum_should_match=1 149 | ), 150 | validate=True, 151 | sort=True) 152 | self.assertEqual(list(results), [self.tj, self.soleil]) 153 | 154 | results = Restaurant.objects.dsl_search( 155 | ElasticsearchQ( 156 | 'bool', 157 | should=[ 158 | ElasticsearchQ('match', categories='french'), 159 | ElasticsearchQ('match', categories='coffee'), 160 | ElasticsearchQ('match', categories='european'), 161 | ElasticsearchQ('match', categories='sushi'), 162 | ], 163 | minimum_should_match=1 164 | ), 165 | sort=True) 166 | self.assertEqual(list(results), [self.soleil, self.tj]) 167 | 168 | def test_query_string_search_score_attr(self): 169 | results = Restaurant.objects.query_string_search( 170 | 'skillman', 171 | sort=True, 172 | score_attr='custom_score') 173 | 174 | self.assertEqual(len(results), 2) 175 | for r in results: 176 | self.assertTrue(hasattr(r, 'custom_score')) 177 | self.assertGreater(r.custom_score, 0) 178 | 179 | def test_dict_search_score_attr(self): 180 | results = Restaurant.objects.dict_search( 181 | {'match': {'street': 'skillman'}}, 182 | sort=True, 183 | score_attr='custom_score') 184 | 185 | self.assertEqual(len(results), 2) 186 | for r in results: 187 | self.assertTrue(hasattr(r, 'custom_score')) 188 | self.assertGreater(r.custom_score, 0) 189 | 190 | def test_dsl_search_score_attr(self): 191 | results = Restaurant.objects.dsl_search( 192 | ElasticsearchQ('match', street='skillman'), 193 | sort=True, 194 | score_attr='custom_score') 195 | 196 | self.assertEqual(len(results), 2) 197 | for r in results: 198 | self.assertTrue(hasattr(r, 'custom_score')) 199 | self.assertGreater(r.custom_score, 0) 200 | 201 | def test_query_string_search_validate(self): 202 | with self.assertRaises(InvalidElasticsearchQuery) as cm: 203 | Restaurant.objects.query_string_search('skillman AND', validate=True) 204 | self.assertRegex( 205 | str(cm.exception), 206 | "Invalid Elasticsearch query: (.+)") 207 | 208 | def test_dict_search_validate(self): 209 | with self.assertRaises(InvalidElasticsearchQuery) as cm: 210 | Restaurant.objects.dict_search({'wrong': 'query'}, validate=True) 211 | self.assertRegex( 212 | str(cm.exception), 213 | "Invalid Elasticsearch query: (.+)") 214 | 215 | def test_dsl_search_validate(self): 216 | query = ElasticsearchQ('bool') 217 | query.name = 'wrong' 218 | with self.assertRaises(InvalidElasticsearchQuery) as cm: 219 | Restaurant.objects.dsl_search(query, validate=True) 220 | self.assertRegex( 221 | str(cm.exception), 222 | "Invalid Elasticsearch query: (.+)") 223 | 224 | def test_dsl_search_cant_use_es_search(self): 225 | query = Search(index="my-index") \ 226 | .filter("term", category="search") \ 227 | .query("match", title="python") \ 228 | .exclude("match", description="beta") 229 | with self.assertRaises(InvalidElasticsearchQuery) as cm: 230 | Restaurant.objects.dsl_search(query, validate=True) 231 | self.assertEqual( 232 | str(cm.exception), 233 | "Do not use the `Search` class. " 234 | "`query` must be an instance of a class inheriting from `DslBase`.") 235 | 236 | def test_filter_search_chain(self): 237 | results = Restaurant.objects.filter( 238 | zip_code='11377' 239 | ).query_string_search('coffee') 240 | self.assertCountEqual(results, [self.soleil]) 241 | 242 | def test_search_fails_if_no_zombodb_index_in_model_and_validate(self): 243 | with self.assertRaises(ImproperlyConfigured) as cm: 244 | RestaurantNoIndex.objects.query_string_search('skillman', validate=True) 245 | self.assertEqual( 246 | str(cm.exception), 247 | "Can't find a ZomboDBIndex at model {model}. " 248 | "Did you forget it? ".format(model=RestaurantNoIndex)) 249 | 250 | def test_query_string_search_no_limit(self): 251 | # duplicate tj and soleil 252 | self.tj.pk = None 253 | self.tj.save() 254 | self.soleil.pk = None 255 | self.soleil.save() 256 | 257 | results = Restaurant.objects.query_string_search( 258 | 'skillman', 259 | sort=True, 260 | limit=None) 261 | 262 | self.assertEqual(len(results), 4) 263 | self.assertEqual( 264 | [r.name for r in results], 265 | [self.soleil.name, self.soleil.name, self.tj.name, self.tj.name]) 266 | 267 | def test_query_string_search_limit(self): 268 | # duplicate tj and soleil 269 | self.tj.pk = None 270 | self.tj.save() 271 | self.soleil.pk = None 272 | self.soleil.save() 273 | 274 | results = Restaurant.objects.query_string_search( 275 | 'skillman', 276 | sort=True, 277 | limit=2) 278 | 279 | self.assertEqual(len(results), 2) 280 | self.assertEqual([r.name for r in results], [self.soleil.name] * 2) 281 | 282 | def test_dict_search_no_limit(self): 283 | # duplicate tj and soleil 284 | self.tj.pk = None 285 | self.tj.save() 286 | self.soleil.pk = None 287 | self.soleil.save() 288 | 289 | results = Restaurant.objects.dict_search( 290 | {'match': {'street': 'skillman'}}, 291 | sort=True, 292 | limit=None) 293 | 294 | self.assertEqual(len(results), 4) 295 | self.assertEqual( 296 | [r.name for r in results], 297 | [self.soleil.name, self.soleil.name, self.tj.name, self.tj.name]) 298 | 299 | def test_dict_search_limit(self): 300 | # duplicate tj and soleil 301 | self.tj.pk = None 302 | self.tj.save() 303 | self.soleil.pk = None 304 | self.soleil.save() 305 | 306 | results = Restaurant.objects.dict_search( 307 | {'match': {'street': 'skillman'}}, 308 | sort=True, 309 | limit=2) 310 | 311 | self.assertEqual(len(results), 2) 312 | self.assertEqual([r.name for r in results], [self.soleil.name] * 2) 313 | 314 | def test_dsl_search_no_limit(self): 315 | # duplicate tj and soleil 316 | self.tj.pk = None 317 | self.tj.save() 318 | self.soleil.pk = None 319 | self.soleil.save() 320 | 321 | results = Restaurant.objects.dsl_search( 322 | ElasticsearchQ('match', street='skillman'), 323 | sort=True, 324 | limit=None) 325 | 326 | self.assertEqual(len(results), 4) 327 | self.assertEqual( 328 | [r.name for r in results], 329 | [self.soleil.name, self.soleil.name, self.tj.name, self.tj.name]) 330 | 331 | def test_dsl_search_limit(self): 332 | # duplicate tj and soleil 333 | self.tj.pk = None 334 | self.tj.save() 335 | self.soleil.pk = None 336 | self.soleil.save() 337 | 338 | results = Restaurant.objects.dsl_search( 339 | ElasticsearchQ('match', street='skillman'), 340 | sort=True, 341 | limit=2) 342 | 343 | self.assertEqual(len(results), 2) 344 | self.assertEqual([r.name for r in results], [self.soleil.name] * 2) 345 | -------------------------------------------------------------------------------- /tests/urls.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | from __future__ import absolute_import, unicode_literals 3 | 4 | from django.conf.urls import url 5 | from django.contrib import admin 6 | 7 | 8 | urlpatterns = [ 9 | url(r'^admin/', admin.site.urls), 10 | ] 11 | -------------------------------------------------------------------------------- /tox.ini: -------------------------------------------------------------------------------- 1 | [tox] 2 | envlist = 3 | {py36,py37}-django-master 4 | {py35,py36,py37}-django-22 5 | {py35,py36,py37}-django-21 6 | {py35,py36,py37}-django-20 7 | quality 8 | docs 9 | skipsdist = true 10 | 11 | [django-master] 12 | deps = 13 | https://github.com/django/django/archive/master.tar.gz 14 | 15 | [testenv] 16 | setenv = 17 | PYTHONPATH = {toxinidir}:{toxinidir}/ 18 | DJANGO_SETTINGS_MODULE = tests.settings 19 | whitelist_externals = bash 20 | deps = 21 | django-master: {[django-master]deps} 22 | django-22: Django>=2.2,<2.3 23 | django-21: Django>=2.1,<2.2 24 | django-20: Django>=2.0,<2.1 25 | -r{toxinidir}/requirements/test.txt 26 | basepython = 27 | py37: python3.7 28 | py36: python3.6 29 | py35: python3.5 30 | commands = 31 | coverage run --source django_zombodb runtests.py 32 | 33 | [testenv:docs] 34 | setenv = 35 | DJANGO_SETTINGS_MODULE = example.example.settings 36 | whitelist_externals = 37 | make 38 | rm 39 | deps = 40 | -r{toxinidir}/requirements/doc.txt 41 | commands = 42 | pip install -e . 43 | make docs 44 | python setup.py check --restructuredtext --strict 45 | basepython = 46 | python3.7 47 | 48 | [testenv:quality] 49 | whitelist_externals = 50 | make 51 | rm 52 | touch 53 | deps = 54 | -r{toxinidir}/requirements/doc.txt 55 | -r{toxinidir}/requirements/quality.txt 56 | -r{toxinidir}/requirements/test.txt 57 | commands = 58 | touch tests/__init__.py 59 | make lint 60 | make selfcheck 61 | basepython = 62 | python3.7 63 | 64 | 65 | [travis:env] 66 | DJANGO = 67 | 2.2: django-22 68 | 2.1: django-21 69 | 2.0: django-20 70 | master: django-master 71 | --------------------------------------------------------------------------------