├── .coveragerc ├── .example.env ├── .github ├── ISSUE_TEMPLATE │ ├── bug_report.md │ └── feature_request.md ├── pull_request_template.md └── workflows │ ├── build.yml │ ├── lint.yml │ └── pypi.yml ├── .gitignore ├── .pre-commit-config.yaml ├── CODE_OF_CONDUCT.md ├── CONTRIBUTING.md ├── LICENSE ├── MANIFEST.in ├── README.md ├── django_typesense ├── __init__.py ├── admin.py ├── apps.py ├── changelist.py ├── collections.py ├── exceptions.py ├── fields.py ├── management │ ├── __init__.py │ └── commands │ │ ├── __init__.py │ │ └── updatecollections.py ├── migrations │ └── __init__.py ├── mixins.py ├── paginator.py ├── signals.py ├── static │ └── admin │ │ └── js │ │ └── search-typesense.js ├── typesense_client.py └── utils.py ├── requirements-dev.txt ├── runtests.py ├── setup.py └── tests ├── __init__.py ├── collections.py ├── factories.py ├── models.py ├── settings.py ├── test_typesense_admin_mixin.py ├── test_typesense_changelist.py ├── test_typesense_fields.py ├── test_typesense_methods.py ├── test_typesense_model_mixin.py ├── test_typesense_paginator.py ├── test_typesense_signals.py ├── test_typesense_utils.py └── utils.py /.coveragerc: -------------------------------------------------------------------------------- 1 | [report] 2 | omit = 3 | tests/* 4 | runtests.py 5 | -------------------------------------------------------------------------------- /.example.env: -------------------------------------------------------------------------------- 1 | TYPESENSE_API_KEY=TYPESENSE_API_KEY 2 | TYPESENSE_HOST=localhost 3 | TYPESENSE_PROTOCOL=http 4 | TYPESENSE_PORT=8108 -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/bug_report.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Bug report 3 | about: Create a report to help us improve 4 | title: 'Please replace with a clear and descriptive title' 5 | labels: '' 6 | assignees: '' 7 | --- 8 | 9 | ## Description 10 | 11 | A clear and concise description of what the bug is. 12 | 13 | ### How To Reproduce 14 | 15 | Steps to reproduce the behavior: 16 | 17 | 1. Go to '...' 18 | 1. Click on '....' 19 | 1. Scroll down to '....' 20 | 1. See error 21 | 22 | ### Expected behavior 23 | 24 | A clear and concise description of what you expected to happen. 25 | 26 | ### Environment Details (please complete the following information) 27 | 28 | - OS: [e.g. Ubuntu 22.04] 29 | 30 | - Versions: 31 | 32 | - Python: [e.g. 3.10] 33 | - Django: [e.g. 4.2] 34 | - django_typesense: [e.g. 0.1.1] 35 | 36 | **Additional context** 37 | Add any other context about the problem here. 38 | 39 | ### Screenshots 40 | 41 | If applicable, add screenshots to help explain your problem. 42 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/feature_request.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Feature request 3 | about: Suggest an idea for this project 4 | title: 'Please replace with a clear and descriptive title' 5 | labels: '' 6 | assignees: '' 7 | --- 8 | 9 | ## Is your feature request related to a problem? Please describe 10 | 11 | A clear and concise description of what the problem is. Ex. I'm always frustrated when [...] 12 | 13 | ### Describe the solution you'd like 14 | 15 | A clear and concise description of what you want to happen. 16 | 17 | ### Describe alternatives you've considered 18 | 19 | A clear and concise description of any alternative solutions or features you've considered. 20 | 21 | **Additional context** 22 | Add any other context or screenshots about the feature request here. 23 | -------------------------------------------------------------------------------- /.github/pull_request_template.md: -------------------------------------------------------------------------------- 1 | Resolves # (issue) 2 | 3 | ## Proposed changes 4 | 5 | Describe the big picture of your changes here to communicate to the maintainers why we should accept this pull request. If it fixes a bug or resolves a feature request, be sure to link to that issue. 6 | 7 | ### Types of changes 8 | 9 | What types of changes does your code introduce to DjangoTypesense? 10 | _Put an `x` in the boxes that apply_ 11 | 12 | - [ ] Bugfix (non-breaking change which fixes an issue) 13 | - [ ] New feature (non-breaking change which adds functionality) 14 | - [ ] Breaking change (fix or feature that would cause existing functionality to not work as expected) 15 | - [ ] Documentation Update (if none of the other choices apply) 16 | 17 | ### Checklist 18 | 19 | _Put an `x` in the boxes that apply. You can also fill these out after creating the PR. If you're unsure about any of them, don't hesitate to ask. We're here to help! This is simply a reminder of what we are going to look for before merging your code._ 20 | 21 | - [ ] I have read the [CONTRIBUTING](https://github.com/Siege-Software/django-typesense/blob/main/CONTRIBUTING.md) doc 22 | - [ ] Lint and unit tests pass locally with my changes 23 | - [ ] I have added tests that prove my fix is effective or that my feature works 24 | - [ ] I have added necessary documentation (if appropriate) 25 | - [ ] Any dependent changes have been merged and published in downstream modules 26 | -------------------------------------------------------------------------------- /.github/workflows/build.yml: -------------------------------------------------------------------------------- 1 | --- 2 | name: "build" 3 | 4 | on: # yamllint disable-line rule:truthy 5 | pull_request: 6 | push: 7 | branches: [main] 8 | 9 | jobs: 10 | build: 11 | name: Python ${{ matrix.python-version }} | Django ${{ matrix.django-version}} | Ubuntu 12 | runs-on: ubuntu-20.04 13 | strategy: 14 | fail-fast: false 15 | matrix: 16 | include: 17 | - python-version: 3.8 18 | django-version: "3.2.*" 19 | - python-version: 3.8 20 | django-version: "4.0.*" 21 | 22 | - python-version: 3.9 23 | django-version: "3.2.*" 24 | - python-version: 3.9 25 | django-version: "4.0.*" 26 | - python-version: 3.9 27 | django-version: "4.1.*" 28 | 29 | - python-version: "3.10" 30 | django-version: "3.2.*" 31 | - python-version: "3.10" 32 | django-version: "4.0.*" 33 | - python-version: "3.10" 34 | django-version: "4.1.*" 35 | coverage: true 36 | 37 | steps: 38 | - name: Checkout 39 | uses: actions/checkout@v2 40 | 41 | - name: Setup Python ${{ matrix.python-version }} 42 | uses: actions/setup-python@v1 43 | with: 44 | python-version: ${{ matrix.python-version }} 45 | 46 | - name: Install Django ${{ matrix.django-version }} 47 | run: | 48 | pip install Django==${{ matrix.django-version }} 49 | pip install coverage 50 | pip install factory-boy 51 | pip install pytz 52 | pip install typesense 53 | 54 | - name: Setup TypeSense 55 | uses: jirevwe/typesense-github-action@v1.0.1 56 | with: 57 | typesense-version: '27.0' 58 | typesense-api-key: sample_key 59 | 60 | - name: Run tests 61 | run: | 62 | coverage run runtests.py 63 | 64 | - name: Upload coverage to Codecov 65 | uses: codecov/codecov-action@v3 66 | with: 67 | token: ${{ secrets.CODECOV_TOKEN }} 68 | -------------------------------------------------------------------------------- /.github/workflows/lint.yml: -------------------------------------------------------------------------------- 1 | --- 2 | name: "Format" 3 | 4 | on: # yamllint disable-line rule:truthy 5 | pull_request: 6 | push: 7 | branches: main 8 | 9 | jobs: 10 | lint: 11 | name: Format 12 | runs-on: ubuntu-latest 13 | 14 | steps: 15 | - name: Checkout 16 | uses: actions/checkout@v2 17 | 18 | - name: Setup Python 3.8 19 | uses: actions/setup-python@v1 20 | with: 21 | python-version: 3.8 22 | 23 | - name: Upgrade Setuptools 24 | run: pip install --upgrade setuptools wheel 25 | 26 | - name: Install requirements 27 | run: pip install -r requirements-dev.txt 28 | 29 | - name: Run Format 30 | run: black . 31 | -------------------------------------------------------------------------------- /.github/workflows/pypi.yml: -------------------------------------------------------------------------------- 1 | --- 2 | name: "PyPI Release" 3 | 4 | on: # yamllint disable-line rule:truthy 5 | push: 6 | tags: 7 | - 'v*' 8 | 9 | jobs: 10 | pypi-publish: 11 | name: PyPI Release 12 | runs-on: ubuntu-latest 13 | environment: 14 | name: pypi 15 | url: https://pypi.org/p/django-typesense 16 | permissions: 17 | id-token: write 18 | 19 | steps: 20 | - name: Checkout 21 | uses: actions/checkout@v2 22 | 23 | - name: Setup Python 3.8 24 | uses: actions/setup-python@v1 25 | with: 26 | python-version: 3.8 27 | 28 | - name: Upgrade Setuptools 29 | run: pip install --upgrade setuptools wheel 30 | 31 | - name: Build Distribution 32 | run: python setup.py sdist bdist_wheel --universal 33 | 34 | - name: Publish to PyPI 35 | uses: pypa/gh-action-pypi-publish@release/v1 36 | with: 37 | user: __token__ 38 | password: ${{ secrets.PYPI_AI_TOKEN }} 39 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Created by https://www.toptal.com/developers/gitignore/api/python,pycharm,pycharm+iml,pycharm+all 2 | # Edit at https://www.toptal.com/developers/gitignore?templates=python,pycharm,pycharm+iml,pycharm+all 3 | 4 | ### PyCharm ### 5 | # Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio, WebStorm and Rider 6 | # Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839 7 | 8 | .idea/* 9 | 10 | ### Python ### 11 | # Byte-compiled / optimized / DLL files 12 | __pycache__/ 13 | *.py[cod] 14 | *$py.class 15 | 16 | # C extensions 17 | *.so 18 | 19 | # Distribution / packaging 20 | .Python 21 | build/ 22 | develop-eggs/ 23 | dist/ 24 | downloads/ 25 | eggs/ 26 | .eggs/ 27 | lib/ 28 | lib64/ 29 | parts/ 30 | sdist/ 31 | var/ 32 | wheels/ 33 | share/python-wheels/ 34 | *.egg-info/ 35 | .installed.cfg 36 | *.egg 37 | MANIFEST 38 | 39 | # PyInstaller 40 | # Usually these files are written by a python script from a template 41 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 42 | *.manifest 43 | *.spec 44 | 45 | # Installer logs 46 | pip-log.txt 47 | pip-delete-this-directory.txt 48 | 49 | # Unit test / coverage reports 50 | htmlcov/ 51 | .tox/ 52 | .nox/ 53 | .coverage 54 | .coverage.* 55 | .cache 56 | nosetests.xml 57 | coverage.xml 58 | *.cover 59 | *.py,cover 60 | .hypothesis/ 61 | .pytest_cache/ 62 | cover/ 63 | 64 | # Translations 65 | *.mo 66 | *.pot 67 | 68 | # Django stuff: 69 | *.log 70 | local_settings.py 71 | db.sqlite3 72 | db.sqlite3-journal 73 | 74 | # Flask stuff: 75 | instance/ 76 | .webassets-cache 77 | 78 | # Scrapy stuff: 79 | .scrapy 80 | 81 | # Sphinx documentation 82 | wid_engine/docs/_build/ 83 | 84 | # PyBuilder 85 | .pybuilder/ 86 | target/ 87 | 88 | # Jupyter Notebook 89 | .ipynb_checkpoints 90 | 91 | # IPython 92 | profile_default/ 93 | ipython_config.py 94 | 95 | # pyenv 96 | # For a library or package, you might want to ignore these files since the code is 97 | # intended to run in multiple environments; otherwise, check them in: 98 | # .python-version 99 | 100 | # pipenv 101 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. 102 | # However, in case of collaboration, if having platform-specific dependencies or dependencies 103 | # having no cross-platform support, pipenv may install dependencies that don't work, or not 104 | # install all needed dependencies. 105 | #Pipfile.lock 106 | 107 | # poetry 108 | # Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. 109 | # This is especially recommended for binary packages to ensure reproducibility, and is more 110 | # commonly ignored for libraries. 111 | # https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control 112 | #poetry.lock 113 | 114 | # pdm 115 | # Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. 116 | #pdm.lock 117 | # pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it 118 | # in version control. 119 | # https://pdm.fming.dev/#use-with-ide 120 | .pdm.toml 121 | 122 | # PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm 123 | __pypackages__/ 124 | 125 | # Celery stuff 126 | celerybeat-schedule 127 | celerybeat.pid 128 | 129 | # SageMath parsed files 130 | *.sage.py 131 | 132 | # Environments 133 | .env 134 | .venv 135 | env/ 136 | venv/ 137 | ENV/ 138 | env.bak/ 139 | venv.bak/ 140 | 141 | # Spyder project settings 142 | .spyderproject 143 | .spyproject 144 | 145 | # Rope project settings 146 | .ropeproject 147 | 148 | # mkdocs documentation 149 | /site 150 | 151 | # mypy 152 | .mypy_cache/ 153 | .dmypy.json 154 | dmypy.json 155 | 156 | # Pyre type checker 157 | .pyre/ 158 | 159 | # pytype static type analyzer 160 | .pytype/ 161 | 162 | # Cython debug symbols 163 | cython_debug/ 164 | 165 | # PyCharm 166 | # JetBrains specific template is maintained in a separate JetBrains.gitignore that can 167 | # be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore 168 | # and can be added to the global gitignore or merged into this file. For a more nuclear 169 | # option (not recommended) you can uncomment the following to ignore the entire idea folder. 170 | #.idea/ 171 | 172 | ### Python Patch ### 173 | # Poetry local configuration file - https://python-poetry.org/docs/configuration/#local-configuration 174 | poetry.toml 175 | 176 | # ruff 177 | .ruff_cache/ 178 | 179 | # LSP config files 180 | pyrightconfig.json 181 | 182 | # End of https://www.toptal.com/developers/gitignore/api/python,pycharm,pycharm+iml,pycharm+all -------------------------------------------------------------------------------- /.pre-commit-config.yaml: -------------------------------------------------------------------------------- 1 | repos: 2 | - repo: https://github.com/psf/black 3 | rev: 23.3.0 4 | hooks: 5 | - id: black -------------------------------------------------------------------------------- /CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- 1 | # Contributor Covenant Code of Conduct 2 | 3 | ## Our Pledge 4 | 5 | We as members, contributors, and leaders pledge to make participation in our 6 | community a harassment-free experience for everyone, regardless of age, body 7 | size, visible or invisible disability, ethnicity, sex characteristics, gender 8 | identity and expression, level of experience, education, socio-economic status, 9 | nationality, personal appearance, race, religion, or sexual identity 10 | and orientation. 11 | 12 | We pledge to act and interact in ways that contribute to an open, welcoming, 13 | diverse, inclusive, and healthy community. 14 | 15 | ## Our Standards 16 | 17 | Examples of behavior that contributes to a positive environment for our 18 | community include: 19 | 20 | * Demonstrating empathy and kindness toward other people 21 | * Being respectful of differing opinions, viewpoints, and experiences 22 | * Giving and gracefully accepting constructive feedback 23 | * Accepting responsibility and apologizing to those affected by our mistakes, 24 | and learning from the experience 25 | * Focusing on what is best not just for us as individuals, but for the 26 | overall community 27 | 28 | Examples of unacceptable behavior include: 29 | 30 | * The use of sexualized language or imagery, and sexual attention or 31 | advances of any kind 32 | * Trolling, insulting or derogatory comments, and personal or political attacks 33 | * Public or private harassment 34 | * Publishing others' private information, such as a physical or email 35 | address, without their explicit permission 36 | * Other conduct which could reasonably be considered inappropriate in a 37 | professional setting 38 | 39 | ## Enforcement Responsibilities 40 | 41 | Community leaders are responsible for clarifying and enforcing our standards of 42 | acceptable behavior and will take appropriate and fair corrective action in 43 | response to any behavior that they deem inappropriate, threatening, offensive, 44 | or harmful. 45 | 46 | Community leaders have the right and responsibility to remove, edit, or reject 47 | comments, commits, code, wiki edits, issues, and other contributions that are 48 | not aligned to this Code of Conduct, and will communicate reasons for moderation 49 | decisions when appropriate. 50 | 51 | ## Scope 52 | 53 | This Code of Conduct applies within all community spaces, and also applies when 54 | an individual is officially representing the community in public spaces. 55 | Examples of representing our community include using an official e-mail address, 56 | posting via an official social media account, or acting as an appointed 57 | representative at an online or offline event. 58 | 59 | ## Enforcement 60 | 61 | Instances of abusive, harassing, or otherwise unacceptable behavior may be 62 | reported to the community leaders responsible for enforcement at 63 | info@siege.ai. 64 | All complaints will be reviewed and investigated promptly and fairly. 65 | 66 | All community leaders are obligated to respect the privacy and security of the 67 | reporter of any incident. 68 | 69 | ## Enforcement Guidelines 70 | 71 | Community leaders will follow these Community Impact Guidelines in determining 72 | the consequences for any action they deem in violation of this Code of Conduct: 73 | 74 | ### 1. Correction 75 | 76 | **Community Impact**: Use of inappropriate language or other behavior deemed 77 | unprofessional or unwelcome in the community. 78 | 79 | **Consequence**: A private, written warning from community leaders, providing 80 | clarity around the nature of the violation and an explanation of why the 81 | behavior was inappropriate. A public apology may be requested. 82 | 83 | ### 2. Warning 84 | 85 | **Community Impact**: A violation through a single incident or series 86 | of actions. 87 | 88 | **Consequence**: A warning with consequences for continued behavior. No 89 | interaction with the people involved, including unsolicited interaction with 90 | those enforcing the Code of Conduct, for a specified period of time. This 91 | includes avoiding interactions in community spaces as well as external channels 92 | like social media. Violating these terms may lead to a temporary or 93 | permanent ban. 94 | 95 | ### 3. Temporary Ban 96 | 97 | **Community Impact**: A serious violation of community standards, including 98 | sustained inappropriate behavior. 99 | 100 | **Consequence**: A temporary ban from any sort of interaction or public 101 | communication with the community for a specified period of time. No public or 102 | private interaction with the people involved, including unsolicited interaction 103 | with those enforcing the Code of Conduct, is allowed during this period. 104 | Violating these terms may lead to a permanent ban. 105 | 106 | ### 4. Permanent Ban 107 | 108 | **Community Impact**: Demonstrating a pattern of violation of community 109 | standards, including sustained inappropriate behavior, harassment of an 110 | individual, or aggression toward or disparagement of classes of individuals. 111 | 112 | **Consequence**: A permanent ban from any sort of public interaction within 113 | the community. 114 | 115 | ## Attribution 116 | 117 | This Code of Conduct is adapted from the [Contributor Covenant][homepage], 118 | version 2.0, available at 119 | https://www.contributor-covenant.org/version/2/0/code_of_conduct.html. 120 | 121 | Community Impact Guidelines were inspired by [Mozilla's code of conduct 122 | enforcement ladder](https://github.com/mozilla/diversity). 123 | 124 | [homepage]: https://www.contributor-covenant.org 125 | 126 | For answers to common questions about this code of conduct, see the FAQ at 127 | https://www.contributor-covenant.org/faq. Translations are available at 128 | https://www.contributor-covenant.org/translations. 129 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contributing to django-typesense 2 | 3 | First off, thanks for taking the time to contribute! ❤️ 4 | 5 | All types of contributions are encouraged and valued. See the [Table of Contents](#table-of-contents) for different ways to help and details about how this project handles them. Please make sure to read the relevant section before making your contribution. It will make it a lot easier for us maintainers and smooth out the experience for all involved. The community looks forward to your contributions. 🎉 6 | 7 | > And if you like the project, but just don't have time to contribute, that's fine. There are other easy ways to support the project and show your appreciation, which we would also be very happy about: 8 | > 9 | > - Star the project 10 | > - Tweet about it 11 | > - Refer this project in your project's readme 12 | > - Mention the project at local meetups and tell your friends/colleagues 13 | 14 | ## Table of Contents 15 | 16 | - [I Have a Question](#i-have-a-question) 17 | - [I Want To Contribute](#i-want-to-contribute) 18 | - [Reporting Bugs](#reporting-bugs) 19 | - [Suggesting Enhancements](#suggesting-enhancements) 20 | - [Your First Code Contribution](#your-first-code-contribution) 21 | - [Styleguides](#styleguides) 22 | - [Commit Messages](#commit-messages) 23 | - [Branch Names](#branch-names) 24 | - [Doc Strings](#doc-strings) 25 | 26 | ## I Have a Question 27 | 28 | > If you want to ask a question, we assume that you have read the available [Documentation](https://github.com/Siege-Software/django-typesense/blob/main/README.md). 29 | 30 | Before you ask a question, it is best to search for existing [Issues](https://github.com/Siege-Software/django-typesense/issues) that might help you. In case you have found a suitable issue and still need clarification, you can write your question in this issue. It is also advisable to search the internet for answers first. 31 | 32 | If you then still feel the need to ask a question and need clarification, we recommend the following: 33 | 34 | - Open an [Issue](https://github.com/Siege-Software/django-typesense/issues/new/choose). 35 | - Provide as much context as you can about what you're running into. 36 | - Provide project and platform versions (python, django, etc), depending on what seems relevant. 37 | 38 | We will then take care of the issue as soon as possible. 39 | 40 | ## I Want To Contribute 41 | 42 | > ### Legal Notice 43 | > 44 | > When contributing to this project, you must agree that you have authored 100% of the content, that you have the necessary rights to the content and that the content you contribute may be provided under the project license. 45 | 46 | ### Reporting Bugs 47 | 48 | #### Before Submitting a Bug Report 49 | 50 | A good bug report shouldn't leave others needing to chase you up for more information. Therefore, we ask you to investigate carefully, collect information and describe the issue in detail in your report. Please complete the following steps in advance to help us fix any potential bug as fast as possible. 51 | 52 | - Make sure that you are using the latest version. 53 | - Determine if your bug is really a bug and not an error on your side e.g. using incompatible environment components/versions (Make sure that you have read the [documentation](https://github.com/Siege-Software/django-typesense/blob/main/README.md). If you are looking for support, you might want to check [this section](#i-have-a-question)). 54 | - To see if other users have experienced (and potentially already solved) the same issue you are having, check if there is not already a bug report existing for your bug or error in the [bug tracker](https://github.com/Siege-Software/django-typesense/issues/?q=label%3Abug). 55 | - Also make sure to search the internet (including Stack Overflow) to see if users outside of the GitHub community have discussed the issue. 56 | - Collect information about the bug: 57 | - Stack trace (Traceback) 58 | - OS, Platform and Version (Windows, Linux, macOS, x86, ARM) 59 | - Version of python, runtime environment, depending on what seems relevant. 60 | - Possibly your input and the output 61 | - Can you reliably reproduce the issue? And can you also reproduce it with older versions? 62 | 63 | #### How Do I Submit a Good Bug Report? 64 | 65 | > You must never report security related issues, vulnerabilities or bugs including sensitive information to the issue tracker, or elsewhere in public. Instead sensitive bugs must be sent by email to . 66 | 67 | We use GitHub issues to track bugs and errors. If you run into an issue with the project: 68 | 69 | - Open an [Issue](https://github.com/Siege-Software/django-typesense/issues/new?template=bug_report.md&title=Please+replace+with+a+clear+and+descriptive+title). (Since we can't be sure at this point whether it is a bug or not, we ask you not to talk about a bug yet and not to label the issue.) 70 | - Explain the behavior you would expect and the actual behavior. 71 | - Please provide as much context as possible and describe the *reproduction steps* that someone else can follow to recreate the issue on their own. This usually includes your code. For good bug reports you should isolate the problem and create a reduced test case. 72 | - Provide the information you collected in the previous section. 73 | 74 | Once it's filed: 75 | 76 | - The project team will label the issue accordingly. 77 | - A team member will try to reproduce the issue with your provided steps. If there are no reproduction steps or no obvious way to reproduce the issue, the team will ask you for those steps and mark the issue as `needs-repro`. Bugs with the `needs-repro` tag will not be addressed until they are reproduced. 78 | - If the team is able to reproduce the issue, it will be marked `needs-fix`, as well as possibly other tags (such as `critical`), and the issue will be left to be [implemented by someone](#your-first-code-contribution). 79 | 80 | ### Suggesting Enhancements 81 | 82 | This section guides you through submitting an enhancement suggestion for django-typesense, **including completely new features and minor improvements to existing functionality**. Following these guidelines will help maintainers and the community to understand your suggestion and find related suggestions. 83 | 84 | #### Before Submitting an Enhancement 85 | 86 | - Make sure that you are using the latest version. 87 | - Read the [documentation](https://github.com/Siege-Software/django-typesense/blob/main/README.md) carefully and find out if the functionality is already covered, maybe by an individual configuration. 88 | - Perform a [search](https://github.com/Siege-Software/django-typesense/issues) to see if the enhancement has already been suggested. If it has, add a comment to the existing issue instead of opening a new one. 89 | - Find out whether your idea fits with the scope and aims of the project. It's up to you to make a strong case to convince the project's developers of the merits of this feature. Keep in mind that we want features that will be useful to the majority of our users and not just a small subset. If you're just targeting a minority of users, consider writing an add-on/plugin library. 90 | 91 | #### How Do I Submit a Good Enhancement Suggestion? 92 | 93 | Enhancement suggestions are tracked as [GitHub issues](https://github.com/Siege-Software/django-typesense/issues). 94 | 95 | - Use a **clear and descriptive title** for the issue to identify the suggestion. 96 | - Provide a **step-by-step description of the suggested enhancement** in as many details as possible. 97 | - **Describe the current behavior** and **explain which behavior you expected to see instead** and why. At this point you can also tell which alternatives do not work for you. 98 | - You may want to **include screenshots and animated GIFs** which help you demonstrate the steps or point out the part which the suggestion is related to. 99 | - **Explain why this enhancement would be useful** to most django-typesense users. You may also want to point out the other projects that solved it better and which could serve as inspiration. 100 | 101 | ### Your First Code Contribution 102 | 103 | #### Getting Started 104 | 105 | - Clone the repo and switch to the project directory 106 | 107 | ```sh 108 | git clone https://github.com/Siege-Software/django-typesense.git 109 | cd django-typesense 110 | ``` 111 | 112 | - Create a virtual environment and activate it 113 | 114 | ```sh 115 | # Python version >=3.8 116 | python -m venv venv 117 | 118 | # For Linux/MacOS 119 | source venv/bin/activate 120 | # For Windows 121 | venv\Scripts\activate 122 | ``` 123 | 124 | - Install project dependencies 125 | 126 | ```sh 127 | pip install --upgrade pip 128 | pip install -r requirements-dev.txt 129 | ``` 130 | 131 | - Enable pre-commit hooks 132 | 133 | ```sh 134 | pre-commit install 135 | ``` 136 | 137 | - Create your branch from the main branch as per [branch name style guide](#branch-names) 138 | 139 | ```sh 140 | git checkout -b 141 | ``` 142 | 143 | #### Running Tests 144 | 145 | - Install `django` before running tests for the first time 146 | 147 | ```sh 148 | pip install django 149 | 150 | # Install specific django version 151 | pip install django==4.2 152 | ``` 153 | 154 | - To run the tests quickly 155 | 156 | ```sh 157 | python runtests.py 158 | ``` 159 | 160 | - Get test coverage report after running the tests 161 | 162 | ```sh 163 | coverage run runtests.py 164 | 165 | # Terminal report 166 | coverage report 167 | 168 | # HTML report 169 | coverage report html 170 | ``` 171 | 172 | ## Styleguides 173 | 174 | ### Commit Messages 175 | 176 | - All commits **must** be [signed](https://docs.github.com/en/authentication/managing-commit-signature-verification/signing-commits) 177 | - Commit messages should end with the issue number 178 | 179 | ```sh 180 | git commit -m "My changes to the project (#23)" 181 | ``` 182 | 183 | - If the commit subject exceeds 100 characters, add a body to the commit message 184 | 185 | ```sh 186 | git commit 187 | ``` 188 | 189 | ```sh 190 | Summarize changes in around 100 characters or less 191 | 192 | More detailed description of changes. 193 | 194 | Resolves: #23 195 | See also: #1620 196 | ``` 197 | 198 | - The subject line should be capitalized and must not end in a period 199 | - The subject line must be written in imperative mood (Fix, not Fixed / Fixes etc.) 200 | - The body copy must be wrapped at 72 columns 201 | - The body copy must only contain explanations as to what and why, never how. The latter belongs in documentation and implementation. 202 | 203 | ### Branch Names 204 | 205 | - There are three types of branches to pick from i.e 206 | - `bug-fix` - This is used for branches that are fixing a bug 207 | - `feature` - This is used for branches that are adding a feature 208 | - `chore` - This is used for branches that improve the project such as updating docs, updating tests 209 | - Branch name format 210 | 211 | ```sh 212 | -/ 213 | 214 | # Examples 215 | 23-bug-fix/fix-m2m-changed-signal 216 | 23-feature/add-typesense-search-util 217 | 23-chore/update-utils-tests 218 | ``` 219 | 220 | ### Doc Strings 221 | 222 | We are using [NumpyDoc](https://numpydoc.readthedocs.io/en/latest/format.html) style for docstrings. 223 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2023 Siege Software 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | recursive-include django_typesense/static * -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # django typesense 2 | 3 | [![Build](https://github.com/Siege-Software/django-typesense/workflows/build/badge.svg?branch=main)](https://github.com/Siege-Software/django-typesense/actions?workflow=CI) 4 | [![codecov](https://codecov.io/gh/Siege-Software/django-typesense/branch/main/graph/badge.svg?token=S4W0E84821)](https://codecov.io/gh/Siege-Software/django-typesense) 5 | [![Code style: black](https://img.shields.io/badge/code%20style-black-000000.svg)](https://github.com/psf/black) 6 | ![PyPI download month](https://img.shields.io/pypi/dm/django-typesense.svg) 7 | [![PyPI version](https://badge.fury.io/py/django-typesense.svg)](https://pypi.python.org/pypi/django-typesense/) 8 | ![Python versions](https://img.shields.io/badge/python-%3E%3D3.8-brightgreen) 9 | ![Django Versions](https://img.shields.io/badge/django-%3E%3D3.2-brightgreen) 10 | [![PyPI License](https://img.shields.io/pypi/l/django-typesense.svg)](https://pypi.org/project/django-typesense/) 11 | 12 | 13 | ## What is it? 14 | Faster Django Admin powered by [Typesense](https://typesense.org/) 15 | 16 | ## Quick Start Guide 17 | 18 | ### Installation 19 | 20 | ```sh 21 | pip install django-typesense 22 | ``` 23 | 24 | or install directly from github to test the most recent version 25 | 26 | ```sh 27 | pip install git+https://github.com/Siege-Software/django-typesense.git 28 | ``` 29 | 30 | ### Configuration 31 | 32 | Update your settings to include the following 33 | 34 | - Add `django_typesense` to the list of installed apps. 35 | 36 | ```py 37 | ... 38 | INSTALLED_APPS = [ 39 | ... 40 | "django_typesense" 41 | ] 42 | ``` 43 | 44 | - Add `TYPESENSE` connection details 45 | 46 | ```py 47 | ... 48 | TYPESENSE = { 49 | "api_key": "xyz", 50 | "nodes": [{"host": "0.0.0.0", "port": "8108", "protocol": "http"}], 51 | "connection_timeout_seconds": 2 52 | } 53 | ``` 54 | 55 | Follow this [guide](https://typesense.org/docs/guide/install-typesense.html#option-1-typesense-cloud) to install and run typesense 56 | 57 | ### Create Collections 58 | Throughout this guide, we’ll refer to the following models, which comprise a song catalogue application: 59 | 60 | ``` 61 | from django.db import models 62 | 63 | 64 | class Genre(models.Model): 65 | name = models.CharField(max_length=100) 66 | 67 | def __str__(self): 68 | return self.name 69 | 70 | 71 | class Artist(models.Model): 72 | name = models.CharField(max_length=200) 73 | 74 | def __str__(self): 75 | return self.name 76 | 77 | 78 | class Song(models.Model): 79 | title = models.CharField(max_length=100) 80 | genre = models.ForeignKey(Genre, on_delete=models.CASCADE) 81 | release_date = models.DateField(blank=True, null=True) 82 | artists = models.ManyToManyField(Artist) 83 | number_of_comments = models.IntegerField(default=0) 84 | number_of_views = models.IntegerField(default=0) 85 | duration = models.DurationField() 86 | description = models.TextField() 87 | 88 | def __str__(self): 89 | return self.title 90 | 91 | @property 92 | def release_date_timestamp(self): 93 | # read https://typesense.org/docs/0.25.0/api/collections.html#indexing-dates 94 | return self.release_date.timestamp() if self.release_date else self.release_date 95 | 96 | def artist_names(self): 97 | return list(self.artists.all().values_list('name', flat=True)) 98 | 99 | ``` 100 | 101 | For such an application, you might be interested in improving the search and load times on the song records list view. 102 | 103 | ``` 104 | from django_typesense.collections import TypesenseCollection 105 | from django_typesense import fields 106 | 107 | 108 | class SongCollection(TypesenseCollection): 109 | # At least one of the indexed fields has to be provided as one of the `query_by_fields`. Must be a CharField 110 | query_by_fields = 'title,artist_names,genre_name' 111 | 112 | title = fields.TypesenseCharField() 113 | genre_name = fields.TypesenseCharField(value='genre.name') 114 | genre_id = fields.TypesenseSmallIntegerField() 115 | release_date = fields.TypesenseDateField(value='release_date_timestamp', optional=True) 116 | artist_names = fields.TypesenseArrayField(base_field=fields.TypesenseCharField(), value='artist_names') 117 | number_of_comments = fields.SmallIntegerField(index=False, optional=True) 118 | number_of_views = fields.SmallIntegerField(index=False, optional=True) 119 | duration = fields.DurationField() 120 | ``` 121 | 122 | It's okay to store fields that you don't intend to search but to display on the admin. Such fields should be marked as un-indexed e.g: 123 | 124 | number_of_views = fields.SmallIntegerField(index=False, optional=True) 125 | 126 | Update the song model as follows: 127 | ``` 128 | from django_typesense.mixins import TypesenseModelMixin 129 | 130 | class Song(TypesenseModelMixin): 131 | ... 132 | collection_class = SongCollection 133 | ... 134 | ``` 135 | 136 | The `TypesenseModelMixin` provides a Manager that overrides the `update` and `delete` methods of the Queryset. 137 | If your model has a custom manager, make sure the custom manager inherits `django_typesense.mixin.TypesenseManager` 138 | 139 | How the value of a field is retrieved from a model instance: 140 | 1. The collection field name is called as a property of the model instance 141 | 2. If `value` is provided, it will be called as a property or method of the model instance 142 | 143 | Where the collections live is totally dependent on you but we recommend having a `collections.py` file 144 | in the django app where the model you are creating a collection for is. 145 | 146 | > [!NOTE] 147 | > We recommend displaying data from ForeignKey or OneToOne fields as string attributes using the display decorator to 148 | > avoid triggering database queries that will negatively affect performance 149 | > [Issue #16](https://github.com/Siege-Software/django-typesense/issues/16). 150 | 151 | Instead of this in the admin: 152 | ``` 153 | @admin.display('Genre') 154 | def genre_name(self, obj): 155 | return obj.genre.name 156 | ``` 157 | 158 | Do this: 159 | 160 | ``` 161 | @admin.display('Genre') 162 | def genre_name(self, obj): 163 | # genre_name is field in the Collection. You can also store the object url as html 164 | return obj.genre_name 165 | ``` 166 | 167 | ### Search Collections 168 | 169 | Using Typesense for search 170 | 171 | ```py 172 | from django_typesense.utils import typesense_search 173 | 174 | from .models import Song 175 | 176 | 177 | def search_songs(request): 178 | search_term = request.GET.get("q", None) 179 | songs = Song.objects.all() 180 | 181 | if search_term: 182 | data = { 183 | "q": search_term, 184 | "query_by": Song.collection_class.query_by_fields, 185 | # Include other search parameters here 186 | # https://typesense.org/docs/27.1/api/search.html#search-parameters 187 | } 188 | res = typesense_search(Song.collection_class.schema_name, **data) 189 | ids = [result["document"]["id"] for result in res["hits"]] 190 | songs = songs.filter(id__in=ids) 191 | 192 | ... 193 | ``` 194 | 195 | ### Update Collection Schema 196 | To add or remove fields to a collection's schema in place, update your collection then run: 197 | `python manage.py updatecollections`. Consider adding this to your CI/CD pipeline. 198 | 199 | This also updates the [synonyms](#synonyms) 200 | 201 | 202 | ### How updates are made to Typesense 203 | 1. Signals - 204 | `django-typesense` listens to signal events (`post_save`, `pre_delete`, `m2m_changed`) to update typesense records. 205 | If [`update_fields`](https://docs.djangoproject.com/en/4.2/ref/models/instances/#specifying-which-fields-to-save) 206 | were provided in the save method, only these fields will be updated in typesense. 207 | 208 | 2. Update query - 209 | `django-typesense` overrides Django's `QuerySet.update` to make updates to typesense on the specified fields 210 | 211 | 3. Manual - 212 | You can also update typesense records manually e.g after doing a `bulk_create` 213 | ``` 214 | objs = Song.objects.bulk_create( 215 | [ 216 | Song(title="Watch What I Do"), 217 | Song(title="Midnight City"), 218 | ] 219 | ) 220 | collection = SongCollection(objs, many=True) 221 | collection.update() 222 | ``` 223 | 224 | ### Admin Integration 225 | To make a model admin display and search from the model's Typesense collection, the admin class should 226 | inherit `TypesenseSearchAdminMixin`. This also adds Live Search to your admin changelist view. 227 | 228 | ``` 229 | from django_typesense.admin import TypesenseSearchAdminMixin 230 | 231 | @admin.register(Song) 232 | class SongAdmin(TypesenseSearchAdminMixin): 233 | ... 234 | list_display = ['title', 'genre_name', 'release_date', 'number_of_views', 'duration'] 235 | 236 | @admin.display(description='Genre') 237 | def genre_name(self, obj): 238 | return obj.genre.name 239 | ... 240 | 241 | ``` 242 | 243 | ### Indexing 244 | For the initial setup, you will need to index in bulk. Bulk updating is multi-threaded. Depending on your system specs, you should set the `batch_size` keyword argument. 245 | 246 | ``` 247 | from django_typesense.utils import bulk_delete_typsense_records, bulk_update_typsense_records 248 | 249 | model_qs = Song.objects.all().order_by('id') # querysets should be ordered 250 | bulk_update_typesense_records(model_qs, batch_size=1024) 251 | ``` 252 | 253 | ### Custom Admin Filters 254 | To make use of custom admin filters, define a `filter_by` property in the filter definition. 255 | Define boolean typesense field `has_views` that gets it's value from a model property. This is example is not necessarily practical but for demo purposes. 256 | 257 | ``` 258 | # models.py 259 | class Song(models.Model): 260 | ... 261 | @property 262 | def has_views(self): 263 | return self.number_of_views > 0 264 | ... 265 | 266 | # collections.py 267 | class SongCollection(TypesenseCollection): 268 | ... 269 | has_views = fields.TypesenseBooleanField() 270 | ... 271 | ``` 272 | 273 | ``` 274 | class HasViewsFilter(admin.SimpleListFilter): 275 | title = _('Has Views') 276 | parameter_name = 'has_views' 277 | 278 | def lookups(self, request, model_admin): 279 | return ( 280 | ('all', 'All'), 281 | ('True', 'Yes'), 282 | ('False', 'No') 283 | ) 284 | 285 | def queryset(self, request, queryset): 286 | # This is used by the default django admin 287 | if self.value() == 'True': 288 | return queryset.filter(number_of_views__gt=0) 289 | elif self.value() == 'False': 290 | return queryset.filter(number_of_views=0) 291 | 292 | return queryset 293 | 294 | @property 295 | def filter_by(self): 296 | # This is used by typesense 297 | if self.value() == 'True': 298 | return {"has_views": "=true"} 299 | elif self.value() == 'False': 300 | return {"has_views": "!=false"} 301 | 302 | return {} 303 | ``` 304 | 305 | Note that simple lookups like the one above are done by default (hence no need to define `filter_by`) if 306 | the `parameter_name` is a field in the collection 307 | 308 | ### Synonyms 309 | The [synonyms](https://typesense.org/docs/0.25.1/api/synonyms.html) feature allows you to define search terms that 310 | should be considered equivalent. Synonyms should be defined with classes that inherit from `Synonym` 311 | 312 | ``` 313 | from django_typesense.collections import Synonym 314 | 315 | # say you need users searching the genre hip-hop to get results if they use the search term rap 316 | 317 | class HipHopSynonym(Synonym): 318 | name = 'hip-hop-synonyms' 319 | synonyms = ['hip-hop', 'rap'] 320 | 321 | # Update the collection to include the synonym 322 | class SongCollection(TypesenseCollection): 323 | ... 324 | synonyms = [HipHopSynonym] 325 | ... 326 | 327 | ``` 328 | To update the collection with any changes made to synonyms run `python manage.py updatecollections` 329 | 330 | 331 | -------------------------------------------------------------------------------- /django_typesense/__init__.py: -------------------------------------------------------------------------------- 1 | __version__ = "0.1.3" 2 | -------------------------------------------------------------------------------- /django_typesense/admin.py: -------------------------------------------------------------------------------- 1 | import logging 2 | 3 | from django.contrib import admin 4 | from django.contrib.auth.admin import csrf_protect_m 5 | from django.db.models import QuerySet 6 | from django.forms import forms 7 | from django.http import JsonResponse 8 | 9 | from django_typesense.mixins import TypesenseModelMixin 10 | from django_typesense.utils import typesense_search, export_documents 11 | from django_typesense.paginator import TypesenseSearchPaginator 12 | 13 | logger = logging.getLogger(__name__) 14 | 15 | 16 | class TypesenseSearchAdminMixin(admin.ModelAdmin): 17 | typesense_search_fields = [] 18 | 19 | def get_typesense_search_fields(self, request): 20 | """ 21 | Return a sequence containing the fields to be searched whenever 22 | somebody submits a search query. 23 | """ 24 | return self.typesense_search_fields 25 | 26 | @property 27 | def media(self): 28 | super_media = super().media 29 | return forms.Media( 30 | js=super_media._js + ["admin/js/search-typesense.js"], 31 | css=super_media._css, 32 | ) 33 | 34 | @csrf_protect_m 35 | def changelist_view(self, request, extra_context=None): 36 | """ 37 | The 'change list' admin view for this model. 38 | """ 39 | template_response = super().changelist_view(request, extra_context) 40 | 41 | is_ajax = request.META.get("HTTP_X_REQUESTED_WITH") == "XMLHttpRequest" 42 | if is_ajax: 43 | html = template_response.render().rendered_content 44 | return JsonResponse(data={"html": html}, safe=False) 45 | 46 | return template_response 47 | 48 | def get_sortable_by(self, request): 49 | """ 50 | Get sortable fields; these are fields that sort is defaulted or set to True. 51 | 52 | Args: 53 | request: the HttpRequest 54 | 55 | Returns: 56 | A list of field names 57 | """ 58 | 59 | sortable_fields = super().get_sortable_by(request) 60 | return set(sortable_fields).intersection( 61 | self.model.collection_class.sortable_fields 62 | ) 63 | 64 | def get_results(self, request): 65 | """ 66 | Get all indexed data without any filtering or specific search terms. Works like `ModelAdmin.get_queryset()` 67 | 68 | Args: 69 | request: the HttpRequest 70 | 71 | Returns: 72 | A list of the typesense results 73 | """ 74 | 75 | return typesense_search( 76 | collection_name=self.model.collection_class.schema_name, 77 | q="*", 78 | query_by=self.model.collection_class.query_by_fields, 79 | ) 80 | 81 | def get_changelist(self, request, **kwargs): 82 | """ 83 | Return the ChangeList class for use on the changelist page. 84 | """ 85 | from django_typesense.changelist import TypesenseChangeList 86 | 87 | return TypesenseChangeList 88 | 89 | def get_paginator( 90 | self, request, results, per_page, orphans=0, allow_empty_first_page=True 91 | ): 92 | # fallback incase we receive a queryset. 93 | if isinstance(results, QuerySet): 94 | return super().get_paginator( 95 | request, results, per_page, orphans, allow_empty_first_page 96 | ) 97 | 98 | return TypesenseSearchPaginator( 99 | results, per_page, orphans, allow_empty_first_page, self.model 100 | ) 101 | 102 | def get_typesense_search_results( 103 | self, 104 | request, 105 | search_term: str, 106 | page_num: int = 1, 107 | filter_by: str = "", 108 | sort_by: str = "", 109 | list_per_page: int = None 110 | ): 111 | """ 112 | Get the results from typesense with the provided filtering, sorting, pagination and search parameters applied 113 | 114 | Args: 115 | search_term: The search term provided in the search form 116 | request: the current request object 117 | page_num: The requested page number 118 | filter_by: The filtering parameters 119 | sort_by: The sort parameters 120 | list_per_page: The number of results to return per page 121 | 122 | Returns: 123 | A list of typesense results 124 | """ 125 | if list_per_page is None: 126 | list_per_page = self.list_per_page 127 | 128 | results = typesense_search( 129 | collection_name=self.model.collection_class.schema_name, 130 | q=search_term or "*", 131 | query_by=self.model.collection_class.query_by_fields, 132 | page=page_num, 133 | per_page=list_per_page, 134 | filter_by=filter_by, 135 | sort_by=sort_by, 136 | ) 137 | return results 138 | 139 | def get_search_results(self, request, queryset, search_term): 140 | if not request.POST.get("action"): 141 | may_have_duplicates = False 142 | results = self.get_typesense_search_results(request, search_term) 143 | ids = [result["document"]["id"] for result in results["hits"]] 144 | queryset = queryset.filter(id__in=ids) 145 | else: 146 | # id_dict_list = export_documents( 147 | # self.model.collection_class.schema_name, include_fields=["id"] 148 | # ) 149 | # queryset = queryset.filter( 150 | # id__in=[id_dict["id"] for id_dict in id_dict_list] 151 | # ) 152 | queryset, may_have_duplicates = super().get_search_results( 153 | request, queryset, search_term 154 | ) 155 | 156 | return queryset, may_have_duplicates 157 | -------------------------------------------------------------------------------- /django_typesense/apps.py: -------------------------------------------------------------------------------- 1 | from django.apps import AppConfig 2 | 3 | 4 | class DjangoTypesenseConfig(AppConfig): 5 | default_auto_field = "django.db.models.BigAutoField" 6 | name = "django_typesense" 7 | 8 | def ready(self): 9 | import django_typesense.signals 10 | -------------------------------------------------------------------------------- /django_typesense/changelist.py: -------------------------------------------------------------------------------- 1 | import logging 2 | 3 | from django import forms 4 | from django.contrib import messages 5 | from django.contrib.admin.exceptions import DisallowedModelAdminToField 6 | from django.contrib.admin.options import ( 7 | IS_POPUP_VAR, 8 | TO_FIELD_VAR, 9 | IncorrectLookupParameters, 10 | ) 11 | from django.contrib.admin.views.main import ChangeList 12 | from django.core.exceptions import ImproperlyConfigured, SuspiciousOperation 13 | from django.core.paginator import InvalidPage 14 | from django.db.models import OrderBy, OuterRef, Exists 15 | from django.utils.translation import gettext 16 | from django.utils.dateparse import parse_datetime 17 | 18 | from django_typesense.fields import TYPESENSE_DATETIME_FIELDS 19 | from django_typesense.utils import get_unix_timestamp 20 | 21 | # Changelist settings 22 | ALL_VAR = "all" 23 | ORDER_VAR = "o" 24 | ORDER_TYPE_VAR = "ot" 25 | PAGE_VAR = "p" 26 | SEARCH_VAR = "q" 27 | ERROR_FLAG = "e" 28 | 29 | IGNORED_PARAMS = ( 30 | ALL_VAR, 31 | ORDER_VAR, 32 | ORDER_TYPE_VAR, 33 | SEARCH_VAR, 34 | IS_POPUP_VAR, 35 | TO_FIELD_VAR, 36 | ) 37 | 38 | logger = logging.getLogger(__name__) 39 | 40 | 41 | class ChangeListSearchForm(forms.Form): 42 | def __init__(self, *args, **kwargs): 43 | super().__init__(*args, **kwargs) 44 | # Populate "fields" dynamically because SEARCH_VAR is a variable: 45 | self.fields = { 46 | SEARCH_VAR: forms.CharField(required=False, strip=False), 47 | } 48 | 49 | 50 | class TypesenseChangeList(ChangeList): 51 | search_form_class = ChangeListSearchForm 52 | 53 | def __init__( 54 | self, 55 | request, 56 | model, 57 | list_display, 58 | list_display_links, 59 | list_filter, 60 | date_hierarchy, 61 | search_fields, 62 | list_select_related, 63 | list_per_page, 64 | list_max_show_all, 65 | list_editable, 66 | model_admin, 67 | sortable_by, 68 | search_help_text, 69 | ): 70 | self.model = model 71 | self.opts = model._meta 72 | self.lookup_opts = self.opts 73 | self.root_queryset = model_admin.get_queryset(request) 74 | 75 | # TYPESENSE 76 | self.root_results = model_admin.get_results(request) 77 | 78 | self.list_display = list_display 79 | self.list_display_links = list_display_links 80 | self.list_filter = list_filter 81 | self.has_filters = None 82 | self.has_active_filters = None 83 | self.clear_all_filters_qs = None 84 | self.date_hierarchy = date_hierarchy 85 | self.search_fields = model_admin.get_typesense_search_fields(request) 86 | self.list_select_related = list_select_related 87 | self.list_per_page = min(list_per_page, 250) # Typesense Max hits per page 88 | self.list_max_show_all = min( 89 | list_max_show_all, 250 90 | ) # Typesense Max hits per page 91 | self.model_admin = model_admin 92 | self.preserved_filters = model_admin.get_preserved_filters(request) 93 | self.sortable_by = sortable_by 94 | self.search_help_text = search_help_text 95 | 96 | # Get django_typesense parameters from the query string. 97 | _search_form = self.search_form_class(request.GET) 98 | if not _search_form.is_valid(): 99 | for error in _search_form.errors.values(): 100 | messages.error(request, ", ".join(error)) 101 | self.query = _search_form.cleaned_data.get(SEARCH_VAR) or "" 102 | try: 103 | self.page_num = int(request.GET.get(PAGE_VAR, 1)) 104 | except ValueError: 105 | self.page_num = 1 106 | self.show_all = ALL_VAR in request.GET 107 | self.is_popup = IS_POPUP_VAR in request.GET 108 | to_field = request.GET.get(TO_FIELD_VAR) 109 | if to_field and not model_admin.to_field_allowed(request, to_field): 110 | raise DisallowedModelAdminToField( 111 | "The field %s cannot be referenced." % to_field 112 | ) 113 | self.to_field = to_field 114 | self.params = dict(request.GET.items()) 115 | if PAGE_VAR in self.params: 116 | del self.params[PAGE_VAR] 117 | if ERROR_FLAG in self.params: 118 | del self.params[ERROR_FLAG] 119 | 120 | if self.is_popup: 121 | self.list_editable = () 122 | else: 123 | self.list_editable = list_editable 124 | 125 | # TYPESENSE 126 | self.results = self.get_typesense_results(request) 127 | self.get_results(request) 128 | 129 | if self.is_popup: 130 | title = gettext("Select %s") 131 | elif self.model_admin.has_change_permission(request): 132 | title = gettext("Select %s to change") 133 | else: 134 | title = gettext("Select %s to view") 135 | self.title = title % self.opts.verbose_name 136 | self.pk_attname = self.lookup_opts.pk.attname 137 | 138 | def get_results(self, request): 139 | paginator = self.model_admin.get_paginator( 140 | request, self.results, self.list_per_page 141 | ) 142 | # Get the number of objects, with admin filters applied. 143 | result_count = paginator.count 144 | 145 | # Get the total number of objects, with no admin filters applied. 146 | if self.model_admin.show_full_result_count: 147 | full_result_count = self.root_results["found"] 148 | else: 149 | full_result_count = None 150 | can_show_all = result_count <= self.list_max_show_all 151 | multi_page = result_count > self.list_per_page 152 | 153 | # Get the list of objects to display on this page. 154 | if (self.show_all and can_show_all) or not multi_page: 155 | paginator = self.model_admin.get_paginator( 156 | request, self.results, self.list_max_show_all 157 | ) 158 | result_list = paginator.results 159 | else: 160 | try: 161 | result_list = paginator.page(self.page_num).object_list 162 | except InvalidPage: 163 | raise IncorrectLookupParameters 164 | 165 | self.result_count = result_count 166 | self.show_full_result_count = self.model_admin.show_full_result_count 167 | # Admin actions are shown if there is at least one entry 168 | # or if entries are not counted because show_full_result_count is disabled 169 | self.show_admin_actions = not self.show_full_result_count or bool( 170 | full_result_count 171 | ) 172 | self.full_result_count = full_result_count 173 | self.result_list = result_list 174 | self.can_show_all = can_show_all 175 | self.multi_page = multi_page 176 | self.paginator = paginator 177 | 178 | def get_typesense_ordering(self, request): 179 | """ 180 | Return the list of ordering fields for the change list. 181 | First check the get_ordering() method in model admin, then check 182 | the object's default ordering. Then, any manually-specified ordering 183 | from the query string overrides anything. Finally, a deterministic 184 | order is guaranteed by calling _get_deterministic_ordering() with the 185 | constructed ordering. 186 | """ 187 | params = self.params 188 | ordering = list( 189 | self.model_admin.get_ordering(request) or self._get_default_ordering() 190 | ) 191 | if ORDER_VAR in params: 192 | # Clear ordering and used params 193 | ordering = [] 194 | order_params = params[ORDER_VAR].split(".") 195 | for p in order_params: 196 | try: 197 | _, pfx, idx = p.rpartition("-") 198 | field_name = self.list_display[int(idx)] 199 | order_field = self.get_ordering_field(field_name) 200 | if not order_field: 201 | continue # No 'admin_order_field', skip it 202 | if isinstance(order_field, OrderBy): 203 | if pfx == "-": 204 | order_field = order_field.copy() 205 | order_field.reverse_ordering() 206 | ordering.append(order_field) 207 | elif hasattr(order_field, "resolve_expression"): 208 | # order_field is an expression. 209 | ordering.append( 210 | order_field.desc() if pfx == "-" else order_field.asc() 211 | ) 212 | # reverse order if order_field has already "-" as prefix 213 | elif order_field.startswith("-") and pfx == "-": 214 | ordering.append(order_field[1:]) 215 | else: 216 | ordering.append(pfx + order_field) 217 | except (IndexError, ValueError): 218 | continue # Invalid ordering specified, skip it. 219 | 220 | return self._get_deterministic_ordering(ordering) 221 | 222 | def get_sort_by(self, ordering): 223 | sort_dict = {} 224 | fields = self.model.collection_class.get_fields() 225 | 226 | for param in ordering: 227 | if param.startswith("-"): 228 | _, field_name = param.split("-") 229 | order = "desc" 230 | else: 231 | field_name = param 232 | order = "asc" 233 | 234 | # Temporarily left out: Could not find a field named `id` in the schema for sorting 235 | if field_name in ["pk", "id"]: 236 | # sort_dict['id'] = order 237 | continue 238 | 239 | if not fields.get(field_name): 240 | continue 241 | 242 | if not fields[field_name].sort: 243 | continue 244 | 245 | sort_dict[field_name] = order 246 | 247 | sort_by = ",".join([f"{key}:{value}" for key, value in sort_dict.items()]) 248 | return sort_by 249 | 250 | def get_search_filters(self, field_name: str, used_parameters: dict): 251 | search_filters_dict = {} 252 | if not used_parameters: 253 | return search_filters_dict 254 | 255 | lookup_to_operator = { 256 | "gte": ">=", 257 | "gt": ">", 258 | "lte": "<=", 259 | "lt": "<", 260 | "iexact": "=", 261 | "exact": "=", 262 | } 263 | max_val, min_val, lookup, value = None, None, None, None 264 | 265 | try: 266 | field = self.model.collection_class.get_field(field_name) 267 | except KeyError as er: 268 | logger.debug( 269 | f"Searching `{field_name}` with parameters `{used_parameters}` produced error: {er}" 270 | ) 271 | return search_filters_dict 272 | 273 | for key, value in used_parameters.items(): 274 | if value is None or value == "": 275 | continue 276 | 277 | try: 278 | _, lookup = key.rsplit("__", maxsplit=1) 279 | except ValueError: 280 | lookup = "" 281 | 282 | lookup = lookup or "exact" 283 | if lookup == "isnull": 284 | # Null search is not supported in typesense 285 | continue 286 | 287 | if isinstance(field, tuple(TYPESENSE_DATETIME_FIELDS)): 288 | datetime_object = parse_datetime(value) 289 | value = get_unix_timestamp(datetime_object) 290 | 291 | if str(value).isdigit(): 292 | if lookup in ["gte", "gt"]: 293 | min_val = value 294 | if lookup in ["lte", "lt"]: 295 | max_val = value 296 | 297 | if max_val and min_val: 298 | search_filters_dict[field_name] = f"[{min_val}..{max_val}]" 299 | value = None 300 | elif max_val or min_val: 301 | search_filters_dict[ 302 | field_name 303 | ] = f"{lookup_to_operator[lookup]}{min_val or max_val}" 304 | value = None 305 | 306 | if value is not None and lookup is not None: 307 | if field.field_type == "string": 308 | search_filters_dict[ 309 | field_name 310 | ] = f":{lookup_to_operator[lookup]}{value}" 311 | elif field.field_type == "bool": 312 | if isinstance(value, str): 313 | value = value.lower() 314 | 315 | boolean_map = { 316 | 0: "false", 317 | 1: "true", 318 | "0": "false", 319 | "1": "true", 320 | "false": "false", 321 | "true": "true", 322 | "no": "false", 323 | "yes": "true", 324 | "n": "false", 325 | "y": "true", 326 | False: "false", 327 | True: "true", 328 | } 329 | search_filters_dict[field_name] = boolean_map[value] 330 | else: 331 | search_filters_dict[field_name] = f"{lookup_to_operator[lookup]}{value}" 332 | 333 | return search_filters_dict 334 | 335 | def get_typesense_results(self, request): 336 | """ 337 | This should do what Changelist.get_queryset does 338 | 339 | Args: 340 | request: 341 | 342 | Returns: 343 | Typesense Search Results in dictionary 344 | """ 345 | 346 | # First, we collect all the declared list filters. 347 | ( 348 | self.filter_specs, 349 | self.has_filters, 350 | remaining_lookup_params, 351 | filters_may_have_duplicates, 352 | self.has_active_filters, 353 | ) = self.get_filters(request) 354 | 355 | # we let every list filter modify the objs to its liking. 356 | filters_dict = {} 357 | 358 | for filter_spec in self.filter_specs: 359 | if hasattr(filter_spec, "filter_by"): 360 | # all custom filters with filter_by defined 361 | filters_dict.update(filter_spec.filter_by) 362 | continue 363 | 364 | if hasattr(filter_spec, "field"): 365 | used_parameters = getattr(filter_spec, "used_parameters") 366 | search_filters = self.get_search_filters( 367 | filter_spec.field.attname, used_parameters 368 | ) 369 | filters_dict.update(search_filters) 370 | else: 371 | # custom filters where filter_by is not defined 372 | used_parameters = getattr(filter_spec, "used_parameters") 373 | remaining_lookup_params.update(used_parameters) 374 | 375 | for k, v in remaining_lookup_params.items(): 376 | try: 377 | field_name, _ = k.split("__", maxsplit=1) 378 | except ValueError: 379 | field_name = k 380 | k = f"{k}__exact" 381 | 382 | search_filters = self.get_search_filters(field_name, {k: v}) 383 | filters_dict.update(search_filters) 384 | 385 | filter_by = " && ".join( 386 | [f"{key}:{value}" for key, value in filters_dict.items()] 387 | ) 388 | 389 | # Set ordering. 390 | ordering = self.get_typesense_ordering(request) 391 | sort_by = self.get_sort_by(ordering) 392 | 393 | # Apply django_typesense search results 394 | query = self.query or "*" 395 | results = self.model_admin.get_typesense_search_results( 396 | request, 397 | query, 398 | self.page_num, 399 | filter_by=filter_by, 400 | sort_by=sort_by, 401 | list_per_page=self.list_max_show_all # so that if we have all the data if we need to show all 402 | ) 403 | 404 | # Set query string for clearing all filters. 405 | self.clear_all_filters_qs = self.get_query_string( 406 | new_params=remaining_lookup_params, 407 | remove=self.get_filters_params(), 408 | ) 409 | 410 | return results 411 | 412 | def get_queryset(self, request): 413 | # this is needed for admin actions that call cl.get_queryset 414 | # exporting is the currently possible way of getting records from typesense without pagination 415 | # Typesense team will work on a flag to disable pagination, until then, we need a way to get this to work. 416 | # Problem happens when django finds fields only present on typesense in its filter i.e IncorrectLookupParameters 417 | # First, we collect all the declared list filters. 418 | ( 419 | self.filter_specs, 420 | self.has_filters, 421 | remaining_lookup_params, 422 | filters_may_have_duplicates, 423 | self.has_active_filters, 424 | ) = self.get_filters(request) 425 | # Then, we let every list filter modify the queryset to its liking. 426 | qs = self.root_queryset 427 | 428 | for filter_spec in self.filter_specs: 429 | new_qs = filter_spec.queryset(request, qs) 430 | if new_qs is not None: 431 | qs = new_qs 432 | 433 | for param, value in remaining_lookup_params.items(): 434 | try: 435 | # Finally, we apply the remaining lookup parameters from the query 436 | # string (i.e. those that haven't already been processed by the 437 | # filters). 438 | qs = qs.filter(**{param: value}) 439 | except (SuspiciousOperation, ImproperlyConfigured): 440 | # Allow certain types of errors to be re-raised as-is so that the 441 | # caller can treat them in a special way. 442 | raise 443 | except Exception as e: 444 | # Every other error is caught with a naked except, because we don't 445 | # have any other way of validating lookup parameters. They might be 446 | # invalid if the keyword arguments are incorrect, or if the values 447 | # are not in the correct type, so we might get FieldError, 448 | # ValueError, ValidationError, or ?. 449 | 450 | # for django-typesense, possibly means k only available in typesense 451 | new_lookup_params = self.model.collection_class.get_django_lookup(param, value, e) 452 | qs = qs.filter(**new_lookup_params) 453 | 454 | # Apply search results 455 | qs, search_may_have_duplicates = self.model_admin.get_search_results( 456 | request, 457 | qs, 458 | self.query, 459 | ) 460 | 461 | # Set query string for clearing all filters. 462 | self.clear_all_filters_qs = self.get_query_string( 463 | new_params=remaining_lookup_params, 464 | remove=self.get_filters_params(), 465 | ) 466 | # Remove duplicates from results, if necessary 467 | if filters_may_have_duplicates | search_may_have_duplicates: 468 | qs = qs.filter(pk=OuterRef("pk")) 469 | qs = self.root_queryset.filter(Exists(qs)) 470 | 471 | # Set ordering. 472 | ordering = self.get_ordering(request, qs) 473 | qs = qs.order_by(*ordering) 474 | 475 | if not qs.query.select_related: 476 | qs = self.apply_select_related(qs) 477 | 478 | return qs 479 | -------------------------------------------------------------------------------- /django_typesense/collections.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | import logging 4 | from operator import methodcaller 5 | from typing import Dict, Iterable, List, Union 6 | 7 | from django.db.models import QuerySet 8 | from django.utils.functional import cached_property 9 | 10 | try: 11 | from django.utils.functional import classproperty 12 | except ImportError: 13 | from django.utils.decorators import classproperty 14 | 15 | from typesense.exceptions import ObjectAlreadyExists, ObjectNotFound 16 | 17 | from django_typesense.fields import TypesenseCharField, TypesenseField 18 | from django_typesense.typesense_client import client 19 | 20 | logger = logging.getLogger(__name__) 21 | 22 | _COLLECTION_META_OPTIONS = { 23 | "schema_name", 24 | "default_sorting_field", 25 | "token_separators", 26 | "symbols_to_index", 27 | "query_by_fields", 28 | } 29 | _SYNONYM_PARAMETERS = {"synonyms", "root", "locale", "symbols_to_index"} 30 | 31 | 32 | class Synonym: 33 | name: str = "" 34 | synonyms: list[str] = None 35 | root: str = "" 36 | locale: str = "" 37 | symbols_to_index: list[str] = None 38 | 39 | @classproperty 40 | def data(cls): 41 | if not cls.name: 42 | raise ValueError("the name attribute must be set") 43 | 44 | if not cls.synonyms: 45 | raise ValueError("the synonyms attribute must be set") 46 | 47 | if cls.symbols_to_index is None: 48 | cls.symbols_to_index = [] 49 | 50 | return { 51 | cls.name: { 52 | param: getattr(cls, param) 53 | for param in _SYNONYM_PARAMETERS 54 | if getattr(cls, param) 55 | } 56 | } 57 | 58 | 59 | class TypesenseCollectionMeta(type): 60 | def __new__(cls, name, bases, namespace): 61 | namespace["schema_name"] = namespace.get("schema_name") or name.lower() 62 | return super().__new__(cls, name, bases, namespace) 63 | 64 | 65 | class TypesenseCollection(metaclass=TypesenseCollectionMeta): 66 | query_by_fields: str = "" 67 | schema_name: str = "" 68 | default_sorting_field: str = "" 69 | token_separators: list = [] 70 | symbols_to_index: list = [] 71 | synonyms: List[Synonym] = [] 72 | 73 | def __init__( 74 | self, 75 | obj: Union[object, QuerySet, Iterable] = None, 76 | many: bool = False, 77 | data: list = None, 78 | update_fields: list = None, 79 | ): 80 | assert ( 81 | self.query_by_fields 82 | ), "`query_by_fields` must be specified in the collection definition" 83 | assert not all([obj, data]), "`obj` and `data` cannot be provided together" 84 | 85 | self.update_fields = update_fields 86 | self._meta = self._get_metadata() 87 | self.fields = self.get_fields() 88 | self._synonyms = [synonym().data for synonym in self.synonyms] 89 | 90 | if data and obj: 91 | raise Exception("'data' and 'obj' are mutually exclusive") 92 | 93 | self._data = data 94 | self.many = many 95 | self.obj = obj 96 | 97 | @cached_property 98 | def data(self): 99 | return self.get_data() 100 | 101 | def get_data(self): 102 | if self._data: 103 | return self._data 104 | 105 | if not self.obj: 106 | return [] 107 | 108 | data = [] 109 | if self.many: 110 | for _obj in self.obj: 111 | if obj_data := self._get_object_data(_obj): 112 | data.append(obj_data) 113 | else: 114 | if obj_data := self._get_object_data(self.obj): 115 | data.append(obj_data) 116 | 117 | return data 118 | 119 | @classmethod 120 | def get_fields(cls) -> Dict[str, TypesenseField]: 121 | """ 122 | Returns: 123 | A dictionary of the fields names to the field definition for this collection 124 | """ 125 | fields = {} 126 | # Avoid Recursion Errors 127 | exclude_attributes = {"sortable_fields"} 128 | 129 | for attr in dir(cls): 130 | if attr in exclude_attributes: 131 | continue 132 | attr_value = getattr(cls, attr, None) 133 | if not isinstance(attr_value, TypesenseField): 134 | continue 135 | 136 | attr_value._name = attr 137 | attr_value._value = attr_value._value or attr 138 | fields[attr] = attr_value 139 | 140 | # Auto adds id if absent 141 | if not fields.get("id"): 142 | _id = TypesenseCharField(sort=True, value="pk") 143 | _id._name = "id" 144 | fields["id"] = _id 145 | 146 | return fields 147 | 148 | @classmethod 149 | def _get_metadata(cls) -> dict: 150 | defined_meta_options = _COLLECTION_META_OPTIONS.intersection(set(dir(cls))) 151 | return { 152 | meta_option: getattr(cls, meta_option) 153 | for meta_option in defined_meta_options 154 | } 155 | 156 | @cached_property 157 | def validated_data(self) -> list: 158 | """ 159 | Returns a list of the collection data with values converted into the correct Python objects 160 | """ 161 | 162 | _validated_data = [] 163 | 164 | for obj in self.data: 165 | data = {} 166 | for key, value in obj.items(): 167 | field = self.get_field(key) 168 | data[key] = field.to_python(value) 169 | 170 | _validated_data.append(data) 171 | 172 | return _validated_data 173 | 174 | def __str__(self): 175 | return f"{self.schema_name} TypesenseCollection" 176 | 177 | @classproperty 178 | def sortable_fields(cls) -> list: 179 | """ 180 | Returns: 181 | The names of sortable fields 182 | """ 183 | fields = cls.get_fields() 184 | return [field.name for field in fields.values() if field.sort] 185 | 186 | @classmethod 187 | def get_field(cls, name) -> TypesenseField: 188 | """ 189 | Get the field with the provided name from the collection 190 | 191 | Args: 192 | name: the field name 193 | 194 | Returns: 195 | A TypesenseField 196 | """ 197 | fields = cls.get_fields() 198 | return fields[name] 199 | 200 | @classmethod 201 | def get_django_lookup(cls, field, value, exception: Exception) -> dict: 202 | """ 203 | Get the lookup that would have been used for this field in django. Expects to find a method on 204 | the collection called `get_FIELD_lookup` otherwise a NotImplementedError is raised 205 | 206 | Args: 207 | field: the name of the field in the collection 208 | value: the value to look for 209 | exception: the django exception that led us here 210 | 211 | Returns: 212 | A dictionary of the fields to the value. 213 | """ 214 | 215 | if "get_%s_lookup" % field not in cls.__dict__: 216 | raise Exception([ 217 | exception, 218 | NotImplementedError("get_%s_lookup is not implemented" % field) 219 | ]) 220 | 221 | method = methodcaller("get_%s_lookup" % field, value) 222 | return method(cls) 223 | 224 | @cached_property 225 | def schema_fields(self) -> list: 226 | """ 227 | Returns: 228 | A list of dictionaries with field attributes needed by typesense for schema creation 229 | """ 230 | return [field.attrs for field in self.fields.values()] 231 | 232 | def _get_object_data(self, obj): 233 | if self.update_fields: 234 | # we need the id for updates and a user can leave it out 235 | update_fields = set(self.fields.keys()).intersection( 236 | set(self.update_fields) 237 | ) 238 | if update_fields: 239 | update_fields.add("id") 240 | fields = [self.get_field(field_name) for field_name in update_fields] 241 | else: 242 | fields = [] 243 | else: 244 | fields = self.fields.values() 245 | 246 | return {field.name: field.value(obj) for field in fields} 247 | 248 | @property 249 | def schema(self) -> dict: 250 | """ 251 | Returns: 252 | The typesense schema 253 | """ 254 | return { 255 | "name": self.schema_name, 256 | "fields": self.schema_fields, 257 | "default_sorting_field": self._meta["default_sorting_field"], 258 | "symbols_to_index": self._meta["symbols_to_index"], 259 | "token_separators": self._meta["token_separators"], 260 | } 261 | 262 | def create_typesense_collection(self): 263 | """ 264 | Create a new typesense collection (schema) on the typesense server 265 | """ 266 | try: 267 | client.collections.create(self.schema) 268 | except ObjectAlreadyExists: 269 | pass 270 | 271 | def update_typesense_collection(self): 272 | """ 273 | Update the schema of an existing collection 274 | """ 275 | try: 276 | current_schema = self.retrieve_typesense_collection() 277 | except ObjectNotFound: 278 | self.create_typesense_collection() 279 | current_schema = self.retrieve_typesense_collection() 280 | 281 | self.create_or_update_synonyms() 282 | 283 | schema_changes = {} 284 | field_changes = [] 285 | 286 | # Update fields 287 | existing_fields = {field["name"]: field for field in current_schema["fields"]} 288 | schema_fields = {field["name"]: field for field in self.schema_fields} 289 | # The collection retrieved from typesense does not include the id field so we remove the one we added 290 | schema_fields.pop("id") 291 | 292 | dropped_fields_names = set(existing_fields.keys()).difference( 293 | schema_fields.keys() 294 | ) 295 | field_changes.extend( 296 | [{"name": field_name, "drop": True} for field_name in dropped_fields_names] 297 | ) 298 | 299 | for field in schema_fields.values(): 300 | if field["name"] not in existing_fields.keys(): 301 | field_changes.append(field) 302 | else: 303 | if field != existing_fields[field["name"]]: 304 | field_changes.append({"name": field["name"], "drop": True}) 305 | field_changes.append(field) 306 | 307 | if field_changes: 308 | schema_changes["fields"] = field_changes 309 | 310 | if not schema_changes: 311 | return 312 | 313 | logger.debug(f"Updating schema changes in {self.schema_name}") 314 | return client.collections[self.schema_name].update(schema_changes) 315 | 316 | def drop_typesense_collection(self): 317 | """ 318 | Drops a typesense collection from the typesense server 319 | """ 320 | client.collections[self.schema_name].delete() 321 | 322 | def retrieve_typesense_collection(self): 323 | """ 324 | Retrieve the details of a collection 325 | """ 326 | return client.collections[self.schema_name].retrieve() 327 | 328 | def delete(self): 329 | if not self.data: 330 | return 331 | 332 | delete_params = { 333 | "filter_by": f"id: {[obj['id'] for obj in self.data]}".replace("'", "") 334 | } 335 | 336 | try: 337 | return client.collections[self.schema_name].documents.delete(delete_params) 338 | except ObjectNotFound: 339 | pass 340 | 341 | def update(self, action_mode: str = "emplace"): 342 | if not self.data: 343 | return 344 | 345 | if len(self.data) == 1: 346 | return self._update_single_document(self.data[0]) 347 | else: 348 | return self._update_multiple_documents(action_mode) 349 | 350 | def _update_single_document(self, document): 351 | document_id = document.pop("id") 352 | 353 | try: 354 | return ( 355 | client.collections[self.schema_name] 356 | .documents[document_id] 357 | .update(document) 358 | ) 359 | except ObjectNotFound: 360 | self.update_fields = [] 361 | # we don't want the cached data 362 | return client.collections[self.schema_name].documents.upsert( 363 | self.get_data()[0] 364 | ) 365 | 366 | def _update_multiple_documents(self, action_mode): 367 | try: 368 | return client.collections[self.schema_name].documents.import_( 369 | self.data, {"action": action_mode} 370 | ) 371 | except ObjectNotFound: 372 | # we don't want the cached data 373 | return client.collections[self.schema_name].documents.import_( 374 | self.get_data(), {"action": action_mode} 375 | ) 376 | 377 | def create_or_update_synonyms(self): 378 | current_synonyms = {} 379 | for synonym in self.get_synonyms().get("synonyms", []): 380 | name = synonym.pop("id") 381 | current_synonyms[name] = synonym 382 | 383 | defined_synonyms = {} 384 | for synonym_data in self._synonyms: 385 | defined_synonyms.update(synonym_data) 386 | 387 | missing_synonyms_names = set(current_synonyms.keys()).difference( 388 | defined_synonyms.keys() 389 | ) 390 | has_changes = False 391 | 392 | for synonym_name in missing_synonyms_names: 393 | has_changes = True 394 | self.delete_synonym(synonym_name) 395 | 396 | for synonym_name, synonym_data in defined_synonyms.items(): 397 | if synonym_name not in current_synonyms: 398 | has_changes = True 399 | client.collections[self.schema_name].synonyms.upsert( 400 | synonym_name, synonym_data 401 | ) 402 | elif synonym_data['synonyms'] != current_synonyms[synonym_name]['synonyms']: 403 | has_changes = True 404 | client.collections[self.schema_name].synonyms.upsert( 405 | synonym_name, synonym_data 406 | ) 407 | 408 | if has_changes: 409 | logger.debug(f"Synonyms updated in {self.schema_name}") 410 | 411 | def get_synonyms(self) -> dict: 412 | """List all synonyms associated with this collection""" 413 | return client.collections[self.schema_name].synonyms.retrieve() 414 | 415 | def get_synonym(self, synonym_name) -> dict: 416 | """Retrieve a single synonym by name""" 417 | return client.collections[self.schema_name].synonyms[synonym_name].retrieve() 418 | 419 | def delete_synonym(self, synonym_name): 420 | """Delete the synonym with the given name associated with this collection""" 421 | return client.collections[self.schema_name].synonyms[synonym_name].delete() 422 | -------------------------------------------------------------------------------- /django_typesense/exceptions.py: -------------------------------------------------------------------------------- 1 | class BatchUpdateError(Exception): 2 | pass 3 | 4 | 5 | class UnorderedQuerySetError(Exception): 6 | pass 7 | -------------------------------------------------------------------------------- /django_typesense/fields.py: -------------------------------------------------------------------------------- 1 | import json 2 | from decimal import Decimal 3 | from datetime import datetime, date, time 4 | from typing import Optional 5 | from operator import attrgetter 6 | 7 | from django_typesense.utils import get_unix_timestamp 8 | 9 | TYPESENSE_SCHEMA_ATTRS = [ 10 | "name", 11 | "_field_type", 12 | "sort", 13 | "index", 14 | "optional", 15 | "facet", 16 | "infix", 17 | "locale", 18 | "stem", 19 | ] 20 | 21 | 22 | class TypesenseField: 23 | _attrs = None 24 | _field_type = None 25 | _sort = False 26 | 27 | def __init__( 28 | self, 29 | value: Optional[str] = None, 30 | sort: bool = None, 31 | index: bool = True, 32 | optional: bool = False, 33 | facet: bool = False, 34 | infix: bool = False, 35 | locale: str = "", 36 | stem: bool = False, 37 | ): 38 | self._value = value 39 | self._name = None 40 | self.sort = self._sort if sort is None else sort 41 | self.index = index 42 | self.optional = optional 43 | self.facet = facet 44 | self.infix = infix 45 | self.locale = locale 46 | self.stem = stem 47 | 48 | def __str__(self): 49 | return f"{self.name}" 50 | 51 | @property 52 | def field_type(self): 53 | return self._field_type 54 | 55 | @property 56 | def name(self): 57 | return self._name 58 | 59 | @property 60 | def attrs(self): 61 | _attrs = {k: getattr(self, k) for k in TYPESENSE_SCHEMA_ATTRS} 62 | _attrs["type"] = _attrs.pop("_field_type") 63 | return _attrs 64 | 65 | def value(self, obj): 66 | try: 67 | __value = attrgetter(self._value)(obj) 68 | except AttributeError as er: 69 | if self.optional: 70 | __value = None 71 | else: 72 | raise er 73 | 74 | if callable(__value): 75 | return __value() 76 | 77 | return __value 78 | 79 | def to_python(self, value): 80 | return value 81 | 82 | 83 | class TypesenseCharField(TypesenseField): 84 | _field_type = "string" 85 | 86 | def value(self, obj): 87 | __value = super().value(obj) 88 | if isinstance(__value, str): 89 | return __value 90 | if __value is None: 91 | return "" 92 | return str(__value) 93 | 94 | 95 | class TypesenseIntegerMixin(TypesenseField): 96 | def value(self, obj): 97 | _value = super().value(obj) 98 | 99 | if _value is None: 100 | return None 101 | try: 102 | return int(_value) 103 | except (TypeError, ValueError) as e: 104 | raise e.__class__( 105 | f"Field '{self.name}' expected a number but got {_value}.", 106 | ) from e 107 | 108 | 109 | class TypesenseSmallIntegerField(TypesenseIntegerMixin): 110 | _field_type = "int32" 111 | _sort = True 112 | 113 | 114 | class TypesenseBigIntegerField(TypesenseIntegerMixin): 115 | _field_type = "int64" 116 | _sort = True 117 | 118 | 119 | class TypesenseFloatField(TypesenseField): 120 | _field_type = "float" 121 | _sort = True 122 | 123 | 124 | class TypesenseDecimalField(TypesenseField): 125 | """ 126 | String type is preferred over float 127 | """ 128 | 129 | _field_type = "string" 130 | _sort = True 131 | 132 | def value(self, obj): 133 | __value = super().value(obj) 134 | return str(__value) 135 | 136 | def to_python(self, value): 137 | return Decimal(value) 138 | 139 | 140 | class TypesenseBooleanField(TypesenseField): 141 | _field_type = "bool" 142 | _sort = True 143 | 144 | 145 | class TypesenseDateTimeFieldBase(TypesenseField): 146 | _field_type = "int64" 147 | _sort = True 148 | 149 | def value(self, obj): 150 | _value = super().value(obj) 151 | 152 | if _value is None: 153 | return None 154 | 155 | if isinstance(_value, int): 156 | return _value 157 | 158 | _value = get_unix_timestamp(_value) 159 | 160 | return _value 161 | 162 | 163 | class TypesenseDateField(TypesenseDateTimeFieldBase): 164 | def to_python(self, value): 165 | return date.fromtimestamp(value) 166 | 167 | 168 | class TypesenseDateTimeField(TypesenseDateTimeFieldBase): 169 | def to_python(self, value): 170 | return datetime.fromtimestamp(value) 171 | 172 | 173 | class TypesenseTimeField(TypesenseDateTimeFieldBase): 174 | def to_python(self, value): 175 | return datetime.fromtimestamp(value).time() 176 | 177 | 178 | class TypesenseJSONField(TypesenseField): 179 | """ 180 | `string` is preferred over `object` 181 | """ 182 | 183 | _field_type = "string" 184 | 185 | def value(self, obj): 186 | __value = super().value(obj) 187 | return json.dumps(__value, default=str) 188 | 189 | def to_python(self, value): 190 | return json.loads(value) 191 | 192 | 193 | class TypesenseArrayField(TypesenseField): 194 | def __init__(self, base_field: TypesenseField, *args, **kwargs): 195 | super().__init__(*args, **kwargs) 196 | self.base_field = base_field 197 | self._field_type = f"{self.base_field._field_type}[]" 198 | 199 | def to_python(self, value): 200 | return list(map(self.base_field.to_python, value)) 201 | 202 | 203 | TYPESENSE_DATETIME_FIELDS = [ 204 | TypesenseDateTimeField, 205 | TypesenseDateField, 206 | TypesenseTimeField, 207 | ] 208 | -------------------------------------------------------------------------------- /django_typesense/management/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Siege-Software/django-typesense/8bb24138e6c096bcaf0985ad650dbfd3ec40c00e/django_typesense/management/__init__.py -------------------------------------------------------------------------------- /django_typesense/management/commands/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Siege-Software/django-typesense/8bb24138e6c096bcaf0985ad650dbfd3ec40c00e/django_typesense/management/commands/__init__.py -------------------------------------------------------------------------------- /django_typesense/management/commands/updatecollections.py: -------------------------------------------------------------------------------- 1 | import sys 2 | 3 | from django.core.management import BaseCommand 4 | from django.apps import apps 5 | 6 | 7 | class Command(BaseCommand): 8 | help = "Create and/or Update Typesense Collections" 9 | 10 | def add_arguments(self, parser): 11 | parser.add_argument( 12 | "args", 13 | metavar="collection_name", 14 | nargs="*", 15 | help="Specify the collection schema name(s) to create or update.", 16 | ) 17 | 18 | def handle(self, *collection_names, **options): 19 | collections = {} 20 | for model_data in apps.all_models.values(): 21 | for model in model_data.values(): 22 | if hasattr(model, 'collection_class'): 23 | collections[model.collection_class.schema_name] = model.collection_class 24 | 25 | collections_for_action = [] 26 | # Make sure the collection name(s) they asked for exists 27 | if collection_names := set(collection_names): 28 | has_bad_names = False 29 | 30 | for collection_name in collection_names: 31 | try: 32 | collection = collections[collection_name] 33 | except KeyError: 34 | self.stderr.write(f"No collection exists with schema name '{collection_name}'") 35 | has_bad_names = True 36 | else: 37 | collections_for_action.append(collection) 38 | 39 | if has_bad_names: 40 | sys.exit(2) 41 | else: 42 | collections_for_action = collections.values() 43 | 44 | for collection in collections_for_action: 45 | collection().update_typesense_collection() 46 | -------------------------------------------------------------------------------- /django_typesense/migrations/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Siege-Software/django-typesense/8bb24138e6c096bcaf0985ad650dbfd3ec40c00e/django_typesense/migrations/__init__.py -------------------------------------------------------------------------------- /django_typesense/mixins.py: -------------------------------------------------------------------------------- 1 | from django.db import models 2 | 3 | 4 | class TypesenseQuerySet(models.QuerySet): 5 | def delete(self): 6 | assert issubclass(self.model, TypesenseModelMixin), ( 7 | f"Model `{self.model}` must inherit `TypesenseMixin` to use the TypesenseQueryset Manager" 8 | ) 9 | collection = self.model.get_collection(self, many=True) 10 | collection.delete() 11 | return super().delete() 12 | 13 | def update(self, **kwargs): 14 | assert issubclass(self.model, TypesenseModelMixin), ( 15 | f"Model `{self.model}` must inherit `TypesenseMixin` to use the TypesenseQueryset Manager" 16 | ) 17 | obj_ids = list(self.values_list('id', flat=True)) 18 | update_result = super().update(**kwargs) 19 | queryset = self.model.objects.filter(id__in=obj_ids) 20 | collection = self.model.get_collection(queryset, many=True, update_fields=kwargs.keys()) 21 | collection.update() 22 | return update_result 23 | 24 | 25 | class TypesenseManager(models.Manager): 26 | def get_queryset(self): 27 | return TypesenseQuerySet(self.model, using=self._db) 28 | 29 | 30 | class TypesenseModelMixin(models.Model): 31 | collection_class = None 32 | objects = TypesenseQuerySet.as_manager() 33 | 34 | class Meta: 35 | abstract = True 36 | 37 | @classmethod 38 | def get_collection_class(cls): 39 | """ 40 | Return the class to use for the typesense collection. 41 | Defaults to using `self.collection_class`. 42 | """ 43 | assert cls.collection_class is not None, ( 44 | "'%s' should either include a `collection_class` attribute, " 45 | "or override the `get_collection_class()` method." 46 | % cls.__name__ 47 | ) 48 | 49 | return cls.collection_class 50 | 51 | @classmethod 52 | def get_collection(cls, *args, **kwargs): 53 | """ 54 | Return the collection obj. 55 | """ 56 | collection_class = cls.get_collection_class() 57 | return collection_class(*args, **kwargs) 58 | 59 | -------------------------------------------------------------------------------- /django_typesense/paginator.py: -------------------------------------------------------------------------------- 1 | import copy 2 | 3 | from django.core.paginator import Paginator 4 | from django.utils.functional import cached_property 5 | 6 | 7 | class TypesenseSearchPaginator(Paginator): 8 | def __init__( 9 | self, object_list, per_page, orphans=0, allow_empty_first_page=True, model=None 10 | ): 11 | super().__init__(object_list, per_page, orphans, allow_empty_first_page) 12 | self.model = model 13 | self.collection_class = self.model.get_collection_class() 14 | self.results = self.prepare_results() 15 | 16 | def prepare_results(self): 17 | """ 18 | Do whatever is required to present the values correctly in the admin. 19 | """ 20 | documents = (hit['document'] for hit in self.object_list["hits"]) 21 | collection = self.model.get_collection(data=documents) 22 | model_field_names = set((local_field.name for local_field in self.model._meta.local_fields)) 23 | results = [] 24 | 25 | for _data in collection.validated_data: 26 | data = copy.deepcopy(_data) 27 | properties = {} 28 | 29 | for field_name in collection.fields.keys(): 30 | if field_name not in model_field_names: 31 | try: 32 | field_val = data.pop(field_name) 33 | except KeyError: 34 | pass 35 | else: 36 | properties[field_name] = field_val 37 | 38 | result_instance = self.model(**data) 39 | for key, value in properties.items(): 40 | try: 41 | setattr(result_instance, key, value) 42 | except AttributeError: 43 | # non-data descriptors 44 | result_instance.__dict__[key] = value 45 | 46 | results.append(result_instance) 47 | 48 | return results 49 | 50 | def page(self, number): 51 | """Return a Page object for the given 1-based page number.""" 52 | number = self.validate_number(number) 53 | bottom = (number - 1) * self.per_page 54 | top = bottom + self.per_page 55 | if top + self.orphans >= self.count: 56 | top = self.count 57 | return self._get_page(self.results[bottom:top], number, self) 58 | 59 | @cached_property 60 | def count(self): 61 | """Return the total number of objects, across all pages.""" 62 | return self.object_list["found"] 63 | -------------------------------------------------------------------------------- /django_typesense/signals.py: -------------------------------------------------------------------------------- 1 | from django.db import transaction 2 | from django.db.models.signals import m2m_changed, post_save, pre_delete 3 | from django.dispatch import receiver 4 | 5 | from django_typesense.mixins import TypesenseModelMixin 6 | 7 | 8 | @receiver(post_save) 9 | def post_save_typesense_models(sender, instance, **kwargs): 10 | if not issubclass(sender, TypesenseModelMixin): 11 | return 12 | 13 | transaction.on_commit( 14 | sender.get_collection(instance, update_fields=kwargs.get('update_fields', [])).update 15 | ) 16 | 17 | 18 | @receiver(pre_delete) 19 | def pre_delete_typesense_models(sender, instance, **kwargs): 20 | if not issubclass(sender, TypesenseModelMixin): 21 | return 22 | 23 | sender.get_collection(instance).delete() 24 | 25 | 26 | @receiver(m2m_changed) 27 | def m2m_changed_typesense_models(instance, model, action, **kwargs): 28 | if action in ["post_add", "post_remove", "post_clear"]: 29 | if isinstance(instance, TypesenseModelMixin): 30 | instance_class = instance.__class__ 31 | instance_class.get_collection(instance).update() 32 | 33 | if issubclass(model, TypesenseModelMixin): 34 | pk_set = list(kwargs.get("pk_set")) 35 | obj = model.objects.filter(pk__in=pk_set) 36 | model.get_collection(obj=obj, many=True).update() 37 | -------------------------------------------------------------------------------- /django_typesense/static/admin/js/search-typesense.js: -------------------------------------------------------------------------------- 1 | (function($) { 2 | $(document).ready(function () { 3 | let ajax_call = function (endpoint, request_parameters) { 4 | $.getJSON(endpoint, request_parameters) 5 | .done(response => { 6 | changelist_form.fadeTo('fast', 0.5).promise().then(() => { 7 | let newChangelistHTML = '

Click Search to complete

' + $(response['html']).find('#changelist-form').html() 8 | changelist_form.html(newChangelistHTML); 9 | changelist_form.fadeTo('fast', 1); 10 | }) 11 | }) 12 | .fail(function( jqxhr, textStatus, error ) { 13 | const err = textStatus + ", " + error; 14 | console.log( "Request Failed: " + err ); 15 | changelist_form.html('Sorry. An error occurred on our end.'); 16 | }); 17 | }; 18 | 19 | const searchbar = $("#searchbar"); 20 | const changelist_form = $('#changelist-form'); 21 | const delay_by_in_ms = 0; 22 | const endpoint = window.location.origin + window.location.pathname 23 | let scheduled_function = false; 24 | 25 | searchbar.on('input', function () { 26 | const search_value = $(this).val().trim(); 27 | const search_parameters = {q:search_value}; 28 | const urlParams = new URLSearchParams(window.location.search); 29 | const request_parameters = Object.assign(search_parameters, urlParams); 30 | 31 | changelist_form.html('

Searching...

'); 32 | if (scheduled_function) { 33 | clearTimeout(scheduled_function); 34 | } 35 | scheduled_function = setTimeout(ajax_call, delay_by_in_ms, endpoint, request_parameters); 36 | }) 37 | }); 38 | })(django.jQuery); 39 | -------------------------------------------------------------------------------- /django_typesense/typesense_client.py: -------------------------------------------------------------------------------- 1 | import typesense 2 | 3 | 4 | from django.conf import settings 5 | 6 | client = typesense.Client(settings.TYPESENSE) 7 | -------------------------------------------------------------------------------- /django_typesense/utils.py: -------------------------------------------------------------------------------- 1 | import concurrent.futures 2 | import json 3 | import logging 4 | import os 5 | from datetime import date, datetime, time 6 | from typing import List 7 | 8 | from django.core.exceptions import FieldError 9 | from django.core.paginator import Paginator 10 | from django.db.models import QuerySet 11 | from typesense.exceptions import TypesenseClientError 12 | 13 | from django_typesense.exceptions import BatchUpdateError, UnorderedQuerySetError 14 | 15 | logger = logging.getLogger(__name__) 16 | 17 | 18 | def update_batch(documents_queryset: QuerySet, collection_class, batch_no: int) -> None: 19 | """Updates a batch of documents using the Typesense API. 20 | 21 | Parameters 22 | ---------- 23 | documents_queryset : QuerySet 24 | The Django objects QuerySet to update. It must be a `TypesenseModelMixin` subclass. 25 | collection_class : TypesenseCollection 26 | The Django Typesense collection to update. 27 | batch_no : int 28 | The batch identifier number. 29 | 30 | Returns 31 | ------- 32 | None 33 | 34 | Raises 35 | ------ 36 | BatchUpdateError 37 | Raised when an error occurs during updating typesense collection. 38 | """ 39 | collection = collection_class(documents_queryset, many=True) 40 | responses = collection.update() 41 | if responses is None: 42 | return 43 | 44 | if isinstance(responses, list): 45 | failure_responses = [ 46 | response for response in responses if not response["success"] 47 | ] 48 | 49 | if failure_responses: 50 | raise BatchUpdateError( 51 | f"An Error occurred during the bulk update: {failure_responses}" 52 | ) 53 | 54 | logger.debug(f"Batch {batch_no} Updated with {len(collection.data)} records ✓") 55 | 56 | 57 | def bulk_update_typesense_records( 58 | records_queryset: QuerySet, 59 | batch_size: int = 1024, 60 | num_threads: int = os.cpu_count(), 61 | ) -> None: 62 | """This method updates Typesense records for both objects .update() calls from 63 | Typesense mixin subclasses. 64 | This function should be called on every model update statement for data consistency. 65 | 66 | Parameters 67 | ---------- 68 | records_queryset : QuerySet 69 | The Django objects QuerySet to update. It must be a `TypesenseModelMixin` subclass. 70 | batch_size : int 71 | The number of objects to be indexed in a single run. Defaults to 1024. 72 | num_threads : int 73 | The number of threads that will be used. Defaults to `os.cpu_count()` 74 | 75 | Returns 76 | ------- 77 | None 78 | 79 | Raises 80 | ------ 81 | UnorderedQuerySetError 82 | Raised when unordered queryset is ordered by `primary_key` and 83 | throws a `FieldError` or `TypeError`. 84 | """ 85 | 86 | from django_typesense.mixins import TypesenseQuerySet 87 | 88 | if not isinstance(records_queryset, TypesenseQuerySet): 89 | logger.error( 90 | f"The objects for {records_queryset.model.__name__} does not use TypesenseQuerySet " 91 | f"as it's manager. Please update the model manager for the class to use Typesense." 92 | ) 93 | return 94 | 95 | if not records_queryset.ordered: 96 | try: 97 | records_queryset = records_queryset.order_by("pk") 98 | except (FieldError, TypeError): 99 | raise UnorderedQuerySetError( 100 | "Pagination may yield inconsistent results with an unordered object_list. " 101 | "Please provide an ordered objects." 102 | ) 103 | 104 | collection_class = records_queryset.model.collection_class 105 | paginator = Paginator(records_queryset, batch_size) 106 | 107 | with concurrent.futures.ThreadPoolExecutor(max_workers=num_threads) as executor: 108 | futures = [] 109 | for page_no in paginator.page_range: 110 | documents_queryset = paginator.page(page_no).object_list 111 | logger.debug(f"Updating batch {page_no} of {paginator.num_pages}") 112 | future = executor.submit( 113 | update_batch, documents_queryset, collection_class, page_no 114 | ) 115 | futures.append(future) 116 | 117 | for future in concurrent.futures.as_completed(futures): 118 | future.result() 119 | 120 | 121 | def bulk_delete_typesense_records(document_ids: list, collection_name: str) -> None: 122 | """This method deletes Typesense records for objects .delete() calls 123 | from Typesense mixin subclasses. 124 | 125 | Parameters 126 | ---------- 127 | document_ids : list 128 | The list of document IDs to be deleted. 129 | collection_name : str 130 | The collection name to delete the documents from. 131 | 132 | Returns 133 | ------- 134 | None 135 | """ 136 | 137 | from django_typesense.typesense_client import client 138 | 139 | try: 140 | client.collections[collection_name].documents.delete( 141 | {"filter_by": f"id:{document_ids}"} 142 | ) 143 | except TypesenseClientError as error: 144 | logger.error( 145 | f"Could not delete the documents IDs {document_ids}\nError: {error}" 146 | ) 147 | 148 | 149 | def typesense_search(collection_name, **kwargs): 150 | """ 151 | Perform a search on the specified collection using the parameters provided. 152 | 153 | Args: 154 | collection_name: the schema name of the collection to perform the search on 155 | **kwargs: typesense search parameters 156 | 157 | Returns: 158 | A list of the typesense results 159 | """ 160 | 161 | from django_typesense.typesense_client import client 162 | 163 | if not collection_name: 164 | return 165 | 166 | search_parameters = {} 167 | 168 | for key, value in kwargs.items(): 169 | search_parameters.update({key: value}) 170 | 171 | return client.collections[collection_name].documents.search(search_parameters) 172 | 173 | 174 | def get_unix_timestamp(datetime_object) -> int: 175 | """Get the unix timestamp from a datetime object with the time part set to midnight 176 | 177 | Parameters 178 | ---------- 179 | datetime_object: date, datetime, time 180 | 181 | Returns 182 | ------- 183 | timestamp : int 184 | Returns the datetime object timestamp. 185 | 186 | Raises 187 | ------ 188 | TypeError 189 | Raised when a non datetime parameter is passed. 190 | """ 191 | 192 | # isinstance can take a union type but for backwards compatibility we call it multiple times 193 | if isinstance(datetime_object, datetime): 194 | timestamp = int(datetime_object.timestamp()) 195 | 196 | elif isinstance(datetime_object, date): 197 | timestamp = int( 198 | datetime.combine(datetime_object, datetime.min.time()).timestamp() 199 | ) 200 | 201 | elif isinstance(datetime_object, time): 202 | timestamp = int(datetime.combine(datetime.today(), datetime_object).timestamp()) 203 | 204 | else: 205 | raise TypeError( 206 | f"Expected a date/datetime/time objects but got {datetime_object} of type {type(datetime_object)}" 207 | ) 208 | 209 | return timestamp 210 | 211 | 212 | def export_documents( 213 | collection_name, 214 | filter_by: str = None, 215 | include_fields: List[str] = None, 216 | exclude_fields: List[str] = None, 217 | ) -> List[dict]: 218 | from django_typesense.typesense_client import client 219 | 220 | params = {} 221 | if filter_by is not None: 222 | params["filter_by"] = filter_by 223 | 224 | if include_fields is not None: 225 | params["include_fields"] = include_fields 226 | 227 | if exclude_fields is not None: 228 | params["exclude_fields"] = exclude_fields 229 | 230 | if not params: 231 | params = None 232 | 233 | jsonlist = ( 234 | client.collections[collection_name].documents.export(params=params).splitlines() 235 | ) 236 | return list(map(json.loads, jsonlist)) 237 | -------------------------------------------------------------------------------- /requirements-dev.txt: -------------------------------------------------------------------------------- 1 | black 2 | coverage 3 | factory-boy 4 | pre-commit 5 | -------------------------------------------------------------------------------- /runtests.py: -------------------------------------------------------------------------------- 1 | import os 2 | import sys 3 | 4 | import django 5 | from django.conf import settings 6 | from django.test.utils import get_runner 7 | from django.core.management import call_command 8 | 9 | os.environ.setdefault("DJANGO_SETTINGS_MODULE", "tests.settings") 10 | 11 | django.setup() 12 | 13 | call_command("updatecollections") 14 | 15 | TestRunner = get_runner(settings) 16 | test_runner = TestRunner() 17 | failures = test_runner.run_tests(["tests"]) 18 | 19 | if failures: 20 | sys.exit(bool(failures)) 21 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | from setuptools import find_namespace_packages, setup 2 | from django_typesense import __version__ 3 | 4 | setup( 5 | name="django_typesense", 6 | author="Siege Software", 7 | author_email="info@siege.ai", 8 | version=__version__, 9 | install_requires=[ 10 | "django", 11 | "typesense", 12 | ], 13 | setup_requires=["wheel"], 14 | packages=find_namespace_packages(), 15 | include_package_data=True, 16 | long_description=open("README.md").read(), 17 | long_description_content_type="text/markdown", 18 | license_files=("LICENSE",), 19 | classifiers=[ 20 | "Development Status :: 3 - Alpha", 21 | "License :: OSI Approved :: MIT License", 22 | "Framework :: Django", 23 | "Intended Audience :: Developers", 24 | "Natural Language :: English", 25 | "Programming Language :: Python :: 3", 26 | "Operating System :: OS Independent", 27 | ] 28 | ) 29 | -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Siege-Software/django-typesense/8bb24138e6c096bcaf0985ad650dbfd3ec40c00e/tests/__init__.py -------------------------------------------------------------------------------- /tests/collections.py: -------------------------------------------------------------------------------- 1 | from django_typesense import fields 2 | from django_typesense.collections import TypesenseCollection 3 | 4 | 5 | class SongCollection(TypesenseCollection): 6 | query_by_fields = "title,artist_names,genre_name" 7 | 8 | title = fields.TypesenseCharField() 9 | genre_name = fields.TypesenseCharField(value="genre.name") 10 | genre_id = fields.TypesenseSmallIntegerField() 11 | release_date = fields.TypesenseDateField(optional=True) 12 | artist_names = fields.TypesenseArrayField( 13 | base_field=fields.TypesenseCharField(), value="artist_names" 14 | ) 15 | number_of_comments = fields.TypesenseSmallIntegerField(index=False, optional=True) 16 | number_of_views = fields.TypesenseSmallIntegerField(index=False, optional=True) 17 | library_ids = fields.TypesenseArrayField( 18 | base_field=fields.TypesenseSmallIntegerField(), value="library_ids" 19 | ) 20 | -------------------------------------------------------------------------------- /tests/factories.py: -------------------------------------------------------------------------------- 1 | from datetime import date, timedelta 2 | 3 | import factory 4 | 5 | from tests.models import Artist, Genre, Library, Song 6 | 7 | 8 | class ArtistFactory(factory.django.DjangoModelFactory): 9 | name = factory.Sequence(lambda n: f"artist {n}") 10 | 11 | class Meta: 12 | model = Artist 13 | 14 | 15 | class GenreFactory(factory.django.DjangoModelFactory): 16 | name = factory.Sequence(lambda n: f"genre {n}") 17 | 18 | class Meta: 19 | model = Genre 20 | 21 | 22 | class SongFactory(factory.django.DjangoModelFactory): 23 | title = factory.Sequence(lambda n: f"song {n}") 24 | genre = factory.SubFactory(GenreFactory) 25 | release_date = date(year=2023, month=3, day=23) 26 | duration = timedelta(minutes=3, seconds=35) 27 | description = factory.Sequence(lambda n: f"Song description {n}") 28 | 29 | class Meta: 30 | model = Song 31 | 32 | @factory.post_generation 33 | def artists(self, create, extracted, **kwargs): 34 | if not create: 35 | return 36 | 37 | if not extracted: 38 | extracted = [ArtistFactory()] 39 | self.artists.add(*extracted) 40 | 41 | 42 | class LibraryFactory(factory.django.DjangoModelFactory): 43 | name = factory.Sequence(lambda n: f"library {n}") 44 | 45 | class Meta: 46 | model = Library 47 | 48 | @factory.post_generation 49 | def songs(self, create, extracted, **kwargs): 50 | if not create: 51 | return 52 | 53 | if not extracted: 54 | extracted = [SongFactory()] 55 | self.songs.add(*extracted) 56 | -------------------------------------------------------------------------------- /tests/models.py: -------------------------------------------------------------------------------- 1 | from django.db import models 2 | from django.utils import dateformat 3 | 4 | from django_typesense.mixins import TypesenseManager, TypesenseModelMixin 5 | from tests.collections import SongCollection 6 | 7 | 8 | class Genre(models.Model): 9 | name = models.CharField(max_length=100) 10 | 11 | def __str__(self): 12 | return self.name 13 | 14 | 15 | class Artist(models.Model): 16 | name = models.CharField(max_length=200) 17 | 18 | def __str__(self): 19 | return self.name 20 | 21 | 22 | class SongManager(TypesenseManager): 23 | def do_something(self): 24 | return None 25 | 26 | 27 | class Song(TypesenseModelMixin): 28 | title = models.CharField(max_length=100) 29 | genre = models.ForeignKey(Genre, on_delete=models.CASCADE) 30 | release_date = models.DateField(blank=True, null=True) 31 | artists = models.ManyToManyField(Artist) 32 | number_of_comments = models.IntegerField(default=0) 33 | number_of_views = models.IntegerField(default=0) 34 | duration = models.DurationField() 35 | description = models.TextField() 36 | collection_class = SongCollection 37 | 38 | objects = SongManager() 39 | 40 | def __str__(self): 41 | return self.title 42 | 43 | @property 44 | def release_date_timestamp(self): 45 | return ( 46 | int(dateformat.format(self.release_date, "U")) 47 | if self.release_date 48 | else self.release_date 49 | ) 50 | 51 | @property 52 | def library_ids(self): 53 | return list(self.libraries.values_list("id", flat=True)) 54 | 55 | def artist_names(self): 56 | return list(self.artists.all().values_list("name", flat=True)) 57 | 58 | 59 | class Library(models.Model): 60 | name = models.CharField(max_length=255) 61 | songs = models.ManyToManyField(Song, related_name="libraries") 62 | -------------------------------------------------------------------------------- /tests/settings.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | BASE_DIR = os.path.dirname(os.path.abspath(__file__)) 4 | 5 | SECRET_KEY = "foobar" 6 | 7 | INSTALLED_APPS = ( 8 | "django.contrib.admin", 9 | "django.contrib.auth", 10 | "django.contrib.contenttypes", 11 | "django.contrib.sessions", 12 | "django.contrib.sites", 13 | "django.contrib.messages", 14 | "django_typesense", 15 | "tests", 16 | ) 17 | 18 | MIDDLEWARE = [ 19 | "django.middleware.security.SecurityMiddleware", 20 | "django.contrib.sessions.middleware.SessionMiddleware", 21 | "django.middleware.common.CommonMiddleware", 22 | "django.middleware.csrf.CsrfViewMiddleware", 23 | "django.contrib.auth.middleware.AuthenticationMiddleware", 24 | "django.contrib.messages.middleware.MessageMiddleware", 25 | "django.middleware.clickjacking.XFrameOptionsMiddleware", 26 | ] 27 | 28 | DATABASES = {"default": {"ENGINE": "django.db.backends.sqlite3", "NAME": "db.sqlite3"}} 29 | 30 | TEMPLATES = [ 31 | { 32 | "BACKEND": "django.template.backends.django.DjangoTemplates", 33 | "DIRS": [], 34 | "APP_DIRS": True, 35 | "OPTIONS": { 36 | "context_processors": [ 37 | "django.template.context_processors.debug", 38 | "django.template.context_processors.request", 39 | "django.contrib.auth.context_processors.auth", 40 | "django.contrib.messages.context_processors.messages", 41 | ], 42 | }, 43 | }, 44 | ] 45 | 46 | TYPESENSE = { 47 | "api_key": "sample_key", 48 | "nodes": [{"host": "localhost", "protocol": "http", "port": "8108"}], 49 | } 50 | 51 | USE_TZ = True 52 | 53 | DEFAULT_AUTO_FIELD = "django.db.models.BigAutoField" 54 | -------------------------------------------------------------------------------- /tests/test_typesense_admin_mixin.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Siege-Software/django-typesense/8bb24138e6c096bcaf0985ad650dbfd3ec40c00e/tests/test_typesense_admin_mixin.py -------------------------------------------------------------------------------- /tests/test_typesense_changelist.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Siege-Software/django-typesense/8bb24138e6c096bcaf0985ad650dbfd3ec40c00e/tests/test_typesense_changelist.py -------------------------------------------------------------------------------- /tests/test_typesense_fields.py: -------------------------------------------------------------------------------- 1 | import json 2 | from datetime import date, datetime, time, timedelta 3 | from decimal import Decimal 4 | 5 | from django.test import TestCase 6 | 7 | from django_typesense import fields 8 | from tests.collections import SongCollection 9 | from tests.models import Artist, Genre, Song 10 | 11 | 12 | class TestTypesenseCharField(TestCase): 13 | def setUp(self): 14 | self.genre = Genre.objects.create(name="genre1") 15 | self.song = Song.objects.create( 16 | title="New Song", 17 | genre=self.genre, 18 | release_date=date.today(), 19 | description="New song description", 20 | duration=timedelta(minutes=3, seconds=35), 21 | ) 22 | self.attrs = [ 23 | attr if attr != "_field_type" else "type" 24 | for attr in fields.TYPESENSE_SCHEMA_ATTRS 25 | ] 26 | 27 | def test_string_representation(self): 28 | self.assertEqual(str(SongCollection.title), "title") 29 | 30 | def test_attrs(self): 31 | title_attrs = SongCollection.title.attrs 32 | self.assertCountEqual(title_attrs.keys(), self.attrs) 33 | self.assertEqual(title_attrs["name"], "title") 34 | self.assertEqual(title_attrs["type"], "string") 35 | self.assertEqual(title_attrs["locale"], "") 36 | self.assertFalse(title_attrs["sort"]) 37 | self.assertFalse(title_attrs["optional"]) 38 | self.assertFalse(title_attrs["facet"]) 39 | self.assertFalse(title_attrs["infix"]) 40 | self.assertTrue(title_attrs["index"]) 41 | 42 | def test_field_type(self): 43 | self.assertEqual(SongCollection.title.field_type, "string") 44 | 45 | def test_value_method(self): 46 | title = SongCollection.title 47 | self.assertEqual(title.value(obj=self.song), self.song.title) 48 | 49 | optional_field = fields.TypesenseCharField(value="name", optional=True) 50 | self.assertEqual(optional_field.value(obj=self.song), "") 51 | 52 | with self.assertRaises(AttributeError): 53 | invalid_field = fields.TypesenseCharField(value="invalid_field") 54 | invalid_field.value(obj=self.song) 55 | 56 | def test_to_python_method(self): 57 | title = SongCollection.title.value(obj=self.song) 58 | self.assertEqual(SongCollection.title.to_python(value=title), self.song.title) 59 | 60 | 61 | class TestTypesenseIntegerMixin(TestCase): 62 | def setUp(self): 63 | self.genre = Genre.objects.create(name="genre1") 64 | self.song = Song.objects.create( 65 | title="New Song", 66 | genre=self.genre, 67 | release_date=date.today(), 68 | description="New song description", 69 | duration=timedelta(minutes=3, seconds=35), 70 | ) 71 | 72 | def test_value_method(self): 73 | genre_id = SongCollection.genre_id 74 | self.assertEqual(genre_id.value(obj=self.song), self.genre.pk) 75 | 76 | views = fields.TypesenseSmallIntegerField(value="views", optional=True) 77 | self.assertIsNone(views.value(obj=self.song)) 78 | 79 | self.song.views = "Wrong type" 80 | with self.assertRaises(ValueError): 81 | views.value(obj=self.song) 82 | 83 | 84 | class TestTypesenseDecimalField(TestCase): 85 | def setUp(self): 86 | self.genre = Genre.objects.create(name="genre1") 87 | self.song = Song.objects.create( 88 | title="New Song", 89 | genre=self.genre, 90 | release_date=date.today(), 91 | description="New song description", 92 | duration=timedelta(minutes=3, seconds=35), 93 | ) 94 | 95 | def test_value_method(self): 96 | price = fields.TypesenseDecimalField(value="price", optional=True) 97 | self.assertEqual(price.value(obj=self.song), "None") 98 | 99 | self.song.price = Decimal("23.00") 100 | self.assertEqual(price.value(obj=self.song), "23.00") 101 | 102 | def test_to_python_method(self): 103 | price = fields.TypesenseDecimalField(value="price", optional=True) 104 | self.song.price = Decimal("23.00") 105 | price_value = price.value(obj=self.song) 106 | self.assertEqual(price.to_python(value=price_value), Decimal("23.00")) 107 | 108 | 109 | class TestTypesenseDateTimeFieldBase(TestCase): 110 | def setUp(self): 111 | self.genre = Genre.objects.create(name="genre1") 112 | self.song = Song.objects.create( 113 | title="New Song", 114 | genre=self.genre, 115 | release_date=date.today(), 116 | description="New song description", 117 | duration=timedelta(minutes=3, seconds=35), 118 | ) 119 | 120 | def test_value_method(self): 121 | optional_date = fields.TypesenseDateField(value="optional_field", optional=True) 122 | self.assertIsNone(optional_date.value(obj=self.song)) 123 | 124 | release_date_timestamp = fields.TypesenseDateField( 125 | value="release_date_timestamp" 126 | ) 127 | release_date_timestamp_value = release_date_timestamp.value(obj=self.song) 128 | self.assertTrue(isinstance(release_date_timestamp_value, int)) 129 | self.assertEqual(release_date_timestamp_value, self.song.release_date_timestamp) 130 | 131 | def test_date_field(self): 132 | release_date = SongCollection.release_date 133 | release_date_value = release_date.value(obj=self.song) 134 | self.assertTrue(isinstance(release_date_value, int)) 135 | self.assertEqual(release_date_value, self.song.release_date_timestamp) 136 | 137 | release_date_to_python = release_date.to_python(value=release_date_value) 138 | self.assertTrue(isinstance(release_date_to_python, date)) 139 | self.assertEqual(release_date_to_python, self.song.release_date) 140 | 141 | def test_time_field(self): 142 | release_time = fields.TypesenseTimeField(value="release_time") 143 | self.song.release_time = time(hour=13, minute=30, second=0) 144 | release_time_value = release_time.value(obj=self.song) 145 | 146 | self.assertTrue(isinstance(release_time.to_python(release_time_value), time)) 147 | self.assertEqual( 148 | release_time.to_python(release_time_value), self.song.release_time 149 | ) 150 | 151 | def test_datetime_field(self): 152 | created_at = fields.TypesenseDateTimeField(value="created_at") 153 | self.song.created_at = datetime( 154 | year=2023, month=9, day=11, hour=16, minute=20, second=0 155 | ) 156 | created_at_value = created_at.value(obj=self.song) 157 | 158 | self.assertTrue(isinstance(created_at.to_python(created_at_value), datetime)) 159 | self.assertEqual( 160 | created_at.to_python(value=created_at_value), self.song.created_at 161 | ) 162 | 163 | 164 | class TestTypesenseJSONField(TestCase): 165 | def setUp(self): 166 | self.genre = Genre.objects.create(name="genre1") 167 | self.song = Song.objects.create( 168 | title="New Song", 169 | genre=self.genre, 170 | release_date=date.today(), 171 | description="New song description", 172 | duration=timedelta(minutes=3, seconds=35), 173 | ) 174 | 175 | def test_value_method(self): 176 | extra_info = fields.TypesenseJSONField(value="extra_info") 177 | self.song.extra_info = {"producer": "DJ Something"} 178 | extra_info_value = extra_info.value(obj=self.song) 179 | 180 | self.assertTrue(isinstance(extra_info_value, str)) 181 | self.assertEqual(extra_info_value, json.dumps(self.song.extra_info)) 182 | 183 | def test_to_python_method(self): 184 | extra_info = fields.TypesenseJSONField(value="extra_info") 185 | self.song.extra_info = {"producer": "DJ Something"} 186 | extra_info_value = extra_info.value(obj=self.song) 187 | 188 | self.assertEqual( 189 | extra_info.to_python(value=extra_info_value), self.song.extra_info 190 | ) 191 | 192 | 193 | class TestTypesenseArrayField(TestCase): 194 | def setUp(self): 195 | self.genre = Genre.objects.create(name="genre1") 196 | self.artist = Artist.objects.create(name="artist1") 197 | self.song = Song.objects.create( 198 | title="New Song", 199 | genre=self.genre, 200 | release_date=date.today(), 201 | description="New song description", 202 | duration=timedelta(minutes=3, seconds=35), 203 | ) 204 | self.song.artists.add(self.artist) 205 | 206 | def test_to_python_method(self): 207 | artist_names = SongCollection.artist_names 208 | artist_names_value = artist_names.value(obj=self.song) 209 | 210 | self.assertCountEqual( 211 | artist_names.to_python(artist_names_value), self.song.artist_names() 212 | ) 213 | -------------------------------------------------------------------------------- /tests/test_typesense_methods.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Siege-Software/django-typesense/8bb24138e6c096bcaf0985ad650dbfd3ec40c00e/tests/test_typesense_methods.py -------------------------------------------------------------------------------- /tests/test_typesense_model_mixin.py: -------------------------------------------------------------------------------- 1 | from django.test import TestCase 2 | 3 | from django_typesense.mixins import TypesenseManager 4 | 5 | from tests.factories import ArtistFactory, GenreFactory, SongFactory 6 | from tests.models import Song 7 | from tests.utils import get_document 8 | 9 | 10 | class TestTypeSenseMixin(TestCase): 11 | def setUp(self): 12 | self.genre = GenreFactory() 13 | self.artist = ArtistFactory() 14 | self.song = SongFactory(genre=self.genre, artists=[self.artist]) 15 | 16 | def test_get_collection_class(self): 17 | collection_class = Song.get_collection_class() 18 | self.assertEqual(collection_class.__name__, "SongCollection") 19 | 20 | def test_typesense_manager(self): 21 | self.assertIsInstance(Song.objects, TypesenseManager) 22 | 23 | def test_update_collection(self): 24 | schema_name = self.song.collection_class.schema_name 25 | song_document = get_document(schema_name, self.song.pk) 26 | self.assertEqual(song_document["genre_name"], self.song.genre.name) 27 | 28 | genre_name = "Dancehall" 29 | self.genre.name = genre_name 30 | self.genre.save(update_fields=["name"]) 31 | 32 | song_document = get_document(schema_name, self.song.pk) 33 | self.assertNotEqual(song_document["genre_name"], self.song.genre.name) 34 | self.assertEqual(self.song.genre.name, genre_name) 35 | 36 | Song.objects.get_queryset().update() 37 | song_document = get_document(schema_name, self.song.pk) 38 | self.assertEqual(song_document["genre_name"], genre_name) 39 | self.assertEqual(song_document["genre_name"], self.song.genre.name) 40 | 41 | def test_delete_collection(self): 42 | schema_name = self.song.collection_class.schema_name 43 | song_document = get_document(schema_name, self.song.pk) 44 | self.assertEqual(song_document["title"], self.song.title) 45 | 46 | Song.objects.get_queryset().delete() 47 | 48 | song_document = get_document(schema_name, self.song.pk) 49 | self.assertIsNone(song_document) 50 | -------------------------------------------------------------------------------- /tests/test_typesense_paginator.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Siege-Software/django-typesense/8bb24138e6c096bcaf0985ad650dbfd3ec40c00e/tests/test_typesense_paginator.py -------------------------------------------------------------------------------- /tests/test_typesense_signals.py: -------------------------------------------------------------------------------- 1 | from django.test import TestCase 2 | 3 | from tests.factories import ArtistFactory, GenreFactory, SongFactory 4 | from tests.models import Artist, Library, Song 5 | from tests.utils import get_document 6 | 7 | 8 | class TestTypeSenseSignals(TestCase): 9 | def setUp(self): 10 | self.genre = GenreFactory() 11 | self.artist = ArtistFactory() 12 | self.song = SongFactory(genre=self.genre) 13 | 14 | def test_post_save_typesense_models(self): 15 | schema_name = self.song.collection_class.schema_name 16 | song_document = get_document(schema_name, self.song.pk) 17 | 18 | self.assertIsNotNone(song_document) 19 | self.assertEqual(song_document["title"], self.song.title) 20 | self.assertEqual(song_document["genre_id"], self.song.genre.pk) 21 | self.assertEqual(song_document["genre_name"], self.song.genre.name) 22 | self.assertEqual( 23 | song_document["release_date"], self.song.release_date_timestamp 24 | ) 25 | self.assertEqual(song_document["number_of_views"], self.song.number_of_views) 26 | self.assertEqual( 27 | song_document["number_of_comments"], self.song.number_of_comments 28 | ) 29 | 30 | def test_pre_delete_typesense_models(self): 31 | schema_name = self.song.collection_class.schema_name 32 | song_pk = self.song.pk 33 | 34 | song_document = get_document(schema_name, song_pk) 35 | self.assertIsNotNone(song_document) 36 | 37 | self.genre.delete() 38 | self.assertFalse(Song.objects.filter(pk=song_pk).exists()) 39 | song_document = get_document(schema_name, song_pk) 40 | self.assertIsNone(song_document) 41 | 42 | def test_m2m_changed_typesense_models(self): 43 | schema_name = self.song.collection_class.schema_name 44 | 45 | song_document = get_document(schema_name, self.song.pk) 46 | self.assertIsNotNone(song_document) 47 | self.assertCountEqual(song_document["artist_names"], self.song.artist_names()) 48 | 49 | self.song.artists.add(self.artist) 50 | song_document = get_document(schema_name, self.song.pk) 51 | self.assertIsNotNone(song_document) 52 | self.assertCountEqual(song_document["artist_names"], self.song.artist_names()) 53 | 54 | artist_2 = Artist.objects.create(name="artist2") 55 | artist_2.song_set.add(self.song) 56 | song_document = get_document(schema_name, self.song.pk) 57 | self.assertIsNotNone(song_document) 58 | self.assertCountEqual(song_document["artist_names"], self.song.artist_names()) 59 | 60 | library = Library.objects.create(name="new album") 61 | library.songs.add(self.song) 62 | 63 | song_document = get_document(schema_name, self.song.pk) 64 | self.assertIsNotNone(song_document) 65 | self.assertCountEqual(song_document["library_ids"], self.song.library_ids) 66 | 67 | self.song.libraries.remove(library) 68 | song_document = get_document(schema_name, self.song.pk) 69 | self.assertIsNotNone(song_document) 70 | self.assertCountEqual(song_document["library_ids"], self.song.library_ids) 71 | -------------------------------------------------------------------------------- /tests/test_typesense_utils.py: -------------------------------------------------------------------------------- 1 | from datetime import date, datetime, time 2 | from unittest import mock 3 | 4 | from django.db.utils import OperationalError 5 | from django.test import TestCase 6 | from typesense.exceptions import TypesenseClientError 7 | 8 | from django_typesense.exceptions import BatchUpdateError, UnorderedQuerySetError 9 | from django_typesense.utils import ( 10 | bulk_delete_typesense_records, 11 | bulk_update_typesense_records, 12 | get_unix_timestamp, 13 | typesense_search, 14 | update_batch, 15 | ) 16 | 17 | from tests.collections import SongCollection 18 | from tests.factories import ArtistFactory, SongFactory 19 | from tests.models import Artist, Song 20 | from tests.utils import get_document 21 | 22 | 23 | class TestUpdateBatch(TestCase): 24 | def setUp(self): 25 | self.song_count = 10 26 | SongFactory.create_batch(size=self.song_count) 27 | 28 | def test_update_batch(self): 29 | songa = Song.objects.all() 30 | self.assertEqual(songa.count(), self.song_count) 31 | 32 | with self.assertLogs(level="DEBUG") as logs: 33 | batch_number = 1 34 | update_batch(songa, SongCollection, batch_number) 35 | self.assertEqual( 36 | logs.output[-1], 37 | f"DEBUG:django_typesense.utils:Batch {batch_number} Updated with {self.song_count} records ✓", 38 | ) 39 | 40 | @mock.patch( 41 | "tests.collections.SongCollection.update", return_value=[{"success": False}] 42 | ) 43 | def test_update_batch_with_error(self, _): 44 | songs = Song.objects.all() 45 | self.assertEqual(songs.count(), self.song_count) 46 | 47 | with self.assertRaises(BatchUpdateError): 48 | update_batch(songs, SongCollection, 1) 49 | 50 | 51 | class TestBulkUpdateTypesenseRecords(TestCase): 52 | def setUp(self): 53 | self.unordered_queryset_message = ( 54 | "Pagination may yield inconsistent results with an unordered object_list. " 55 | "Please provide an ordered objects" 56 | ) 57 | self.song_count = 10 58 | SongFactory.create_batch(size=self.song_count) 59 | 60 | def test_bulk_update_typesense_records(self): 61 | songs = Song.objects.all().order_by("pk") 62 | 63 | # This exception is thrown because the tests are running 64 | # against SQLite which doesn't support a high level of concurrency. 65 | # https://docs.djangoproject.com/en/4.2/ref/databases/#database-is-locked-errors 66 | with self.assertRaises(OperationalError): 67 | bulk_update_typesense_records(songs, batch_size=200, num_threads=2) 68 | 69 | def test_bulk_update_typesense_records_invalid_type(self): 70 | ArtistFactory.create_batch(size=20) 71 | artists = Artist.objects.all().order_by("pk") 72 | 73 | with self.assertLogs(level="ERROR") as logs: 74 | bulk_update_typesense_records(artists, batch_size=200, num_threads=2) 75 | expected_log = ( 76 | f"The objects for {artists.model.__name__} does not use TypesenseQuerySet " 77 | "as it's manager. Please update the model manager for the class to use Typesense." 78 | ) 79 | last_log_message = logs[-1][0].replace("ERROR:django_typesense.utils:", "") 80 | self.assertEqual(last_log_message, expected_log) 81 | 82 | @mock.patch("django.db.models.QuerySet.order_by", side_effect=TypeError) 83 | def test_bulk_update_typesense_records_unordered_queryset(self, _): 84 | songs = Song.objects.all() 85 | 86 | with self.assertRaises(UnorderedQuerySetError): 87 | bulk_update_typesense_records(songs, batch_size=200, num_threads=2) 88 | 89 | 90 | class TestBulkDeleteTypesenseRecords(TestCase): 91 | def setUp(self): 92 | self.schema_name = Song.collection_class.schema_name 93 | SongFactory.create_batch(size=20) 94 | 95 | def test_bulk_delete_typesense_records(self): 96 | songs = Song.objects.all().order_by("pk") 97 | first_ten_songs = songs[:10] 98 | 99 | song_ids = [] 100 | for song in first_ten_songs: 101 | song_document = get_document(self.schema_name, song.pk) 102 | self.assertIsNotNone(song_document) 103 | self.assertEqual(song_document["title"], song.title) 104 | song_ids.append(song.pk) 105 | 106 | bulk_delete_typesense_records(song_ids, self.schema_name) 107 | 108 | for song in songs: 109 | song_document = get_document(self.schema_name, song.pk) 110 | if song.pk in song_ids: 111 | self.assertIsNone(song_document) 112 | else: 113 | self.assertIsNotNone(song_document) 114 | self.assertEqual(song_document["title"], song.title) 115 | 116 | @mock.patch( 117 | "typesense.documents.Documents.delete", side_effect=TypesenseClientError 118 | ) 119 | def test_bulk_delete_typesense_records_exception_raised(self, _): 120 | songs = Song.objects.all().order_by("pk") 121 | first_ten_songs = songs[:10] 122 | 123 | song_ids = [] 124 | for song in first_ten_songs: 125 | song_document = get_document(self.schema_name, song.pk) 126 | self.assertIsNotNone(song_document) 127 | self.assertEqual(song_document["title"], song.title) 128 | song_ids.append(song.pk) 129 | 130 | with self.assertLogs(level="ERROR") as logs: 131 | bulk_delete_typesense_records(song_ids, self.schema_name) 132 | last_log = logs[-1][0].replace("ERROR:django_typesense.utils:", "") 133 | expected_log_message = ( 134 | f"Could not delete the documents IDs {song_ids}\nError: " 135 | ) 136 | self.assertEqual(last_log, expected_log_message) 137 | 138 | for song in songs: 139 | song_document = get_document(self.schema_name, song.pk) 140 | self.assertIsNotNone(song_document) 141 | self.assertEqual(song_document["title"], song.title) 142 | 143 | 144 | class TestTypesenseSearch(TestCase): 145 | def setUp(self): 146 | self.collection_name = Song.collection_class.schema_name 147 | self.query_fields = Song.collection_class.query_by_fields 148 | SongFactory.create_batch(size=20) 149 | 150 | def test_typesense_search(self): 151 | data = {"q": "song", "query_by": self.query_fields} 152 | results = typesense_search(self.collection_name, **data) 153 | self.assertIsNotNone(results) 154 | self.assertEqual(results["found"], 20) 155 | 156 | # def test_typesense_search_invalid_parameters(self): 157 | # data = {"q": "song", "query_by": self.query_fields} 158 | # results = typesense_search("", **data) 159 | # self.assertIsNone(results) 160 | # 161 | # data = {"q": "song"} 162 | # results = typesense_search(self.collection_name, **data) 163 | # self.assertIsNone(results) 164 | # 165 | # data = {"q": "song", "query_by": self.query_fields} 166 | # results = typesense_search("invalid_collection", **data) 167 | # self.assertIsNone(results) 168 | 169 | 170 | class TestGetUnixTimestamp(TestCase): 171 | def test_get_unix_timestamp_datetime(self): 172 | now = datetime(year=2023, month=11, day=23, hour=16, minute=20, second=00) 173 | now_timestamp = get_unix_timestamp(now) 174 | self.assertTrue(isinstance(now_timestamp, int)) 175 | self.assertEqual(now_timestamp, now.timestamp()) 176 | 177 | def test_get_unix_timestamp_date(self): 178 | today = date(year=2023, month=11, day=23) 179 | today_timestamp = get_unix_timestamp(today) 180 | self.assertTrue(isinstance(today_timestamp, int)) 181 | self.assertEqual( 182 | today_timestamp, datetime.combine(today, datetime.min.time()).timestamp() 183 | ) 184 | 185 | def test_get_unix_timestamp_time(self): 186 | now = time(hour=16, minute=20, second=00) 187 | now_timestamp = get_unix_timestamp(now) 188 | self.assertTrue(isinstance(now_timestamp, int)) 189 | self.assertEqual( 190 | now_timestamp, datetime.combine(datetime.today(), now).timestamp() 191 | ) 192 | 193 | def test_get_unix_timestamp_invalid_type(self): 194 | invalid_datetime = "2023-11-23 16:20:00" 195 | with self.assertRaises(TypeError): 196 | get_unix_timestamp(invalid_datetime) 197 | -------------------------------------------------------------------------------- /tests/utils.py: -------------------------------------------------------------------------------- 1 | from typesense import exceptions 2 | 3 | from django_typesense.typesense_client import client 4 | 5 | 6 | def get_document(schema_name, document_id): 7 | try: 8 | return client.collections[schema_name].documents[document_id].retrieve() 9 | except exceptions.ObjectNotFound: 10 | return None 11 | --------------------------------------------------------------------------------