├── .github └── workflows │ ├── lint.yml │ └── test.yml ├── .gitignore ├── .readthedocs.yaml ├── CHANGES.rst ├── LICENSE ├── README.rst ├── docs ├── Makefile ├── alembic_migrations.rst ├── conf.py ├── configuration.rst ├── index.rst ├── integrations.rst ├── make.bat ├── quickstart.rst ├── requirements.in ├── requirements.txt ├── search_query_parser.rst └── vectorizers.rst ├── pyproject.toml ├── sqlalchemy_searchable ├── __init__.py ├── expressions.sql └── vectorizers.py ├── tests ├── __init__.py ├── conftest.py ├── schema_test_case.py ├── test_class_configuration.py ├── test_drop_trigger.py ├── test_multiple_vectors_per_class.py ├── test_schema_creation.py ├── test_searchable.py ├── test_single_table_inheritance.py ├── test_sql_functions.py ├── test_sync_trigger.py ├── test_vectorizers.py └── test_weighted_search_vector.py └── tox.ini /.github/workflows/lint.yml: -------------------------------------------------------------------------------- 1 | name: Lint 2 | 3 | on: 4 | - push 5 | - pull_request 6 | 7 | jobs: 8 | test: 9 | name: Lint 10 | runs-on: ubuntu-latest 11 | steps: 12 | - uses: actions/checkout@v3 13 | 14 | - uses: actions/setup-python@v4 15 | with: 16 | python-version: 3.11 17 | 18 | - name: Install dependencies 19 | run: | 20 | python -m pip install --upgrade pip 21 | pip install --upgrade tox setuptools 22 | 23 | - name: Run linting 24 | run: tox -e lint 25 | -------------------------------------------------------------------------------- /.github/workflows/test.yml: -------------------------------------------------------------------------------- 1 | name: Test 2 | 3 | on: 4 | - push 5 | - pull_request 6 | 7 | jobs: 8 | tests: 9 | name: Python ${{ matrix.python }} + PostgreSQL ${{ matrix.postgresql }} 10 | runs-on: ubuntu-latest 11 | services: 12 | postgres: 13 | image: postgres:${{ matrix.postgresql }} 14 | env: 15 | POSTGRES_DB: sqlalchemy_searchable_test 16 | POSTGRES_USER: postgres 17 | POSTGRES_PASSWORD: postgres 18 | # Set health checks to wait until postgres has started 19 | options: >- 20 | --health-cmd pg_isready 21 | --health-interval 10s 22 | --health-timeout 5s 23 | --health-retries 5 24 | ports: 25 | - 5432:5432 26 | strategy: 27 | fail-fast: false 28 | matrix: 29 | include: 30 | - python: "3.12" 31 | postgresql: "16" 32 | 33 | - python: "3.11" 34 | postgresql: "15" 35 | 36 | - python: "3.10" 37 | postgresql: "14" 38 | 39 | - python: "3.9" 40 | postgresql: "13" 41 | 42 | - python: "3.8" 43 | postgresql: "12" 44 | 45 | - python: "pypy3.9" 46 | postgresql: "11" 47 | 48 | steps: 49 | - uses: actions/checkout@v3 50 | 51 | - uses: actions/setup-python@v4 52 | with: 53 | python-version: ${{ matrix.python }} 54 | 55 | - name: Install dependencies 56 | run: | 57 | python -m pip install --upgrade pip 58 | pip install --upgrade tox setuptools 59 | 60 | - name: Run tests 61 | env: 62 | SQLALCHEMY_SEARCHABLE_TEST_PASSWORD: postgres 63 | TOXENV: py-sqla1.4, py-sqla2.0 64 | run: tox 65 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.py[cod] 2 | 3 | # C extensions 4 | *.so 5 | 6 | # Packages 7 | *.egg 8 | *.egg-info 9 | dist 10 | build 11 | eggs 12 | parts 13 | bin 14 | var 15 | sdist 16 | develop-eggs 17 | .installed.cfg 18 | lib 19 | lib64 20 | docs/_build 21 | 22 | # Installer logs 23 | pip-log.txt 24 | 25 | # Unit test / coverage reports 26 | .coverage 27 | .tox 28 | nosetests.xml 29 | 30 | # Translations 31 | *.mo 32 | 33 | # Mr Developer 34 | .mr.developer.cfg 35 | .project 36 | .pydevproject 37 | -------------------------------------------------------------------------------- /.readthedocs.yaml: -------------------------------------------------------------------------------- 1 | version: 2 2 | build: 3 | os: ubuntu-22.04 4 | tools: 5 | python: "3.11" 6 | python: 7 | install: 8 | - method: pip 9 | path: . 10 | - requirements: docs/requirements.txt 11 | sphinx: 12 | fail_on_warning: false 13 | -------------------------------------------------------------------------------- /CHANGES.rst: -------------------------------------------------------------------------------- 1 | Changelog 2 | --------- 3 | 4 | Here you can see the full list of changes between each SQLAlchemy-Searchable release. 5 | 6 | 2.1.0 (2024-02-19) 7 | ^^^^^^^^^^^^^^^^^^ 8 | 9 | - Add support for Python 3.12 10 | - Add support for PostgreSQL 16 11 | - Allow specifying schema in ``drop_trigger`` and ``sync_trigger`` (#95, pull 12 | request by @acarapetis) 13 | - Add ``update_rows`` parameter to ``sync_trigger`` (#76, pull request by 14 | @scribu) 15 | 16 | 2.0.0 (2023-08-28) 17 | ^^^^^^^^^^^^^^^^^^ 18 | 19 | - **BREAKING CHANGE**: Drop support for Python 3.6 and 3.7. 20 | - **BREAKING CHANGE**: Drop support for SQLAlchemy 1.3. 21 | - **BREAKING CHANGE**: Remove ``quote_identifier`` function. 22 | - **BREAKING CHANGE**: Remove ``SearchManager.search_function_ddl`` method. Use 23 | ``CreateSearchFunctionSQL(column)`` instead. 24 | - **BREAKING CHANGE**: Remove ``SearchManager.search_trigger_ddl`` method. Use 25 | ``CreateSearchTriggerSQL(column)`` instead. 26 | - Migrate from Travis CI to Github workflows in order to have a working CI 27 | again. 28 | - Remove ``validators`` dependency 29 | - Add support for Python 3.10 and 3.11. 30 | - Use the ``pyproject.toml`` standard to specify project metadata, dependencies 31 | and tool configuration. Use Hatch to build the project. 32 | - Use Ruff for linting the project, replacing both isort and flake8. 33 | - Upgrade Python syntax with pyupgrade to the minimum Python version supported. 34 | - Use Black to format Python code. 35 | - Add support for SQLAlchemy 2.0. 36 | - Use SQLAlchemy's compilation extension to build the SQL for creating and 37 | dropping the search functions and triggers. 38 | - Update SQLAlchemy-Utils dependency to >=0.40.0. 39 | - Fix the deprecation warning for ``sqlalchemy.orm.mapper()`` in 40 | ``make_searchable()`` and ``remove_listeners()``. 41 | - Migrate the Read the Docs configuration to use `.readthedocs.yaml` 42 | configuration file. 43 | - Rewrite the test suite to use pytest fixtures ove classic xunit-style set-up. 44 | - Fix ``parse_websearch`` tests on PostgreSQL 14 and later. 45 | - Add PostgreSQL versions 11, 13, 14 and 15 to the CI build matrix. Previously, 46 | the tests were only run on PostgreSQL 12. 47 | - Rewrite the documentation to be up-to-date with the codebase, to fix the 48 | grammar and formatting issues, and to ensure the code examples are correct. 49 | - The query interface is considered legacy in SQLAlchemy 2.0. Migrate the tests 50 | and documentation to use ``Session.execute()`` in conjunction with ``select()`` to 51 | run ORM queries. 52 | 53 | 1.4.1 (2021-06-15) 54 | ^^^^^^^^^^^^^^^^^^ 55 | 56 | - Added auto_index option 57 | 58 | 59 | 1.4.0 (2021-06-13) 60 | ^^^^^^^^^^^^^^^^^^ 61 | 62 | - Simplify search parsing 63 | - Fix parser errors with search keywords containing special characters such as underscores 64 | 65 | 66 | 1.3.0 (2021-06-02) 67 | ^^^^^^^^^^^^^^^^^^ 68 | 69 | - Raise PostgreSQL requirement to version 11 70 | - Use websearch_to_tsquery internally rather than own parsing functions 71 | - Drop py34, py35 support 72 | 73 | 74 | 1.2.0 (2020-07-10) 75 | ^^^^^^^^^^^^^^^^^^ 76 | 77 | - Fixed 'or' keyword parsing (#93) 78 | - Dropped py27 support 79 | 80 | 81 | 1.1.0 (2019-07-05) 82 | ^^^^^^^^^^^^^^^^^^ 83 | 84 | - Fixed some issues with query parsing 85 | - Fixed 'or' keyword parsing (#85) 86 | - Dropped py33 support 87 | - Fixed deprecation warnings (#81, pull request courtesy of Le-Stagiaire) 88 | 89 | 90 | 1.0.3 (2018-02-22) 91 | ^^^^^^^^^^^^^^^^^^ 92 | 93 | - Add missing expressions.sql 94 | 95 | 96 | 1.0.2 (2018-02-22) 97 | ^^^^^^^^^^^^^^^^^^ 98 | 99 | - Fixed import issue with expressions.sql 100 | 101 | 102 | 1.0.1 (2018-02-20) 103 | ^^^^^^^^^^^^^^^^^^ 104 | 105 | - Made all parser functions immutable 106 | 107 | 108 | 1.0 (2018-02-20) 109 | ^^^^^^^^^^^^^^^^ 110 | 111 | - Added pure PostgreSQL search query parsing (faster and can be used on SQL level) 112 | - PostgreSQL >= 9.6 required 113 | - Added support for phrase searching 114 | - Removed python search query parsing 115 | - Removed pyparsing from requirements 116 | - Removed symbol removal (now handled implicitly on PostgreSQL side) 117 | 118 | 119 | 0.10.6 (2017-10-12) 120 | ^^^^^^^^^^^^^^^^^^^ 121 | 122 | - Fixed Flask-SQLAlchemy support (#63, pull request by quantus) 123 | 124 | 125 | 0.10.5 (2017-07-25) 126 | ^^^^^^^^^^^^^^^^^^^ 127 | 128 | - Added drop_trigger utility function (#58, pull request by ilya-chistyakov) 129 | 130 | 131 | 0.10.4 (2017-06-28) 132 | ^^^^^^^^^^^^^^^^^^^ 133 | 134 | - Index generation no longer manipulates table args (#55, pull request by jmuhlich) 135 | 136 | 137 | 0.10.3 (2017-01-26) 138 | ^^^^^^^^^^^^^^^^^^^ 139 | 140 | - Fixed 'Lo' unicode letter parsing (#50, pull request courtesy by StdCarrot) 141 | 142 | 143 | 0.10.2 (2016-09-02) 144 | ^^^^^^^^^^^^^^^^^^^ 145 | 146 | - Fixed vector matching to use global configuration regconfig as fallback 147 | 148 | 149 | 0.10.1 (2016-04-14) 150 | ^^^^^^^^^^^^^^^^^^^ 151 | 152 | - Use identifier quoting for reserved keywords (#45, pull request by cristen) 153 | 154 | 155 | 0.10.0 (2016-03-31) 156 | ^^^^^^^^^^^^^^^^^^ 157 | 158 | - Fixed unicode parsing in search query parser, #42 159 | - Removed Python 2.6 support 160 | 161 | 162 | 0.9.3 (2015-05-31) 163 | ^^^^^^^^^^^^^^^^^^ 164 | 165 | - Added support for search term weights 166 | 167 | 168 | 0.9.2 (2015-04-01) 169 | ^^^^^^^^^^^^^^^^^^ 170 | 171 | - Fixed listener configuration (#31) 172 | 173 | 174 | 0.9.1 (2015-03-25) 175 | ^^^^^^^^^^^^^^^^^^ 176 | 177 | - Added sort param to search function for ordering search results by relevance 178 | 179 | 180 | 0.9.0 (2015-03-19) 181 | ^^^^^^^^^^^^^^^^^^ 182 | 183 | - Added PyPy support 184 | - Added isort and flake8 checks 185 | - Added support for custom vectorizers in sync_trigger, #25 186 | - Fixed and / or parsing where search word started with keyword, #22 187 | - Removed 'and' as keyword from search query parser (spaces are always considered as 'and' keywords) 188 | 189 | 190 | 0.8.0 (2015-01-03) 191 | ^^^^^^^^^^^^^^^^^^ 192 | 193 | - Made search function support for queries without entity_zero 194 | - Changed catalog configuration option name to regconfig to be compatible with the PostgreSQL and SQLAlchemy naming 195 | - Added custom type and column vectorizers 196 | - SQLAlchemy requirement updated to 0.9.0 197 | - SQLAlchemy-Utils requirement updated to 0.29.0 198 | 199 | 200 | 0.7.1 (2014-12-16) 201 | ^^^^^^^^^^^^^^^^^^ 202 | 203 | - Changed GIN indexes to table args Index constructs. This means current version of alembic should be able to create these indexes automatically. 204 | - Changed GIN index naming to adhere to SQLAlchemy index naming conventions 205 | 206 | 207 | 0.7.0 (2014-11-17) 208 | ^^^^^^^^^^^^^^^^^^ 209 | 210 | - Replaced remove_hyphens configuration option by more generic remove_symbols configuration option 211 | - Emails are no longer considered as special tokens by default. 212 | 213 | 214 | 0.6.0 (2014-09-21) 215 | ^^^^^^^^^^^^^^^^^^ 216 | 217 | - Added sync_trigger alembic helper function 218 | 219 | 220 | 0.5.0 (2014-03-19) 221 | ^^^^^^^^^^^^^^^^^^ 222 | 223 | - Python 3 support 224 | - Enhanced email token handling 225 | - New configuration option: remove_hyphens 226 | 227 | 228 | 0.4.5 (2013-10-22) 229 | ^^^^^^^^^^^^^^^^^^ 230 | 231 | - Updated validators dependency to 0.2.0 232 | 233 | 234 | 0.4.4 (2013-10-17) 235 | ^^^^^^^^^^^^^^^^^^ 236 | 237 | - Search query string parser now notices emails and leaves them as they are (same behavious as in PostgreSQL tsvector parser) 238 | 239 | 240 | 0.4.3 (2013-10-07) 241 | ^^^^^^^^^^^^^^^^^^ 242 | 243 | - Fixed index/trigger creation when multiple vectors attached to single class 244 | - Search vector without columns do not generate triggers anymore 245 | 246 | 247 | 0.4.2 (2013-10-07) 248 | ^^^^^^^^^^^^^^^^^^ 249 | 250 | - Fixed single table inheritance handling in define_triggers_and_indexes manager method. 251 | 252 | 253 | 0.4.1 (2013-10-04) 254 | ^^^^^^^^^^^^^^^^^^ 255 | 256 | - Fixed negation operator parsing 257 | 258 | 259 | 0.4.0 (2013-10-04) 260 | ^^^^^^^^^^^^^^^^^^ 261 | 262 | - Completely rewritten search API 263 | - Renamed SearchQueryMixin.search and main module search function's 'language' parameter to 'catalog' 264 | - Support for multiple search vectors per class 265 | 266 | 267 | 0.3.3 (2013-10-03) 268 | ^^^^^^^^^^^^^^^^^^ 269 | 270 | - Fixed support for numbers in parse_search_query 271 | 272 | 273 | 0.3.2 (2013-10-03) 274 | ^^^^^^^^^^^^^^^^^^ 275 | 276 | - Added support for hyphens between words 277 | 278 | 279 | 0.3.1 (2013-10-02) 280 | ^^^^^^^^^^^^^^^^^^ 281 | 282 | - Fixed parse_search_query to support nested parenthesis and negation operator 283 | 284 | 285 | 0.3.0 (2013-10-01) 286 | ^^^^^^^^^^^^^^^^^^ 287 | 288 | - Added better search query parsing capabilities (support for nested parenthesis, or operator and negation operator) 289 | 290 | 291 | 0.2.1 (2013-08-01) 292 | ^^^^^^^^^^^^^^^^^^ 293 | 294 | - Made psycopg dependency more permissive 295 | 296 | 297 | 0.2.0 (2013-08-01) 298 | ^^^^^^^^^^^^^^^^^^ 299 | 300 | - Added dependency to SQLAlchemy-Utils 301 | - Search vectors must be added manually to each class 302 | 303 | 304 | 0.1.8 (2013-07-30) 305 | ^^^^^^^^^^^^^^^^^^ 306 | 307 | - Fixed safe_search_terms single quote handling 308 | 309 | 310 | 0.1.7 (2013-05-22) 311 | ^^^^^^^^^^^^^^^^^^ 312 | 313 | - Language set explicitly on each query condition 314 | 315 | 316 | 0.1.6 (2013-04-17) 317 | ^^^^^^^^^^^^^^^^^^ 318 | 319 | - Fixed search function when using session based queries 320 | 321 | 322 | 0.1.5 (2013-04-03) 323 | ^^^^^^^^^^^^^^^^^^ 324 | 325 | - Added table name identifier quoting 326 | 327 | 328 | 0.1.4 (2013-01-30) 329 | ^^^^^^^^^^^^^^^^^^ 330 | 331 | - Fixed search_filter func when using empty or undefined search options 332 | 333 | 334 | 0.1.3 (2013-01-30) 335 | ^^^^^^^^^^^^^^^^^^ 336 | 337 | - Added support for custom language parameter in query search functions 338 | 339 | 340 | 0.1.2 (2013-01-30) 341 | ^^^^^^^^^^^^^^^^^^ 342 | 343 | - Added psycopg2 to requirements, fixed travis.yml 344 | 345 | 346 | 0.1.1 (2013-01-12) 347 | ^^^^^^^^^^^^^^^^^^ 348 | 349 | - safe_search_terms support for other than english catalogs 350 | 351 | 352 | 0.1.0 (2013-01-12) 353 | ^^^^^^^^^^^^^^^^^^ 354 | 355 | - Initial public release 356 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2012, Konsta Vesterinen 2 | 3 | All rights reserved. 4 | 5 | Redistribution and use in source and binary forms, with or without 6 | modification, are permitted provided that the following conditions are met: 7 | 8 | * Redistributions of source code must retain the above copyright notice, this 9 | list of conditions and the following disclaimer. 10 | 11 | * Redistributions in binary form must reproduce the above copyright notice, 12 | this list of conditions and the following disclaimer in the documentation 13 | and/or other materials provided with the distribution. 14 | 15 | * The names of the contributors may not be used to endorse or promote products 16 | derived from this software without specific prior written permission. 17 | 18 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND 19 | ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 20 | WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 21 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER BE LIABLE FOR ANY DIRECT, 22 | INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, 23 | BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 24 | DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF 25 | LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE 26 | OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF 27 | ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 | -------------------------------------------------------------------------------- /README.rst: -------------------------------------------------------------------------------- 1 | SQLAlchemy-Searchable 2 | ===================== 3 | 4 | |Version Status| |Downloads| 5 | 6 | Fulltext searchable models for SQLAlchemy. Only supports PostgreSQL 7 | 8 | 9 | Resources 10 | --------- 11 | 12 | - `Documentation `_ 13 | - `Issue Tracker `_ 14 | - `Code `_ 15 | 16 | 17 | .. |Version Status| image:: https://img.shields.io/pypi/v/SQLAlchemy-Searchable.svg 18 | :target: https://pypi.python.org/pypi/SQLAlchemy-Searchable/ 19 | .. |Downloads| image:: https://img.shields.io/pypi/dm/SQLAlchemy-Searchable.svg 20 | :target: https://pypi.python.org/pypi/SQLAlchemy-Searchable/ 21 | -------------------------------------------------------------------------------- /docs/Makefile: -------------------------------------------------------------------------------- 1 | # Makefile for Sphinx documentation 2 | # 3 | 4 | # You can set these variables from the command line. 5 | SPHINXOPTS = 6 | SPHINXBUILD = sphinx-build 7 | PAPER = 8 | BUILDDIR = _build 9 | 10 | # Internal variables. 11 | PAPEROPT_a4 = -D latex_paper_size=a4 12 | PAPEROPT_letter = -D latex_paper_size=letter 13 | ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . 14 | # the i18n builder cannot share the environment and doctrees with the others 15 | I18NSPHINXOPTS = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . 16 | 17 | .PHONY: help clean html dirhtml singlehtml pickle json htmlhelp qthelp devhelp epub latex latexpdf text man changes linkcheck doctest gettext 18 | 19 | help: 20 | @echo "Please use \`make ' where is one of" 21 | @echo " html to make standalone HTML files" 22 | @echo " dirhtml to make HTML files named index.html in directories" 23 | @echo " singlehtml to make a single large HTML file" 24 | @echo " pickle to make pickle files" 25 | @echo " json to make JSON files" 26 | @echo " htmlhelp to make HTML files and a HTML help project" 27 | @echo " qthelp to make HTML files and a qthelp project" 28 | @echo " devhelp to make HTML files and a Devhelp project" 29 | @echo " epub to make an epub" 30 | @echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter" 31 | @echo " latexpdf to make LaTeX files and run them through pdflatex" 32 | @echo " text to make text files" 33 | @echo " man to make manual pages" 34 | @echo " texinfo to make Texinfo files" 35 | @echo " info to make Texinfo files and run them through makeinfo" 36 | @echo " gettext to make PO message catalogs" 37 | @echo " changes to make an overview of all changed/added/deprecated items" 38 | @echo " linkcheck to check all external links for integrity" 39 | @echo " doctest to run all doctests embedded in the documentation (if enabled)" 40 | 41 | clean: 42 | -rm -rf $(BUILDDIR)/* 43 | 44 | html: 45 | $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html 46 | @echo 47 | @echo "Build finished. The HTML pages are in $(BUILDDIR)/html." 48 | 49 | dirhtml: 50 | $(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml 51 | @echo 52 | @echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml." 53 | 54 | singlehtml: 55 | $(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) $(BUILDDIR)/singlehtml 56 | @echo 57 | @echo "Build finished. The HTML page is in $(BUILDDIR)/singlehtml." 58 | 59 | pickle: 60 | $(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle 61 | @echo 62 | @echo "Build finished; now you can process the pickle files." 63 | 64 | json: 65 | $(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json 66 | @echo 67 | @echo "Build finished; now you can process the JSON files." 68 | 69 | htmlhelp: 70 | $(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp 71 | @echo 72 | @echo "Build finished; now you can run HTML Help Workshop with the" \ 73 | ".hhp project file in $(BUILDDIR)/htmlhelp." 74 | 75 | qthelp: 76 | $(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp 77 | @echo 78 | @echo "Build finished; now you can run "qcollectiongenerator" with the" \ 79 | ".qhcp project file in $(BUILDDIR)/qthelp, like this:" 80 | @echo "# qcollectiongenerator $(BUILDDIR)/qthelp/SQLAlchemy-Searchable.qhcp" 81 | @echo "To view the help file:" 82 | @echo "# assistant -collectionFile $(BUILDDIR)/qthelp/SQLAlchemy-Searchable.qhc" 83 | 84 | devhelp: 85 | $(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILDDIR)/devhelp 86 | @echo 87 | @echo "Build finished." 88 | @echo "To view the help file:" 89 | @echo "# mkdir -p $$HOME/.local/share/devhelp/SQLAlchemy-Searchable" 90 | @echo "# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/SQLAlchemy-Searchable" 91 | @echo "# devhelp" 92 | 93 | epub: 94 | $(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILDDIR)/epub 95 | @echo 96 | @echo "Build finished. The epub file is in $(BUILDDIR)/epub." 97 | 98 | latex: 99 | $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex 100 | @echo 101 | @echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex." 102 | @echo "Run \`make' in that directory to run these through (pdf)latex" \ 103 | "(use \`make latexpdf' here to do that automatically)." 104 | 105 | latexpdf: 106 | $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex 107 | @echo "Running LaTeX files through pdflatex..." 108 | $(MAKE) -C $(BUILDDIR)/latex all-pdf 109 | @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." 110 | 111 | text: 112 | $(SPHINXBUILD) -b text $(ALLSPHINXOPTS) $(BUILDDIR)/text 113 | @echo 114 | @echo "Build finished. The text files are in $(BUILDDIR)/text." 115 | 116 | man: 117 | $(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man 118 | @echo 119 | @echo "Build finished. The manual pages are in $(BUILDDIR)/man." 120 | 121 | texinfo: 122 | $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo 123 | @echo 124 | @echo "Build finished. The Texinfo files are in $(BUILDDIR)/texinfo." 125 | @echo "Run \`make' in that directory to run these through makeinfo" \ 126 | "(use \`make info' here to do that automatically)." 127 | 128 | info: 129 | $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo 130 | @echo "Running Texinfo files through makeinfo..." 131 | make -C $(BUILDDIR)/texinfo info 132 | @echo "makeinfo finished; the Info files are in $(BUILDDIR)/texinfo." 133 | 134 | gettext: 135 | $(SPHINXBUILD) -b gettext $(I18NSPHINXOPTS) $(BUILDDIR)/locale 136 | @echo 137 | @echo "Build finished. The message catalogs are in $(BUILDDIR)/locale." 138 | 139 | changes: 140 | $(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes 141 | @echo 142 | @echo "The overview file is in $(BUILDDIR)/changes." 143 | 144 | linkcheck: 145 | $(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck 146 | @echo 147 | @echo "Link check complete; look for any errors in the above output " \ 148 | "or in $(BUILDDIR)/linkcheck/output.txt." 149 | 150 | doctest: 151 | $(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest 152 | @echo "Testing of doctests in the sources finished, look at the " \ 153 | "results in $(BUILDDIR)/doctest/output.txt." 154 | -------------------------------------------------------------------------------- /docs/alembic_migrations.rst: -------------------------------------------------------------------------------- 1 | Alembic migrations 2 | ------------------ 3 | 4 | .. currentmodule:: sqlalchemy_searchable 5 | 6 | When making changes to your database schema, you have to ensure the associated 7 | search triggers and trigger functions get updated also. SQLAlchemy-Searchable 8 | offers two helper functions for this: :func:`sync_trigger` and 9 | :func:`drop_trigger`. 10 | 11 | .. autofunction:: sync_trigger 12 | .. autofunction:: drop_trigger 13 | -------------------------------------------------------------------------------- /docs/conf.py: -------------------------------------------------------------------------------- 1 | # SQLAlchemy-Searchable documentation build configuration file, created by 2 | # sphinx-quickstart on Fri Jan 11 14:55:47 2013. 3 | # 4 | # This file is execfile()d with the current directory set to its containing dir. 5 | # 6 | # Note that not all possible configuration values are present in this 7 | # autogenerated file. 8 | # 9 | # All configuration values have a default; values that are commented out 10 | # serve to show the default. 11 | 12 | import os 13 | import sys 14 | 15 | # If extensions (or modules to document with autodoc) are in another directory, 16 | # add these directories to sys.path here. If the directory is relative to the 17 | # documentation root, use os.path.abspath to make it absolute, like shown here. 18 | sys.path.insert(0, os.path.abspath("..")) 19 | from sqlalchemy_searchable import __version__ # noqa: E402 20 | 21 | # -- General configuration ----------------------------------------------------- 22 | 23 | # If your documentation needs a minimal Sphinx version, state it here. 24 | # needs_sphinx = '1.0' 25 | 26 | # Add any Sphinx extension module names here, as strings. They can be extensions 27 | # coming with Sphinx (named 'sphinx.ext.*') or your custom ones. 28 | extensions = [ 29 | "sphinx.ext.autodoc", 30 | "sphinx.ext.doctest", 31 | "sphinx.ext.intersphinx", 32 | "sphinx.ext.todo", 33 | "sphinx.ext.coverage", 34 | "sphinx.ext.ifconfig", 35 | "sphinx.ext.viewcode", 36 | ] 37 | 38 | # Add any paths that contain templates here, relative to this directory. 39 | templates_path = ["_templates"] 40 | 41 | # The suffix of source filenames. 42 | source_suffix = ".rst" 43 | 44 | # The encoding of source files. 45 | # source_encoding = 'utf-8-sig' 46 | 47 | # The master toctree document. 48 | master_doc = "index" 49 | 50 | # General information about the project. 51 | project = "SQLAlchemy-Searchable" 52 | copyright = "2013, Konsta Vesterinen" 53 | 54 | # The version info for the project you're documenting, acts as replacement for 55 | # |version| and |release|, also used in various other places throughout the 56 | # built documents. 57 | # 58 | # The short X.Y version. 59 | version = __version__ 60 | # The full version, including alpha/beta/rc tags. 61 | release = version 62 | 63 | # The language for content autogenerated by Sphinx. Refer to documentation 64 | # for a list of supported languages. 65 | # language = None 66 | 67 | # There are two options for replacing |today|: either, you set today to some 68 | # non-false value, then it is used: 69 | # today = '' 70 | # Else, today_fmt is used as the format for a strftime call. 71 | # today_fmt = '%B %d, %Y' 72 | 73 | # List of patterns, relative to source directory, that match files and 74 | # directories to ignore when looking for source files. 75 | exclude_patterns = ["_build"] 76 | 77 | # The reST default role (used for this markup: `text`) to use for all documents. 78 | # default_role = None 79 | 80 | # If true, '()' will be appended to :func: etc. cross-reference text. 81 | # add_function_parentheses = True 82 | 83 | # If true, the current module name will be prepended to all description 84 | # unit titles (such as .. function::). 85 | # add_module_names = True 86 | 87 | # If true, sectionauthor and moduleauthor directives will be shown in the 88 | # output. They are ignored by default. 89 | # show_authors = False 90 | 91 | # The name of the Pygments (syntax highlighting) style to use. 92 | pygments_style = "sphinx" 93 | 94 | # A list of ignored prefixes for module index sorting. 95 | # modindex_common_prefix = [] 96 | 97 | 98 | # -- Options for HTML output --------------------------------------------------- 99 | 100 | # The theme to use for HTML and HTML Help pages. See the documentation for 101 | # a list of builtin themes. 102 | html_theme = "furo" 103 | 104 | # Theme options are theme-specific and customize the look and feel of a theme 105 | # further. For a list of options available for each theme, see the 106 | # documentation. 107 | # html_theme_options = {} 108 | 109 | # Add any paths that contain custom themes here, relative to this directory. 110 | # html_theme_path = [] 111 | 112 | # The name for this set of Sphinx documents. If None, it defaults to 113 | # " v documentation". 114 | # html_title = None 115 | 116 | # A shorter title for the navigation bar. Default is the same as html_title. 117 | # html_short_title = None 118 | 119 | # The name of an image file (relative to this directory) to place at the top 120 | # of the sidebar. 121 | # html_logo = None 122 | 123 | # The name of an image file (within the static path) to use as favicon of the 124 | # docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 125 | # pixels large. 126 | # html_favicon = None 127 | 128 | # Add any paths that contain custom static files (such as style sheets) here, 129 | # relative to this directory. They are copied after the builtin static files, 130 | # so a file named "default.css" will overwrite the builtin "default.css". 131 | # html_static_path = ["_static"] 132 | 133 | # If not '', a 'Last updated on:' timestamp is inserted at every page bottom, 134 | # using the given strftime format. 135 | # html_last_updated_fmt = '%b %d, %Y' 136 | 137 | # If true, SmartyPants will be used to convert quotes and dashes to 138 | # typographically correct entities. 139 | # html_use_smartypants = True 140 | 141 | # Custom sidebar templates, maps document names to template names. 142 | # html_sidebars = {} 143 | 144 | # Additional templates that should be rendered to pages, maps page names to 145 | # template names. 146 | # html_additional_pages = {} 147 | 148 | # If false, no module index is generated. 149 | # html_domain_indices = True 150 | 151 | # If false, no index is generated. 152 | # html_use_index = True 153 | 154 | # If true, the index is split into individual pages for each letter. 155 | # html_split_index = False 156 | 157 | # If true, links to the reST sources are added to the pages. 158 | # html_show_sourcelink = True 159 | 160 | # If true, "Created using Sphinx" is shown in the HTML footer. Default is True. 161 | # html_show_sphinx = True 162 | 163 | # If true, "(C) Copyright ..." is shown in the HTML footer. Default is True. 164 | # html_show_copyright = True 165 | 166 | # If true, an OpenSearch description file will be output, and all pages will 167 | # contain a tag referring to it. The value of this option must be the 168 | # base URL from which the finished HTML is served. 169 | # html_use_opensearch = '' 170 | 171 | # This is the file name suffix for HTML files (e.g. ".xhtml"). 172 | # html_file_suffix = None 173 | 174 | # Output file base name for HTML help builder. 175 | htmlhelp_basename = "SQLAlchemy-Searchabledoc" 176 | 177 | 178 | # -- Options for LaTeX output -------------------------------------------------- 179 | 180 | latex_elements = { 181 | # The paper size ('letterpaper' or 'a4paper'). 182 | #'papersize': 'letterpaper', 183 | # The font size ('10pt', '11pt' or '12pt'). 184 | #'pointsize': '10pt', 185 | # Additional stuff for the LaTeX preamble. 186 | #'preamble': '', 187 | } 188 | 189 | # Grouping the document tree into LaTeX files. List of tuples 190 | # (source start file, target name, title, author, documentclass [howto/manual]). 191 | latex_documents = [ 192 | ( 193 | "index", 194 | "SQLAlchemy-Searchable.tex", 195 | "SQLAlchemy-Searchable Documentation", 196 | "Konsta Vesterinen", 197 | "manual", 198 | ), 199 | ] 200 | 201 | # The name of an image file (relative to this directory) to place at the top of 202 | # the title page. 203 | # latex_logo = None 204 | 205 | # For "manual" documents, if this is true, then toplevel headings are parts, 206 | # not chapters. 207 | # latex_use_parts = False 208 | 209 | # If true, show page references after internal links. 210 | # latex_show_pagerefs = False 211 | 212 | # If true, show URL addresses after external links. 213 | # latex_show_urls = False 214 | 215 | # Documents to append as an appendix to all manuals. 216 | # latex_appendices = [] 217 | 218 | # If false, no module index is generated. 219 | # latex_domain_indices = True 220 | 221 | 222 | # -- Options for manual page output -------------------------------------------- 223 | 224 | # One entry per manual page. List of tuples 225 | # (source start file, name, description, authors, manual section). 226 | man_pages = [ 227 | ( 228 | "index", 229 | "sqlalchemy-searchable", 230 | "SQLAlchemy-Searchable Documentation", 231 | ["Konsta Vesterinen"], 232 | 1, 233 | ) 234 | ] 235 | 236 | # If true, show URL addresses after external links. 237 | # man_show_urls = False 238 | 239 | 240 | # -- Options for Texinfo output ------------------------------------------------ 241 | 242 | # Grouping the document tree into Texinfo files. List of tuples 243 | # (source start file, target name, title, author, 244 | # dir menu entry, description, category) 245 | texinfo_documents = [ 246 | ( 247 | "index", 248 | "SQLAlchemy-Searchable", 249 | "SQLAlchemy-Searchable Documentation", 250 | "Konsta Vesterinen", 251 | "SQLAlchemy-Searchable", 252 | "One line description of project.", 253 | "Miscellaneous", 254 | ), 255 | ] 256 | 257 | # Documents to append as an appendix to all manuals. 258 | # texinfo_appendices = [] 259 | 260 | # If false, no module index is generated. 261 | # texinfo_domain_indices = True 262 | 263 | # How to display URL addresses: 'footnote', 'no', or 'inline'. 264 | # texinfo_show_urls = 'footnote' 265 | 266 | 267 | # Example configuration for intersphinx: refer to the Python standard library. 268 | intersphinx_mapping = { 269 | "python": ("https://docs.python.org/", None), 270 | "sqlalchemy": ("https://docs.sqlalchemy.org/", None), 271 | "sqlalchemy_utils": ("https://sqlalchemy-utils.readthedocs.io/en/latest/", None), 272 | } 273 | -------------------------------------------------------------------------------- /docs/configuration.rst: -------------------------------------------------------------------------------- 1 | Configuration 2 | ============= 3 | 4 | .. currentmodule:: sqlalchemy_searchable 5 | 6 | SQLAlchemy-Searchable provides a number of customization options for the automatically generated 7 | search trigger, index and search vector columns. 8 | 9 | Global configuration options 10 | ---------------------------- 11 | 12 | The following configuration options can be defined globally by passing them to 13 | :func:`make_searchable` function: 14 | 15 | ``search_trigger_name`` 16 | Defines the name of the search database trigger. The default naming 17 | convention is ``"{table}_{column}_trigger"``. 18 | 19 | ``search_trigger_function_name`` 20 | Defines the name of the database search vector updating function. The 21 | default naming convention is ``{table}_{column}_update``. 22 | 23 | ``regconfig`` 24 | This is the PostgreSQL text search configuration that determines the 25 | language configuration used for searching. The default setting is 26 | ``"pg_catalog.english"``. 27 | 28 | Here's an example of how to leverage these options:: 29 | 30 | make_searchable(Base.metadata, options={"regconfig": "pg_catalog.finnish"}) 31 | 32 | Changing catalog for search vector 33 | ---------------------------------- 34 | 35 | In some cases, you might want to switch from the default language configuration 36 | to another language for your search vector. You can achieve this by providing 37 | the ``regconfig`` parameter for the 38 | :class:`~sqlalchemy_utils.types.ts_vector.TSVectorType`. In the following 39 | example, we use Finnish instead of the default English one:: 40 | 41 | class Article(Base): 42 | __tablename__ = "article" 43 | 44 | name = sa.Column(sa.Text(255)) 45 | search_vector = TSVectorType("name", regconfig="pg_catalog.finnish") 46 | 47 | Weighting search results 48 | ------------------------ 49 | 50 | To further refine your search results, PostgreSQL's `term weighting system`_ 51 | (ranging from A to D) can be applied. This example demonstrates how to 52 | prioritize terms found in the article title over those in the content:: 53 | 54 | class Article(Base): 55 | __tablename__ = "article" 56 | 57 | id = sa.Column(sa.Integer, primary_key=True) 58 | title = sa.Column(sa.String(255)) 59 | content = sa.Column(sa.Text) 60 | search_vector = sa.Column( 61 | TSVectorType("title", "content", weights={"title": "A", "content": "B"}) 62 | ) 63 | 64 | Remember, when working with weighted search terms, you need to conduct your 65 | searches using the ``sort=True`` option:: 66 | 67 | query = search(sa.select(Article), "search text", sort=True) 68 | 69 | .. _term weighting system: http://www.postgresql.org/docs/current/static/textsearch-controls.html#TEXTSEARCH-PARSING-DOCUMENTS 70 | 71 | Multiple search vectors per class 72 | --------------------------------- 73 | 74 | In cases where a model requires multiple search vectors, SQLAlchemy-Searchable 75 | has you covered. Here's how you can set up multiple search vectors for an 76 | ``Article`` class:: 77 | 78 | class Article(Base): 79 | __tablename__ = "article" 80 | 81 | id = sa.Column(sa.Integer, primary_key=True) 82 | name = sa.Column(sa.String(255)) 83 | content = sa.Column(sa.Text) 84 | description = sa.Column(sa.Text) 85 | simple_search_vector = sa.Column(TSVectorType("name")) 86 | 87 | fat_search_vector = sa.Column(TSVectorType("name", "content", "description")) 88 | 89 | You can then choose which search vector to use when querying:: 90 | 91 | query = search(sa.select(Article), "first", vector=Article.fat_search_vector) 92 | 93 | Combined search vectors 94 | ----------------------- 95 | 96 | Sometimes you may want to search from multiple tables at the same time. This can 97 | be achieved using combined search vectors. Consider the following model 98 | definition where each article has one category:: 99 | 100 | import sqlalchemy as sa 101 | from sqlalchemy.orm import declarative_base 102 | from sqlalchemy_utils.types import TSVectorType 103 | 104 | Base = declarative_base() 105 | 106 | 107 | class Category(Base): 108 | __tablename__ = "category" 109 | 110 | id = sa.Column(sa.Integer, primary_key=True) 111 | name = sa.Column(sa.String(255)) 112 | search_vector = sa.Column(TSVectorType("name")) 113 | 114 | 115 | class Article(Base): 116 | __tablename__ = "article" 117 | 118 | id = sa.Column(sa.Integer, primary_key=True) 119 | name = sa.Column(sa.String(255)) 120 | content = sa.Column(sa.Text) 121 | search_vector = sa.Column(TSVectorType("name", "content")) 122 | category_id = sa.Column(sa.Integer, sa.ForeignKey(Category.id)) 123 | category = sa.orm.relationship(Category) 124 | 125 | Now consider a situation where we want to find all articles where either article 126 | content or name or category name contains the word "matrix". This can be 127 | achieved as follows:: 128 | 129 | combined_search_vector = Article.search_vector | Category.search_vector 130 | query = search( 131 | sa.select(Article).join(Category), "matrix", vector=combined_search_vector 132 | ) 133 | 134 | This query becomes a little more complex when using left joins. Then, you have 135 | to take into account situations where ``Category.search_vector`` might be 136 | ``None`` using the ``coalesce`` function:: 137 | 138 | combined_search_vector = Article.search_vector | sa.func.coalesce( 139 | Category.search_vector, "" 140 | ) 141 | -------------------------------------------------------------------------------- /docs/index.rst: -------------------------------------------------------------------------------- 1 | SQLAlchemy-Searchable 2 | ===================== 3 | 4 | 5 | SQLAlchemy-Searchable provides `full text search`_ capabilities for SQLAlchemy_ models. Currently, it only supports PostgreSQL_. 6 | 7 | 8 | .. toctree:: 9 | :maxdepth: 2 10 | 11 | quickstart 12 | search_query_parser 13 | configuration 14 | vectorizers 15 | alembic_migrations 16 | integrations 17 | 18 | 19 | .. _`full text search`: https://en.wikipedia.org/wiki/Full_text_search 20 | .. _SQLAlchemy: https://www.sqlalchemy.org/ 21 | .. _PostgreSQL: https://www.postgresql.org/ 22 | 23 | 24 | -------------------------------------------------------------------------------- /docs/integrations.rst: -------------------------------------------------------------------------------- 1 | Flask-SQLAlchemy integration 2 | ---------------------------- 3 | 4 | .. warning:: 5 | The query interface is considered legacy in SQLAlchemy. Prefer using 6 | ``session.execute(search(...))`` instead. 7 | 8 | SQLAlchemy-Searchable can be integrated into Flask-SQLAlchemy using 9 | ``SearchQueryMixin`` class:: 10 | 11 | from flask import Flask 12 | from flask_sqlalchemy import SQLAlchemy 13 | from flask_sqlalchemy.query import Query 14 | from sqlalchemy_utils.types import TSVectorType 15 | from sqlalchemy_searchable import SearchQueryMixin, make_searchable 16 | 17 | app = Flask(__name__) 18 | db = SQLAlchemy(app) 19 | 20 | make_searchable(db.metadata) 21 | 22 | 23 | class ArticleQuery(Query, SearchQueryMixin): 24 | pass 25 | 26 | 27 | class Article(db.Model): 28 | query_class = ArticleQuery 29 | __tablename__ = "article" 30 | 31 | id = db.Column(db.Integer, primary_key=True) 32 | name = db.Column(db.String(255)) 33 | content = db.Column(db.Text) 34 | search_vector = db.Column(TSVectorType("name", "content")) 35 | 36 | 37 | db.configure_mappers() # very important! 38 | 39 | with app.app_context(): 40 | db.create_all() 41 | 42 | The ``SearchQueryMixin`` provides a ``search`` method to ``ArticleQuery``. You 43 | can chain calls just like when using query filter calls. Here we search for 44 | first five articles that contain the word "Finland":: 45 | 46 | Article.query.search("Finland").limit(5).all() 47 | -------------------------------------------------------------------------------- /docs/make.bat: -------------------------------------------------------------------------------- 1 | @ECHO OFF 2 | 3 | REM Command file for Sphinx documentation 4 | 5 | if "%SPHINXBUILD%" == "" ( 6 | set SPHINXBUILD=sphinx-build 7 | ) 8 | set BUILDDIR=_build 9 | set ALLSPHINXOPTS=-d %BUILDDIR%/doctrees %SPHINXOPTS% . 10 | set I18NSPHINXOPTS=%SPHINXOPTS% . 11 | if NOT "%PAPER%" == "" ( 12 | set ALLSPHINXOPTS=-D latex_paper_size=%PAPER% %ALLSPHINXOPTS% 13 | set I18NSPHINXOPTS=-D latex_paper_size=%PAPER% %I18NSPHINXOPTS% 14 | ) 15 | 16 | if "%1" == "" goto help 17 | 18 | if "%1" == "help" ( 19 | :help 20 | echo.Please use `make ^` where ^ is one of 21 | echo. html to make standalone HTML files 22 | echo. dirhtml to make HTML files named index.html in directories 23 | echo. singlehtml to make a single large HTML file 24 | echo. pickle to make pickle files 25 | echo. json to make JSON files 26 | echo. htmlhelp to make HTML files and a HTML help project 27 | echo. qthelp to make HTML files and a qthelp project 28 | echo. devhelp to make HTML files and a Devhelp project 29 | echo. epub to make an epub 30 | echo. latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter 31 | echo. text to make text files 32 | echo. man to make manual pages 33 | echo. texinfo to make Texinfo files 34 | echo. gettext to make PO message catalogs 35 | echo. changes to make an overview over all changed/added/deprecated items 36 | echo. linkcheck to check all external links for integrity 37 | echo. doctest to run all doctests embedded in the documentation if enabled 38 | goto end 39 | ) 40 | 41 | if "%1" == "clean" ( 42 | for /d %%i in (%BUILDDIR%\*) do rmdir /q /s %%i 43 | del /q /s %BUILDDIR%\* 44 | goto end 45 | ) 46 | 47 | if "%1" == "html" ( 48 | %SPHINXBUILD% -b html %ALLSPHINXOPTS% %BUILDDIR%/html 49 | if errorlevel 1 exit /b 1 50 | echo. 51 | echo.Build finished. The HTML pages are in %BUILDDIR%/html. 52 | goto end 53 | ) 54 | 55 | if "%1" == "dirhtml" ( 56 | %SPHINXBUILD% -b dirhtml %ALLSPHINXOPTS% %BUILDDIR%/dirhtml 57 | if errorlevel 1 exit /b 1 58 | echo. 59 | echo.Build finished. The HTML pages are in %BUILDDIR%/dirhtml. 60 | goto end 61 | ) 62 | 63 | if "%1" == "singlehtml" ( 64 | %SPHINXBUILD% -b singlehtml %ALLSPHINXOPTS% %BUILDDIR%/singlehtml 65 | if errorlevel 1 exit /b 1 66 | echo. 67 | echo.Build finished. The HTML pages are in %BUILDDIR%/singlehtml. 68 | goto end 69 | ) 70 | 71 | if "%1" == "pickle" ( 72 | %SPHINXBUILD% -b pickle %ALLSPHINXOPTS% %BUILDDIR%/pickle 73 | if errorlevel 1 exit /b 1 74 | echo. 75 | echo.Build finished; now you can process the pickle files. 76 | goto end 77 | ) 78 | 79 | if "%1" == "json" ( 80 | %SPHINXBUILD% -b json %ALLSPHINXOPTS% %BUILDDIR%/json 81 | if errorlevel 1 exit /b 1 82 | echo. 83 | echo.Build finished; now you can process the JSON files. 84 | goto end 85 | ) 86 | 87 | if "%1" == "htmlhelp" ( 88 | %SPHINXBUILD% -b htmlhelp %ALLSPHINXOPTS% %BUILDDIR%/htmlhelp 89 | if errorlevel 1 exit /b 1 90 | echo. 91 | echo.Build finished; now you can run HTML Help Workshop with the ^ 92 | .hhp project file in %BUILDDIR%/htmlhelp. 93 | goto end 94 | ) 95 | 96 | if "%1" == "qthelp" ( 97 | %SPHINXBUILD% -b qthelp %ALLSPHINXOPTS% %BUILDDIR%/qthelp 98 | if errorlevel 1 exit /b 1 99 | echo. 100 | echo.Build finished; now you can run "qcollectiongenerator" with the ^ 101 | .qhcp project file in %BUILDDIR%/qthelp, like this: 102 | echo.^> qcollectiongenerator %BUILDDIR%\qthelp\SQLAlchemy-Searchable.qhcp 103 | echo.To view the help file: 104 | echo.^> assistant -collectionFile %BUILDDIR%\qthelp\SQLAlchemy-Searchable.ghc 105 | goto end 106 | ) 107 | 108 | if "%1" == "devhelp" ( 109 | %SPHINXBUILD% -b devhelp %ALLSPHINXOPTS% %BUILDDIR%/devhelp 110 | if errorlevel 1 exit /b 1 111 | echo. 112 | echo.Build finished. 113 | goto end 114 | ) 115 | 116 | if "%1" == "epub" ( 117 | %SPHINXBUILD% -b epub %ALLSPHINXOPTS% %BUILDDIR%/epub 118 | if errorlevel 1 exit /b 1 119 | echo. 120 | echo.Build finished. The epub file is in %BUILDDIR%/epub. 121 | goto end 122 | ) 123 | 124 | if "%1" == "latex" ( 125 | %SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex 126 | if errorlevel 1 exit /b 1 127 | echo. 128 | echo.Build finished; the LaTeX files are in %BUILDDIR%/latex. 129 | goto end 130 | ) 131 | 132 | if "%1" == "text" ( 133 | %SPHINXBUILD% -b text %ALLSPHINXOPTS% %BUILDDIR%/text 134 | if errorlevel 1 exit /b 1 135 | echo. 136 | echo.Build finished. The text files are in %BUILDDIR%/text. 137 | goto end 138 | ) 139 | 140 | if "%1" == "man" ( 141 | %SPHINXBUILD% -b man %ALLSPHINXOPTS% %BUILDDIR%/man 142 | if errorlevel 1 exit /b 1 143 | echo. 144 | echo.Build finished. The manual pages are in %BUILDDIR%/man. 145 | goto end 146 | ) 147 | 148 | if "%1" == "texinfo" ( 149 | %SPHINXBUILD% -b texinfo %ALLSPHINXOPTS% %BUILDDIR%/texinfo 150 | if errorlevel 1 exit /b 1 151 | echo. 152 | echo.Build finished. The Texinfo files are in %BUILDDIR%/texinfo. 153 | goto end 154 | ) 155 | 156 | if "%1" == "gettext" ( 157 | %SPHINXBUILD% -b gettext %I18NSPHINXOPTS% %BUILDDIR%/locale 158 | if errorlevel 1 exit /b 1 159 | echo. 160 | echo.Build finished. The message catalogs are in %BUILDDIR%/locale. 161 | goto end 162 | ) 163 | 164 | if "%1" == "changes" ( 165 | %SPHINXBUILD% -b changes %ALLSPHINXOPTS% %BUILDDIR%/changes 166 | if errorlevel 1 exit /b 1 167 | echo. 168 | echo.The overview file is in %BUILDDIR%/changes. 169 | goto end 170 | ) 171 | 172 | if "%1" == "linkcheck" ( 173 | %SPHINXBUILD% -b linkcheck %ALLSPHINXOPTS% %BUILDDIR%/linkcheck 174 | if errorlevel 1 exit /b 1 175 | echo. 176 | echo.Link check complete; look for any errors in the above output ^ 177 | or in %BUILDDIR%/linkcheck/output.txt. 178 | goto end 179 | ) 180 | 181 | if "%1" == "doctest" ( 182 | %SPHINXBUILD% -b doctest %ALLSPHINXOPTS% %BUILDDIR%/doctest 183 | if errorlevel 1 exit /b 1 184 | echo. 185 | echo.Testing of doctests in the sources finished, look at the ^ 186 | results in %BUILDDIR%/doctest/output.txt. 187 | goto end 188 | ) 189 | 190 | :end 191 | -------------------------------------------------------------------------------- /docs/quickstart.rst: -------------------------------------------------------------------------------- 1 | Quick start 2 | =========== 3 | 4 | .. currentmodule:: sqlalchemy_searchable 5 | 6 | Installation 7 | ------------ 8 | 9 | SQLAlchemy-Searchable is available on PyPI_. It can be installed using pip_:: 10 | 11 | pip install SQLAlchemy-Searchable 12 | 13 | SQLAlchemy-Searchable requires Python 3.8 or newer, either the cPython or PyPy 14 | implementation. 15 | 16 | .. _PyPI: https://pypi.python.org/pypi/SQLAlchemy-Searchable 17 | .. _pip: https://pip.pypa.io/ 18 | 19 | Configuration 20 | ------------- 21 | 22 | The first step to enable full-text search functionality in your app is to 23 | configure SQLAlchemy-Searchable using :func:`make_searchable` function by 24 | passing it your declarative base class:: 25 | 26 | from sqlalchemy.orm import declarative_base 27 | from sqlalchemy_searchable import make_searchable 28 | 29 | Base = declarative_base() 30 | make_searchable(Base.metadata) 31 | 32 | Define models 33 | ------------- 34 | 35 | Then, add a search vector column to your model and specify which columns you want to 36 | be included in the full-text search. Here's an example using an ``Article`` 37 | model:: 38 | 39 | from sqlalchemy import Column, Integer, String, Text 40 | from sqlalchemy_utils.types import TSVectorType 41 | 42 | class Article(Base): 43 | __tablename__ = "article" 44 | 45 | id = Column(Integer, primary_key=True) 46 | name = Column(String(255)) 47 | content = Column(Text) 48 | search_vector = Column(TSVectorType("name", "content")) 49 | 50 | The search vector is a special column of 51 | :class:`~sqlalchemy_utils.types.ts_vector.TSVectorType` data type that is 52 | optimized for text search. Here, we want the ``name`` and ``content`` columns to 53 | be full-text indexed, which we have indicated by giving them as arguments to the 54 | :class:`~sqlalchemy_utils.types.ts_vector.TSVectorType` constructor. 55 | 56 | Create and populate the tables 57 | ------------------------------ 58 | 59 | Now, let's create the tables and add some sample data. Before creating the 60 | tables, make sure to call :func:`sqlalchemy.orm.configure_mappers` to ensure 61 | that mappers have been configured for the models:: 62 | 63 | from sqlalchemy import create_engine 64 | from sqlalchemy.orm import configure_mappers, Session 65 | 66 | engine = create_engine("postgresql://localhost/sqlalchemy_searchable_test") 67 | configure_mappers() # IMPORTANT! 68 | Base.metadata.create_all(engine) 69 | 70 | session = Session(engine) 71 | 72 | article1 = Article(name="First article", content="This is the first article") 73 | article2 = Article(name="Second article", content="This is the second article") 74 | 75 | session.add(article1) 76 | session.add(article2) 77 | session.commit() 78 | 79 | Performing searches 80 | ------------------- 81 | 82 | After we've created the articles and populated the database, we can now perform 83 | full-text searches on them using the :func:`~sqlalchemy_searchable.search` 84 | function:: 85 | 86 | from sqlalchemy import select 87 | from sqlalchemy_searchable import search 88 | 89 | query = search(select(Article), "first") 90 | article = session.scalars(query).first() 91 | print(article.name) 92 | # Output: First article 93 | 94 | API 95 | --- 96 | 97 | .. autofunction:: make_searchable 98 | .. autofunction:: search 99 | 100 | -------------------------------------------------------------------------------- /docs/requirements.in: -------------------------------------------------------------------------------- 1 | furo 2 | sphinx 3 | -------------------------------------------------------------------------------- /docs/requirements.txt: -------------------------------------------------------------------------------- 1 | # 2 | # This file is autogenerated by pip-compile with Python 3.9 3 | # by the following command: 4 | # 5 | # pip-compile 6 | # 7 | alabaster==0.7.13 8 | # via sphinx 9 | babel==2.12.1 10 | # via sphinx 11 | beautifulsoup4==4.12.2 12 | # via furo 13 | certifi==2024.7.4 14 | # via requests 15 | charset-normalizer==3.2.0 16 | # via requests 17 | docutils==0.20.1 18 | # via sphinx 19 | furo==2023.7.26 20 | # via -r requirements.in 21 | idna==3.7 22 | # via requests 23 | imagesize==1.4.1 24 | # via sphinx 25 | importlib-metadata==7.1.0 26 | # via sphinx 27 | jinja2==3.1.6 28 | # via sphinx 29 | markupsafe==2.1.3 30 | # via jinja2 31 | packaging==23.1 32 | # via sphinx 33 | pygments==2.16.1 34 | # via 35 | # furo 36 | # sphinx 37 | requests==2.32.0 38 | # via sphinx 39 | snowballstemmer==2.2.0 40 | # via sphinx 41 | soupsieve==2.4.1 42 | # via beautifulsoup4 43 | sphinx==7.1.2 44 | # via 45 | # -r requirements.in 46 | # furo 47 | # sphinx-basic-ng 48 | # sphinxcontrib-applehelp 49 | # sphinxcontrib-devhelp 50 | # sphinxcontrib-htmlhelp 51 | # sphinxcontrib-qthelp 52 | # sphinxcontrib-serializinghtml 53 | sphinx-basic-ng==1.0.0b2 54 | # via furo 55 | sphinxcontrib-applehelp==1.0.6 56 | # via sphinx 57 | sphinxcontrib-devhelp==1.0.4 58 | # via sphinx 59 | sphinxcontrib-htmlhelp==2.0.3 60 | # via sphinx 61 | sphinxcontrib-jsmath==1.0.1 62 | # via sphinx 63 | sphinxcontrib-qthelp==1.0.5 64 | # via sphinx 65 | sphinxcontrib-serializinghtml==1.1.7 66 | # via sphinx 67 | urllib3==2.2.2 68 | # via requests 69 | zipp==3.19.1 70 | # via importlib-metadata 71 | -------------------------------------------------------------------------------- /docs/search_query_parser.rst: -------------------------------------------------------------------------------- 1 | Search query parser 2 | =================== 3 | 4 | SQLAlchemy-Searchable includes a search query parser that enables the conversion 5 | of human-readable search queries into PostgreSQL search query syntax. 6 | 7 | AND operator 8 | ------------ 9 | 10 | The search query parser treats search terms as if they are connected with an 11 | implied AND operator. To search for articles containing both "star" and "wars", 12 | simply use the query "star wars":: 13 | 14 | query = search(query, 'star wars') 15 | 16 | OR operator 17 | ------------ 18 | 19 | The OR operator in the search query parser allows you to broaden your search to 20 | include results that contain any of the specified terms. To search for articles 21 | containing either "star" or "wars", you can utilize the OR operator as follows:: 22 | 23 | query = search(query, 'star or wars') 24 | 25 | Negation operator 26 | ----------------- 27 | 28 | Th search query parser supports excluding words from the search. Enter ``-`` in 29 | front of the word you want to leave out. To search for articles containing 30 | "star" but not "wars", you can use the query "star -wars":: 31 | 32 | query = search(query, 'star -wars') 33 | 34 | Phrase searching 35 | ---------------- 36 | 37 | If you need to search for a specific phrase, enclose the phrase in double quotes:: 38 | 39 | query = search(query, '"star wars"') 40 | 41 | Internals 42 | --------- 43 | 44 | If you wish to use only the query parser, this can be achieved by invoking the 45 | ``parse_websearch`` function. This function parses human readable search query into 46 | PostgreSQL ``tsquery`` format:: 47 | 48 | >>> session.execute("SELECT parse_websearch('(star wars) or luke')").scalar() 49 | '(star:* & wars:*) | luke:*' 50 | -------------------------------------------------------------------------------- /docs/vectorizers.rst: -------------------------------------------------------------------------------- 1 | Vectorizers 2 | =========== 3 | 4 | Vectorizers provide means for turning various column types and columns into fulltext 5 | search vector. While PostgreSQL inherently knows how to vectorize string columns, 6 | situations arise where additional vectorization rules are neede. This section outlines 7 | the process of creating and utilizing vectorization rules for both specific column 8 | instances and column types. 9 | 10 | Type vectorizers 11 | ---------------- 12 | 13 | By default, PostgreSQL can only directly vectorize string columns. However, scenarios 14 | may arise where vectorizing non-string columns becomes essential. For instance, when 15 | dealing with an :class:`~sqlalchemy.dialects.postgresql.HSTORE` column within your model 16 | that requires fulltext indexing, a dedicated vectorization rule must be defined. 17 | 18 | To establish a vectorization rule, use the :data:`~sqlalchemy_searchable.vectorizer` 19 | decorator. The subsequent example demonstrates how to apply a vectorization rule to the 20 | values within all :class:`~sqlalchemy.dialects.postgresql.HSTORE`-typed columns present 21 | in your models:: 22 | 23 | from sqlalchemy import cast, func, Text 24 | from sqlalchemy.dialects.postgresql import HSTORE 25 | from sqlalchemy_searchable import vectorizer 26 | 27 | 28 | @vectorizer(HSTORE) 29 | def hstore_vectorizer(column): 30 | return cast(func.avals(column), Text) 31 | 32 | The expression returned by the vectorizer is then employed for all fulltext indexed 33 | columns of type :class:`~sqlalchemy.dialects.postgresql.HSTORE`. Consider the following 34 | model as an illustration:: 35 | 36 | from sqlalchemy import Column, Integer 37 | from sqlalchemy_utils import TSVectorType 38 | 39 | 40 | class Article(Base): 41 | __tablename__ = 'article' 42 | 43 | id = Column(Integer, primary_key=True, autoincrement=True) 44 | name_translations = Column(HSTORE) 45 | content_translations = Column(HSTORE) 46 | search_vector = Column( 47 | TSVectorType( 48 | "name_translations", 49 | "content_translations", 50 | ) 51 | ) 52 | 53 | In this scenario, SQLAlchemy-Searchable would create the following search trigger for 54 | the model using the default configuration: 55 | 56 | .. code-block:: postgres 57 | 58 | CREATE FUNCTION 59 | article_search_vector_update() RETURNS TRIGGER AS $$ 60 | BEGIN 61 | NEW.search_vector = to_tsvector( 62 | 'pg_catalog.english', 63 | coalesce(CAST(avals(NEW.name_translations) AS TEXT), '') 64 | ) || to_tsvector( 65 | 'pg_catalog.english', 66 | coalesce(CAST(avals(NEW.content_translations) AS TEXT), '') 67 | ); 68 | RETURN NEW; 69 | END 70 | $$ LANGUAGE 'plpgsql'; 71 | 72 | 73 | Column vectorizers 74 | ------------------ 75 | 76 | Sometimes you may want to set special vectorizer only for specific column. This 77 | can be achieved as follows:: 78 | 79 | class Article(Base): 80 | __tablename__ = "article" 81 | 82 | id = Column(Integer, primary_key=True, autoincrement=True) 83 | name_translations = Column(HSTORE) 84 | search_vector = Column(TSVectorType("name_translations")) 85 | 86 | 87 | @vectorizer(Article.name_translations) 88 | def name_vectorizer(column): 89 | return cast(func.avals(column), Text) 90 | 91 | 92 | .. note:: 93 | 94 | Column vectorizers always have precedence over type vectorizers. 95 | 96 | API 97 | ^^^ 98 | 99 | .. currentmodule:: sqlalchemy_searchable 100 | .. autodata:: vectorizer 101 | .. autoclass:: Vectorizer 102 | :members: 103 | :special-members: __call__ 104 | -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [build-system] 2 | requires = ["hatchling"] 3 | build-backend = "hatchling.build" 4 | 5 | [project] 6 | name = "sqlalchemy-searchable" 7 | dynamic = ["version"] 8 | description = "Provides fulltext search capabilities for declarative SQLAlchemy models." 9 | readme = "README.rst" 10 | license = "bsd-3-clause" 11 | requires-python = ">=3.8" 12 | authors = [ 13 | { name = "Konsta Vesterinen", email = "konsta@fastmonkeys.com" }, 14 | ] 15 | classifiers = [ 16 | "Environment :: Web Environment", 17 | "Intended Audience :: Developers", 18 | "License :: OSI Approved :: BSD License", 19 | "Operating System :: OS Independent", 20 | "Programming Language :: Python", 21 | "Programming Language :: Python :: 3", 22 | "Programming Language :: Python :: 3.8", 23 | "Programming Language :: Python :: 3.9", 24 | "Programming Language :: Python :: 3.10", 25 | "Programming Language :: Python :: 3.11", 26 | "Programming Language :: Python :: 3.12", 27 | "Topic :: Internet :: WWW/HTTP :: Dynamic Content", 28 | "Topic :: Software Development :: Libraries :: Python Modules", 29 | ] 30 | dependencies = [ 31 | "SQLAlchemy-Utils>=0.40.0", 32 | "SQLAlchemy>=1.4", 33 | ] 34 | 35 | [project.urls] 36 | Code = "https://github.com/falcony-io/sqlalchemy-searchable" 37 | Documentation = "https://sqlalchemy-searchable.readthedocs.io/" 38 | "Issue Tracker" = "http://github.com/falcony-io/sqlalchemy-searchable/issues" 39 | 40 | [tool.hatch.version] 41 | path = "sqlalchemy_searchable/__init__.py" 42 | 43 | [tool.hatch.build.targets.sdist] 44 | include = [ 45 | "/CHANGES.rst", 46 | "/docs", 47 | "/sqlalchemy_searchable", 48 | "/tests", 49 | ] 50 | exclude = [ 51 | "/docs/_build", 52 | ] 53 | 54 | [tool.ruff.lint] 55 | select = ["E", "F", "I", "UP"] 56 | 57 | [tool.ruff.lint.isort] 58 | known-first-party = ["sqlalchemy_searchable", "tests"] 59 | order-by-type = false 60 | -------------------------------------------------------------------------------- /sqlalchemy_searchable/__init__.py: -------------------------------------------------------------------------------- 1 | import os 2 | from functools import reduce 3 | 4 | import sqlalchemy as sa 5 | from sqlalchemy import event 6 | from sqlalchemy.ext.compiler import compiles 7 | from sqlalchemy.schema import DDL, DDLElement 8 | from sqlalchemy.sql.expression import Executable 9 | from sqlalchemy_utils import TSVectorType 10 | 11 | from .vectorizers import Vectorizer 12 | 13 | __version__ = "2.1.0" 14 | 15 | 16 | vectorizer = Vectorizer() 17 | """ 18 | An instance of :class:`Vectorizer` that keeps a track of the registered vectorizers. Use 19 | this as a decorator to register a function as a vectorizer. 20 | """ 21 | 22 | 23 | class SearchQueryMixin: 24 | def search(self, search_query, vector=None, regconfig=None, sort=False): 25 | """ 26 | Search given query with full text search. 27 | 28 | :param search_query: the search query 29 | :param vector: search vector to use 30 | :param regconfig: postgresql regconfig to be used 31 | :param sort: order results by relevance (quality of hit) 32 | """ 33 | return search(self, search_query, vector=vector, regconfig=regconfig, sort=sort) 34 | 35 | 36 | def inspect_search_vectors(entity): 37 | return [ 38 | getattr(entity, key).property.columns[0] 39 | for key, column in sa.inspect(entity).columns.items() 40 | if isinstance(column.type, TSVectorType) 41 | ] 42 | 43 | 44 | def search(query, search_query, vector=None, regconfig=None, sort=False): 45 | """ 46 | Search given query with full text search. 47 | 48 | :param search_query: the search query 49 | :param vector: search vector to use 50 | :param regconfig: postgresql regconfig to be used 51 | :param sort: Order the results by relevance. This uses `cover density`_ ranking 52 | algorithm (``ts_rank_cd``) for sorting. 53 | 54 | .. _cover density: https://www.postgresql.org/docs/devel/textsearch-controls.html#TEXTSEARCH-RANKING 55 | """ 56 | if not search_query.strip(): 57 | return query 58 | 59 | if vector is None: 60 | entity = query.column_descriptions[0]["entity"] 61 | search_vectors = inspect_search_vectors(entity) 62 | vector = search_vectors[0] 63 | 64 | if regconfig is None: 65 | regconfig = search_manager.options["regconfig"] 66 | 67 | query = query.filter( 68 | vector.op("@@")(sa.func.parse_websearch(regconfig, search_query)) 69 | ) 70 | if sort: 71 | query = query.order_by( 72 | sa.desc(sa.func.ts_rank_cd(vector, sa.func.parse_websearch(search_query))) 73 | ) 74 | 75 | return query.params(term=search_query) 76 | 77 | 78 | class SQLConstruct: 79 | def __init__(self, tsvector_column, indexed_columns=None, options=None): 80 | self.table = tsvector_column.table 81 | self.tsvector_column = tsvector_column 82 | self.options = self.init_options(options) 83 | if indexed_columns: 84 | self.indexed_columns = list(indexed_columns) 85 | elif hasattr(self.tsvector_column.type, "columns"): 86 | self.indexed_columns = list(self.tsvector_column.type.columns) 87 | else: 88 | self.indexed_columns = None 89 | 90 | def init_options(self, options=None): 91 | if not options: 92 | options = {} 93 | for key, value in SearchManager.default_options.items(): 94 | try: 95 | option = self.tsvector_column.type.options[key] 96 | except (KeyError, AttributeError): 97 | option = value 98 | options.setdefault(key, option) 99 | return options 100 | 101 | @property 102 | def table_name(self): 103 | if self.table.schema: 104 | return f'{self.table.schema}."{self.table.name}"' 105 | else: 106 | return '"' + self.table.name + '"' 107 | 108 | @property 109 | def search_function_name(self): 110 | return self.options["search_trigger_function_name"].format( 111 | table=self.table.name, column=self.tsvector_column.name 112 | ) 113 | 114 | @property 115 | def search_trigger_name(self): 116 | return self.options["search_trigger_name"].format( 117 | table=self.table.name, column=self.tsvector_column.name 118 | ) 119 | 120 | def column_vector(self, column): 121 | value = sa.text(f"NEW.{sa.column(column.name)}") 122 | try: 123 | vectorizer_func = vectorizer[column] 124 | except KeyError: 125 | pass 126 | else: 127 | value = vectorizer_func(value) 128 | value = sa.func.coalesce(value, sa.text("''")) 129 | value = sa.func.to_tsvector(sa.literal(self.options["regconfig"]), value) 130 | if column.name in self.options["weights"]: 131 | weight = self.options["weights"][column.name] 132 | value = sa.func.setweight(value, weight) 133 | return value 134 | 135 | def search_vector(self, compiler): 136 | vectors = ( 137 | self.column_vector(getattr(self.table.c, column_name)) 138 | for column_name in self.indexed_columns 139 | ) 140 | concatenated = reduce(lambda x, y: x.op("||")(y), vectors) 141 | return compiler.sql_compiler.process(concatenated, literal_binds=True) 142 | 143 | 144 | class CreateSearchFunctionSQL(SQLConstruct, DDLElement, Executable): 145 | pass 146 | 147 | 148 | @compiles(CreateSearchFunctionSQL) 149 | def compile_create_search_function_sql(element, compiler): 150 | return f"""CREATE FUNCTION 151 | {element.search_function_name}() RETURNS TRIGGER AS $$ 152 | BEGIN 153 | NEW.{element.tsvector_column.name} = {element.search_vector(compiler)}; 154 | RETURN NEW; 155 | END 156 | $$ LANGUAGE 'plpgsql'; 157 | """ 158 | 159 | 160 | class CreateSearchTriggerSQL(SQLConstruct, DDLElement, Executable): 161 | @property 162 | def search_trigger_function_with_trigger_args(self): 163 | if self.options["weights"] or any( 164 | getattr(self.table.c, column) in vectorizer 165 | for column in self.indexed_columns 166 | ): 167 | return self.search_function_name + "()" 168 | return "tsvector_update_trigger({arguments})".format( 169 | arguments=", ".join( 170 | [self.tsvector_column.name, "'%s'" % self.options["regconfig"]] 171 | + self.indexed_columns 172 | ) 173 | ) 174 | 175 | 176 | @compiles(CreateSearchTriggerSQL) 177 | def compile_create_search_trigger_sql(element, compiler): 178 | return ( 179 | f"CREATE TRIGGER {element.search_trigger_name}" 180 | f" BEFORE UPDATE OR INSERT ON {element.table_name}" 181 | " FOR EACH ROW EXECUTE PROCEDURE" 182 | f" {element.search_trigger_function_with_trigger_args}" 183 | ) 184 | 185 | 186 | class DropSearchFunctionSQL(SQLConstruct, DDLElement, Executable): 187 | pass 188 | 189 | 190 | @compiles(DropSearchFunctionSQL) 191 | def compile_drop_search_function_sql(element, compiler): 192 | return "DROP FUNCTION IF EXISTS %s()" % element.search_function_name 193 | 194 | 195 | class DropSearchTriggerSQL(SQLConstruct, DDLElement, Executable): 196 | pass 197 | 198 | 199 | @compiles(DropSearchTriggerSQL) 200 | def compile_drop_search_trigger_sql(element, compiler): 201 | return ( 202 | f"DROP TRIGGER IF EXISTS {element.search_trigger_name} ON {element.table_name}" 203 | ) 204 | 205 | 206 | class SearchManager: 207 | default_options = { 208 | "search_trigger_name": "{table}_{column}_trigger", 209 | "search_trigger_function_name": "{table}_{column}_update", 210 | "regconfig": "pg_catalog.english", 211 | "weights": (), 212 | "auto_index": True, 213 | } 214 | 215 | def __init__(self, options={}): 216 | self.options = self.default_options 217 | self.options.update(options) 218 | self.processed_columns = [] 219 | self.listeners = [] 220 | 221 | def option(self, column, name): 222 | try: 223 | return column.type.options[name] 224 | except (AttributeError, KeyError): 225 | return self.options[name] 226 | 227 | def inspect_columns(self, table): 228 | """ 229 | Inspects all searchable columns for given class. 230 | 231 | :param table: SQLAlchemy Table 232 | """ 233 | return [column for column in table.c if isinstance(column.type, TSVectorType)] 234 | 235 | def append_index(self, cls, column): 236 | sa.Index( 237 | "_".join(("ix", column.table.name, column.name)), 238 | column, 239 | postgresql_using="gin", 240 | ) 241 | 242 | def process_mapper(self, mapper, cls): 243 | columns = self.inspect_columns(mapper.persist_selectable) 244 | for column in columns: 245 | if column in self.processed_columns: 246 | continue 247 | 248 | if self.option(column, "auto_index"): 249 | self.append_index(cls, column) 250 | 251 | self.processed_columns.append(column) 252 | 253 | def add_listener(self, args): 254 | self.listeners.append(args) 255 | event.listen(*args) 256 | 257 | def remove_listeners(self): 258 | for listener in self.listeners: 259 | event.remove(*listener) 260 | self.listeners = [] 261 | 262 | def attach_ddl_listeners(self): 263 | # Remove all previously added listeners, so that same listener don't 264 | # get added twice in situations where class configuration happens in 265 | # multiple phases (issue #31). 266 | self.remove_listeners() 267 | 268 | for column in self.processed_columns: 269 | # This sets up the trigger that keeps the tsvector column up to 270 | # date. 271 | if column.type.columns: 272 | table = column.table 273 | if self.option(column, "weights") or vectorizer.contains_tsvector( 274 | column 275 | ): 276 | self.add_listener( 277 | (table, "after_create", CreateSearchFunctionSQL(column)) 278 | ) 279 | self.add_listener( 280 | (table, "after_drop", DropSearchFunctionSQL(column)) 281 | ) 282 | self.add_listener( 283 | (table, "after_create", CreateSearchTriggerSQL(column)) 284 | ) 285 | 286 | 287 | search_manager = SearchManager() 288 | 289 | 290 | def sync_trigger( 291 | conn, 292 | table_name, 293 | tsvector_column, 294 | indexed_columns, 295 | metadata=None, 296 | options=None, 297 | schema=None, 298 | update_rows=True, 299 | ): 300 | """Synchronize the search trigger and trigger function for the given table and 301 | search vector column. Internally, this function executes the following SQL 302 | queries: 303 | 304 | - Drop the search trigger for the given table and column if it exists. 305 | - Drop the search function for the given table and column if it exists. 306 | - Create the search function for the given table and column. 307 | - Create the search trigger for the given table and column. 308 | - Update all rows for the given search vector by executing a column=column update 309 | query for the given table. 310 | 311 | Example:: 312 | 313 | from sqlalchemy_searchable import sync_trigger 314 | 315 | 316 | sync_trigger( 317 | conn, 318 | 'article', 319 | 'search_vector', 320 | ['name', 'content'] 321 | ) 322 | 323 | This function is especially useful when working with Alembic migrations. In the 324 | following example, we add a ``content`` column to the ``article`` table and then 325 | synchronize the trigger to contain this new column:: 326 | 327 | from alembic import op 328 | from sqlalchemy_searchable import sync_trigger 329 | 330 | 331 | def upgrade(): 332 | conn = op.get_bind() 333 | op.add_column('article', sa.Column('content', sa.Text)) 334 | 335 | sync_trigger(conn, 'article', 'search_vector', ['name', 'content']) 336 | 337 | # ... same for downgrade 338 | 339 | If you are using vectorizers, you need to initialize them in your migration 340 | file and pass them to this function:: 341 | 342 | import sqlalchemy as sa 343 | from alembic import op 344 | from sqlalchemy.dialects.postgresql import HSTORE 345 | from sqlalchemy_searchable import sync_trigger, vectorizer 346 | 347 | 348 | def upgrade(): 349 | vectorizer.clear() 350 | 351 | conn = op.get_bind() 352 | op.add_column('article', sa.Column('name_translations', HSTORE)) 353 | 354 | metadata = sa.MetaData(bind=conn) 355 | articles = sa.Table('article', metadata, autoload=True) 356 | 357 | @vectorizer(articles.c.name_translations) 358 | def hstore_vectorizer(column): 359 | return sa.cast(sa.func.avals(column), sa.Text) 360 | 361 | op.add_column('article', sa.Column('content', sa.Text)) 362 | sync_trigger( 363 | conn, 364 | 'article', 365 | 'search_vector', 366 | ['name_translations', 'content'], 367 | metadata=metadata 368 | ) 369 | 370 | # ... same for downgrade 371 | 372 | :param conn: SQLAlchemy Connection object 373 | :param table_name: name of the table to apply search trigger syncing 374 | :param tsvector_column: 375 | TSVector typed column which is used as the search index column 376 | :param indexed_columns: 377 | Full text indexed column names as a list 378 | :param metadata: 379 | Optional SQLAlchemy metadata object that is being used for autoloaded 380 | Table. If None is given, then a new MetaData object is initialized within 381 | this function. 382 | :param options: Dictionary of configuration options 383 | :param schema: The schema name for this table. Defaults to ``None``. 384 | :param update_rows: 385 | If set to False, the values in the vector column will remain unchanged 386 | until one of the indexed columns is updated. 387 | """ 388 | if metadata is None: 389 | metadata = sa.MetaData() 390 | table = sa.Table( 391 | table_name, 392 | metadata, 393 | autoload_with=conn, 394 | schema=schema, 395 | ) 396 | params = dict( 397 | tsvector_column=getattr(table.c, tsvector_column), 398 | indexed_columns=indexed_columns, 399 | options=options, 400 | ) 401 | classes = [ 402 | DropSearchTriggerSQL, 403 | DropSearchFunctionSQL, 404 | CreateSearchFunctionSQL, 405 | CreateSearchTriggerSQL, 406 | ] 407 | for class_ in classes: 408 | conn.execute(class_(**params)) 409 | 410 | if update_rows: 411 | update_sql = table.update().values( 412 | {indexed_columns[0]: sa.text(indexed_columns[0])} 413 | ) 414 | conn.execute(update_sql) 415 | 416 | 417 | def drop_trigger( 418 | conn, 419 | table_name, 420 | tsvector_column, 421 | metadata=None, 422 | options=None, 423 | schema=None, 424 | ): 425 | """ 426 | Drop the search trigger and trigger function for the given table and 427 | search vector column. Internally, this function executes the following SQL 428 | queries: 429 | 430 | - Drop the search trigger for the given table if it exists. 431 | - Drop the search function for the given table if it exists. 432 | 433 | Example:: 434 | 435 | from alembic import op 436 | from sqlalchemy_searchable import drop_trigger 437 | 438 | 439 | def downgrade(): 440 | conn = op.get_bind() 441 | 442 | drop_trigger(conn, 'article', 'search_vector') 443 | op.drop_index('ix_article_search_vector', table_name='article') 444 | op.drop_column('article', 'search_vector') 445 | 446 | :param conn: SQLAlchemy Connection object 447 | :param table_name: name of the table to apply search trigger dropping 448 | :param tsvector_column: 449 | TSVector typed column which is used as the search index column 450 | :param metadata: 451 | Optional SQLAlchemy metadata object that is being used for autoloaded 452 | Table. If None is given, then a new MetaData object is initialized within 453 | this function. 454 | :param options: Dictionary of configuration options 455 | :param schema: The schema name for this table. Defaults to ``None``. 456 | """ 457 | if metadata is None: 458 | metadata = sa.MetaData() 459 | table = sa.Table( 460 | table_name, 461 | metadata, 462 | autoload_with=conn, 463 | schema=schema, 464 | ) 465 | params = dict(tsvector_column=getattr(table.c, tsvector_column), options=options) 466 | classes = [ 467 | DropSearchTriggerSQL, 468 | DropSearchFunctionSQL, 469 | ] 470 | for class_ in classes: 471 | conn.execute(class_(**params)) 472 | 473 | 474 | path = os.path.dirname(os.path.abspath(__file__)) 475 | 476 | 477 | with open(os.path.join(path, "expressions.sql")) as file: 478 | sql_expressions = DDL(file.read()) 479 | 480 | 481 | def make_searchable(metadata, mapper=sa.orm.Mapper, manager=search_manager, options={}): 482 | """ 483 | Configure SQLAlchemy-Searchable for given SQLAlchemy metadata object. 484 | 485 | :param metadata: SQLAlchemy metadata object 486 | :param options: Dictionary of configuration options 487 | """ 488 | manager.options.update(options) 489 | event.listen(mapper, "instrument_class", manager.process_mapper) 490 | event.listen(mapper, "after_configured", manager.attach_ddl_listeners) 491 | event.listen(metadata, "before_create", sql_expressions) 492 | 493 | 494 | def remove_listeners(metadata, manager=search_manager, mapper=sa.orm.Mapper): 495 | event.remove(mapper, "instrument_class", manager.process_mapper) 496 | event.remove(mapper, "after_configured", manager.attach_ddl_listeners) 497 | manager.remove_listeners() 498 | event.remove(metadata, "before_create", sql_expressions) 499 | -------------------------------------------------------------------------------- /sqlalchemy_searchable/expressions.sql: -------------------------------------------------------------------------------- 1 | CREATE OR REPLACE FUNCTION parse_websearch(config regconfig, search_query text) 2 | RETURNS tsquery AS $$ 3 | SELECT 4 | string_agg( 5 | ( 6 | CASE 7 | WHEN position('''' IN words.word) > 0 THEN CONCAT(words.word, ':*') 8 | ELSE words.word 9 | END 10 | ), 11 | ' ' 12 | )::tsquery 13 | FROM ( 14 | SELECT trim( 15 | regexp_split_to_table( 16 | websearch_to_tsquery(config, lower(search_query))::text, 17 | ' ' 18 | ) 19 | ) AS word 20 | ) AS words 21 | $$ LANGUAGE SQL IMMUTABLE; 22 | 23 | 24 | CREATE OR REPLACE FUNCTION parse_websearch(search_query text) 25 | RETURNS tsquery AS $$ 26 | SELECT parse_websearch('pg_catalog.simple', search_query); 27 | $$ LANGUAGE SQL IMMUTABLE; 28 | -------------------------------------------------------------------------------- /sqlalchemy_searchable/vectorizers.py: -------------------------------------------------------------------------------- 1 | from functools import wraps 2 | from inspect import isclass 3 | 4 | import sqlalchemy as sa 5 | from sqlalchemy.orm.attributes import InstrumentedAttribute 6 | from sqlalchemy.sql.type_api import TypeEngine 7 | 8 | 9 | class Vectorizer: 10 | def __init__(self, type_vectorizers=None, column_vectorizers=None): 11 | self.type_vectorizers = {} if type_vectorizers is None else type_vectorizers 12 | self.column_vectorizers = ( 13 | {} if column_vectorizers is None else column_vectorizers 14 | ) 15 | 16 | def clear(self): 17 | """Clear all registered vectorizers.""" 18 | self.type_vectorizers = {} 19 | self.column_vectorizers = {} 20 | 21 | def contains_tsvector(self, tsvector_column): 22 | if not hasattr(tsvector_column.type, "columns"): 23 | return False 24 | return any( 25 | getattr(tsvector_column.table.c, column) in self 26 | for column in tsvector_column.type.columns 27 | ) 28 | 29 | def __contains__(self, column): 30 | try: 31 | self[column] 32 | return True 33 | except KeyError: 34 | return False 35 | 36 | def __getitem__(self, column): 37 | if column in self.column_vectorizers: 38 | return self.column_vectorizers[column] 39 | type_class = column.type.__class__ 40 | 41 | if type_class in self.type_vectorizers: 42 | return self.type_vectorizers[type_class] 43 | raise KeyError(column) 44 | 45 | def __call__(self, type_or_column): 46 | """Decorator to register a function as a vectorizer. 47 | 48 | :param type_or_column: the SQLAlchemy database data type or the column to 49 | register a vectorizer for 50 | """ 51 | 52 | def outer(func): 53 | @wraps(func) 54 | def wrapper(*args, **kwargs): 55 | return func(*args, **kwargs) 56 | 57 | if isclass(type_or_column) and issubclass(type_or_column, TypeEngine): 58 | self.type_vectorizers[type_or_column] = wrapper 59 | elif isinstance(type_or_column, sa.Column): 60 | self.column_vectorizers[type_or_column] = wrapper 61 | elif isinstance(type_or_column, InstrumentedAttribute): 62 | prop = type_or_column.property 63 | if not isinstance(prop, sa.orm.ColumnProperty): 64 | raise TypeError( 65 | "Given InstrumentedAttribute does not wrap " 66 | "ColumnProperty. Only instances of ColumnProperty are " 67 | "supported for vectorizer." 68 | ) 69 | column = type_or_column.property.columns[0] 70 | 71 | self.column_vectorizers[column] = wrapper 72 | else: 73 | raise TypeError( 74 | "First argument should be either valid SQLAlchemy type, " 75 | "Column, ColumnProperty or InstrumentedAttribute object." 76 | ) 77 | 78 | return wrapper 79 | 80 | return outer 81 | -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/falcony-io/sqlalchemy-searchable/cf0b4c462c4a17c872923f6b3a3191a4bb08c2f7/tests/__init__.py -------------------------------------------------------------------------------- /tests/conftest.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | import pytest 4 | from sqlalchemy import ( 5 | Column, 6 | create_engine, 7 | DateTime, 8 | ForeignKey, 9 | Integer, 10 | String, 11 | Text, 12 | text, 13 | ) 14 | from sqlalchemy.orm import ( 15 | close_all_sessions, 16 | configure_mappers, 17 | declarative_base, 18 | sessionmaker, 19 | ) 20 | from sqlalchemy_utils import TSVectorType 21 | 22 | from sqlalchemy_searchable import ( 23 | make_searchable, 24 | remove_listeners, 25 | search_manager, 26 | vectorizer, 27 | ) 28 | 29 | try: 30 | import __pypy__ 31 | except ImportError: 32 | __pypy__ = None 33 | 34 | 35 | if __pypy__: 36 | from psycopg2cffi import compat 37 | 38 | compat.register() 39 | 40 | 41 | def pytest_configure(config): 42 | config.addinivalue_line( 43 | "markers", 44 | "postgresql_min_version(min_version): " 45 | "skip the test if PostgreSQL version is less than min_version", 46 | ) 47 | config.addinivalue_line( 48 | "markers", 49 | "postgresql_max_version(max_version): " 50 | "skip the test if PostgreSQL version is greater than max_version", 51 | ) 52 | 53 | 54 | @pytest.fixture 55 | def engine(): 56 | db_user = os.environ.get("SQLALCHEMY_SEARCHABLE_TEST_USER", "postgres") 57 | db_password = os.environ.get("SQLALCHEMY_SEARCHABLE_TEST_PASSWORD", "") 58 | db_name = os.environ.get( 59 | "SQLALCHEMY_SEARCHABLE_TEST_DB", "sqlalchemy_searchable_test" 60 | ) 61 | url = f"postgresql://{db_user}:{db_password}@localhost/{db_name}" 62 | 63 | engine = create_engine(url, future=True) 64 | with engine.begin() as conn: 65 | conn.execute(text("CREATE EXTENSION IF NOT EXISTS hstore")) 66 | 67 | yield engine 68 | engine.dispose() 69 | 70 | 71 | @pytest.fixture 72 | def postgresql_version(engine): 73 | with engine.connect(): 74 | major, _ = engine.dialect.server_version_info 75 | return major 76 | 77 | 78 | @pytest.fixture(autouse=True) 79 | def check_postgresql_min_version(request, postgresql_version): 80 | postgresql_min_version_mark = request.node.get_closest_marker( 81 | "postgresql_min_version" 82 | ) 83 | if postgresql_min_version_mark: 84 | min_version = postgresql_min_version_mark.args[0] 85 | if postgresql_version < min_version: 86 | pytest.skip(f"Requires PostgreSQL >= {min_version}") 87 | 88 | 89 | @pytest.fixture(autouse=True) 90 | def check_postgresql_max_version(request, postgresql_version): 91 | postgresql_max_version_mark = request.node.get_closest_marker( 92 | "postgresql_max_version" 93 | ) 94 | if postgresql_max_version_mark: 95 | max_version = postgresql_max_version_mark.args[0] 96 | if postgresql_version > max_version: 97 | pytest.skip(f"Requires PostgreSQL <= {max_version}") 98 | 99 | 100 | @pytest.fixture 101 | def session(engine): 102 | Session = sessionmaker(bind=engine, future=True) 103 | session = Session(future=True) 104 | 105 | yield session 106 | 107 | session.expunge_all() 108 | close_all_sessions() 109 | 110 | 111 | @pytest.fixture 112 | def search_manager_regconfig(): 113 | return None 114 | 115 | 116 | @pytest.fixture 117 | def Base(search_manager_regconfig): 118 | Base = declarative_base() 119 | make_searchable(Base.metadata) 120 | if search_manager_regconfig: 121 | search_manager.options["regconfig"] = search_manager_regconfig 122 | 123 | yield Base 124 | 125 | search_manager.options["regconfig"] = "pg_catalog.english" 126 | search_manager.processed_columns = [] 127 | vectorizer.clear() 128 | remove_listeners(Base.metadata) 129 | 130 | 131 | @pytest.fixture 132 | def search_trigger_name(): 133 | return "{table}_{column}_trigger" 134 | 135 | 136 | @pytest.fixture 137 | def search_trigger_function_name(): 138 | return "{table}_{column}_update" 139 | 140 | 141 | @pytest.fixture 142 | def ts_vector_options(search_trigger_name, search_trigger_function_name): 143 | return { 144 | "search_trigger_name": search_trigger_name, 145 | "search_trigger_function_name": search_trigger_function_name, 146 | "auto_index": True, 147 | } 148 | 149 | 150 | @pytest.fixture(autouse=True) 151 | def create_tables(Base, engine, models): 152 | configure_mappers() 153 | Base.metadata.create_all(engine) 154 | yield 155 | Base.metadata.drop_all(engine) 156 | 157 | 158 | @pytest.fixture 159 | def models(TextItem, Article): 160 | pass 161 | 162 | 163 | @pytest.fixture 164 | def TextItem(Base, ts_vector_options): 165 | class TextItem(Base): 166 | __tablename__ = "textitem" 167 | 168 | id = Column(Integer, primary_key=True, autoincrement=True) 169 | 170 | name = Column(String(255)) 171 | 172 | search_vector = Column(TSVectorType("name", "content", **ts_vector_options)) 173 | content_search_vector = Column(TSVectorType("content", **ts_vector_options)) 174 | 175 | content = Column(Text) 176 | 177 | return TextItem 178 | 179 | 180 | @pytest.fixture 181 | def Article(TextItem): 182 | class Article(TextItem): 183 | __tablename__ = "article" 184 | id = Column(Integer, ForeignKey(TextItem.id), primary_key=True) 185 | created_at = Column(DateTime) 186 | 187 | return Article 188 | -------------------------------------------------------------------------------- /tests/schema_test_case.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | from sqlalchemy import text 3 | 4 | 5 | class SchemaTestCase: 6 | @pytest.fixture 7 | def should_create_indexes(self): 8 | return [] 9 | 10 | @pytest.fixture 11 | def should_create_triggers(self): 12 | return [] 13 | 14 | def test_creates_search_index(self, session, should_create_indexes): 15 | rows = session.execute( 16 | text( 17 | """SELECT relname 18 | FROM pg_class 19 | WHERE oid IN ( 20 | SELECT indexrelid 21 | FROM pg_index, pg_class 22 | WHERE pg_class.relname = 'textitem' 23 | AND pg_class.oid = pg_index.indrelid 24 | AND indisunique != 't' 25 | AND indisprimary != 't' 26 | ) ORDER BY relname""" 27 | ) 28 | ).fetchall() 29 | assert should_create_indexes == [row[0] for row in rows] 30 | 31 | def test_creates_search_trigger(self, session, should_create_triggers): 32 | rows = session.execute( 33 | text( 34 | """SELECT DISTINCT trigger_name 35 | FROM information_schema.triggers 36 | WHERE event_object_table = 'textitem' 37 | AND trigger_schema NOT IN 38 | ('pg_catalog', 'information_schema') 39 | ORDER BY trigger_name""" 40 | ) 41 | ).fetchall() 42 | assert should_create_triggers == [row[0] for row in rows] 43 | -------------------------------------------------------------------------------- /tests/test_class_configuration.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | import sqlalchemy as sa 3 | 4 | 5 | class TestClassConfiguration: 6 | @pytest.fixture 7 | def create_tables(self): 8 | pass 9 | 10 | def test_attaches_listener_only_once(self, Base, engine): 11 | sa.orm.configure_mappers() 12 | 13 | class SomeClass(Base): 14 | __tablename__ = "some_class" 15 | id = sa.Column(sa.Integer, primary_key=True) 16 | 17 | sa.orm.configure_mappers() 18 | 19 | Base.metadata.create_all(engine) 20 | -------------------------------------------------------------------------------- /tests/test_drop_trigger.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | from sqlalchemy import text 3 | 4 | from sqlalchemy_searchable import drop_trigger, sync_trigger 5 | 6 | 7 | class TestDropTrigger: 8 | @pytest.fixture( 9 | params=[ 10 | "{table}_{column}_trigger", 11 | "{table}_{column}_trg", 12 | ] 13 | ) 14 | def search_trigger_name(self, request): 15 | return request.param 16 | 17 | @pytest.fixture( 18 | params=[ 19 | "{table}_{column}_update_trigger", 20 | "{table}_{column}_update", 21 | ] 22 | ) 23 | def search_trigger_function_name(self, request): 24 | return request.param 25 | 26 | @pytest.fixture(autouse=True) 27 | def create_tables(self, engine): 28 | with engine.begin() as conn: 29 | conn.execute( 30 | text( 31 | """ 32 | CREATE TABLE article ( 33 | name TEXT, 34 | content TEXT, 35 | "current_user" TEXT, 36 | search_vector TSVECTOR 37 | ) 38 | """ 39 | ) 40 | ) 41 | 42 | yield 43 | 44 | with engine.begin() as conn: 45 | conn.execute(text("DROP TABLE article")) 46 | 47 | def test_drops_triggers_and_functions(self, engine, ts_vector_options): 48 | def trigger_exist(conn): 49 | return conn.execute( 50 | text( 51 | """SELECT COUNT(*) 52 | FROM pg_trigger 53 | WHERE tgname = :trigger_name 54 | """ 55 | ), 56 | { 57 | "trigger_name": ts_vector_options["search_trigger_name"].format( 58 | table="article", 59 | column="search_vector", 60 | ) 61 | }, 62 | ).scalar() 63 | 64 | def function_exist(conn): 65 | return conn.execute( 66 | text( 67 | """SELECT COUNT(*) 68 | FROM pg_proc 69 | WHERE proname = :function_name 70 | """ 71 | ), 72 | { 73 | "function_name": ts_vector_options[ 74 | "search_trigger_function_name" 75 | ].format( 76 | table="article", 77 | column="search_vector", 78 | ) 79 | }, 80 | ).scalar() 81 | 82 | with engine.begin() as conn: 83 | sync_trigger( 84 | conn, 85 | "article", 86 | "search_vector", 87 | ["name", "content"], 88 | options=ts_vector_options, 89 | ) 90 | 91 | assert trigger_exist(conn) == 1 92 | assert function_exist(conn) == 1 93 | 94 | drop_trigger(conn, "article", "search_vector", options=ts_vector_options) 95 | 96 | assert trigger_exist(conn) == 0 97 | assert function_exist(conn) == 0 98 | -------------------------------------------------------------------------------- /tests/test_multiple_vectors_per_class.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | import sqlalchemy as sa 3 | from sqlalchemy_utils import TSVectorType 4 | 5 | from sqlalchemy_searchable import search 6 | from tests.schema_test_case import SchemaTestCase 7 | 8 | 9 | class TestMultipleSearchVectorsPerClass(SchemaTestCase): 10 | @pytest.fixture 11 | def should_create_indexes(self): 12 | return [ 13 | "ix_textitem_content_vector", 14 | "ix_textitem_name_vector", 15 | ] 16 | 17 | @pytest.fixture 18 | def should_create_triggers(self): 19 | return [ 20 | "textitem_content_vector_trigger", 21 | "textitem_name_vector_trigger", 22 | ] 23 | 24 | @pytest.fixture 25 | def models(self, Base): 26 | class TextItem(Base): 27 | __tablename__ = "textitem" 28 | 29 | id = sa.Column(sa.Integer, primary_key=True, autoincrement=True) 30 | 31 | name = sa.Column(sa.String(255)) 32 | 33 | content = sa.Column(sa.Text) 34 | 35 | name_vector = sa.Column(TSVectorType("name", auto_index=True)) 36 | 37 | content_vector = sa.Column(TSVectorType("content", auto_index=True)) 38 | 39 | 40 | class TestMultipleSearchVectorsSearchFunction: 41 | @pytest.fixture 42 | def TextMultiItem(self, Base): 43 | class TextMultiItem(Base): 44 | __tablename__ = "textmultiitem" 45 | 46 | id = sa.Column(sa.Integer, primary_key=True, autoincrement=True) 47 | 48 | name = sa.Column(sa.String(255)) 49 | content = sa.Column(sa.Text) 50 | name_vector = sa.Column(TSVectorType("name", auto_index=False)) 51 | content_vector = sa.Column(TSVectorType("content", auto_index=False)) 52 | 53 | return TextMultiItem 54 | 55 | @pytest.fixture 56 | def models(self, TextMultiItem): 57 | pass 58 | 59 | def test_choose_vector(self, session, TextMultiItem): 60 | session.add(TextMultiItem(name="index", content="lorem ipsum")) 61 | session.add(TextMultiItem(name="ipsum", content="admin content")) 62 | session.commit() 63 | 64 | s1 = search(sa.select(TextMultiItem), "ipsum", vector=TextMultiItem.name_vector) 65 | assert session.scalars(s1).first().name == "ipsum" 66 | 67 | def test_without_auto_index(self, TextMultiItem): 68 | indexes = TextMultiItem.__table__.indexes 69 | assert indexes == set() 70 | -------------------------------------------------------------------------------- /tests/test_schema_creation.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | import sqlalchemy as sa 3 | from sqlalchemy_utils import TSVectorType 4 | 5 | from tests.schema_test_case import SchemaTestCase 6 | 7 | 8 | class TestAutomaticallyCreatedSchemaItems(SchemaTestCase): 9 | @pytest.fixture 10 | def should_create_indexes(self): 11 | return [ 12 | "ix_textitem_content_search_vector", 13 | "ix_textitem_search_vector", 14 | ] 15 | 16 | @pytest.fixture 17 | def should_create_triggers(self): 18 | return [ 19 | "textitem_content_search_vector_trigger", 20 | "textitem_search_vector_trigger", 21 | ] 22 | 23 | 24 | class TestSearchVectorWithoutColumns(SchemaTestCase): 25 | @pytest.fixture 26 | def should_create_indexes(self): 27 | return ["ix_textitem_search_vector"] 28 | 29 | @pytest.fixture 30 | def should_create_triggers(self): 31 | return [] 32 | 33 | @pytest.fixture 34 | def models(self, Base): 35 | class TextItem(Base): 36 | __tablename__ = "textitem" 37 | 38 | id = sa.Column(sa.Integer, primary_key=True, autoincrement=True) 39 | 40 | name = sa.Column(sa.String(255)) 41 | 42 | search_vector = sa.Column(TSVectorType(auto_index=True)) 43 | 44 | content = sa.Column(sa.Text) 45 | -------------------------------------------------------------------------------- /tests/test_searchable.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | from sqlalchemy import func, select 3 | from sqlalchemy.orm.query import Query 4 | 5 | from sqlalchemy_searchable import search, SearchQueryMixin 6 | 7 | 8 | class TextItemQuery(Query, SearchQueryMixin): 9 | pass 10 | 11 | 12 | class TestSearchQueryMixin: 13 | @pytest.fixture( 14 | params=[ 15 | "{table}_{column}_trigger", 16 | "{table}_{column}_trg", 17 | ] 18 | ) 19 | def search_trigger_name(self, request): 20 | return request.param 21 | 22 | @pytest.fixture( 23 | params=[ 24 | "{table}_{column}_update_trigger", 25 | "{table}_{column}_update", 26 | ] 27 | ) 28 | def search_trigger_function_name(self, request): 29 | return request.param 30 | 31 | @pytest.fixture(autouse=True) 32 | def items(self, session, TextItem): 33 | items = [ 34 | TextItem(name="index", content="some content"), 35 | TextItem(name="admin", content="admin content"), 36 | TextItem( 37 | name="home", content="this is the home page of someone@example.com" 38 | ), 39 | TextItem(name="not a some content", content="not a name"), 40 | ] 41 | session.add_all(items) 42 | session.commit() 43 | return items 44 | 45 | def test_searches_through_all_fulltext_indexed_fields(self, TextItem, session): 46 | assert TextItemQuery(TextItem, session).search("admin").count() == 1 47 | 48 | def test_search_supports_term_splitting(self, TextItem, session): 49 | assert TextItemQuery(TextItem, session).search("content").count() == 3 50 | 51 | def test_term_splitting_supports_multiple_spaces(self, TextItem, session): 52 | query = TextItemQuery(TextItem, session) 53 | assert query.search("content some").first().name == "index" 54 | assert query.search("content some").first().name == "index" 55 | assert query.search(" ").count() == 4 56 | 57 | def test_search_by_email(self, TextItem, session): 58 | assert TextItemQuery(TextItem, session).search("someone@example.com").count() 59 | 60 | def test_supports_regconfig_parameter(self, TextItem, session): 61 | query = TextItemQuery(TextItem, session) 62 | query = query.search("orrimorri", regconfig="finnish") 63 | assert "parse_websearch(%(parse_websearch_1)s, %(parse_websearch_2)s)" in str( 64 | query.statement.compile(session.bind) 65 | ) 66 | 67 | def test_supports_vector_parameter(self, TextItem, session): 68 | vector = TextItem.content_search_vector 69 | query = TextItemQuery(TextItem, session) 70 | query = query.search("content", vector=vector) 71 | assert query.count() == 2 72 | 73 | def test_search_specific_columns(self, TextItem, session): 74 | query = search(select(TextItem.id), "admin").subquery() 75 | assert session.scalar(select(func.count()).select_from(query)) == 1 76 | 77 | def test_sorted_search_results(self, TextItem, session, items): 78 | query = TextItemQuery(TextItem, session) 79 | sorted_results = query.search("some content", sort=True).all() 80 | assert sorted_results == items[0:2] + [items[3]] 81 | 82 | 83 | class TestUsesGlobalConfigOptionsAsFallbacks: 84 | @pytest.fixture 85 | def search_manager_regconfig(self): 86 | return "pg_catalog.simple" 87 | 88 | @pytest.fixture(autouse=True) 89 | def items(self, session, TextItem): 90 | items = [ 91 | TextItem(name="index", content="some content"), 92 | TextItem(name="admin", content="admin content"), 93 | TextItem( 94 | name="home", content="this is the home page of someone@example.com" 95 | ), 96 | TextItem(name="not a some content", content="not a name"), 97 | ] 98 | session.add_all(items) 99 | session.commit() 100 | 101 | def test_uses_global_regconfig_as_fallback(self, session, TextItem): 102 | query = search(select(TextItem.id), "the").subquery() 103 | assert session.scalar(select(func.count()).select_from(query)) == 1 104 | 105 | 106 | class TestSearchableInheritance: 107 | @pytest.fixture(autouse=True) 108 | def articles(self, session, Article): 109 | session.add(Article(name="index", content="some content")) 110 | session.add(Article(name="admin", content="admin content")) 111 | session.add(Article(name="home", content="this is the home page")) 112 | session.commit() 113 | 114 | def test_supports_inheritance(self, session, Article): 115 | assert TextItemQuery(Article, session).search("content").count() == 2 116 | -------------------------------------------------------------------------------- /tests/test_single_table_inheritance.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | import sqlalchemy as sa 3 | from sqlalchemy_utils import TSVectorType 4 | 5 | from tests.schema_test_case import SchemaTestCase 6 | 7 | 8 | class TestSearchableWithSingleTableInheritance(SchemaTestCase): 9 | @pytest.fixture 10 | def should_create_indexes(self): 11 | return ["ix_textitem_search_vector"] 12 | 13 | @pytest.fixture 14 | def should_create_triggers(self): 15 | return ["textitem_search_vector_trigger"] 16 | 17 | @pytest.fixture 18 | def models(self, Base): 19 | class TextItem(Base): 20 | __tablename__ = "textitem" 21 | 22 | id = sa.Column(sa.Integer, primary_key=True, autoincrement=True) 23 | 24 | name = sa.Column(sa.String(255)) 25 | 26 | search_vector = sa.Column(TSVectorType("name", "content")) 27 | 28 | content = sa.Column(sa.Text) 29 | 30 | class Article(TextItem): 31 | created_at = sa.Column(sa.DateTime) 32 | -------------------------------------------------------------------------------- /tests/test_sql_functions.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | from sqlalchemy import text 3 | 4 | 5 | class TestParse: 6 | @pytest.mark.parametrize( 7 | ("input", "output"), 8 | ( 9 | ("", ""), 10 | ("()", ""), 11 | ("))someone(", "'someone':*"), 12 | ("((()))", ""), 13 | ("(( )) ( )", ""), 14 | ("))()()))))(((((( () ) (() )))) (( )", ""), 15 | ("(((((((((((((())))))))))))))", ""), 16 | ('"" "())"")"(()"))""""', ""), 17 | ("()-()", ""), 18 | ("STAR", "'star':*"), 19 | ('""', ""), 20 | ("or or", "'or':* & 'or':*"), 21 | ("-or or", "!'or':* & 'or':*"), 22 | ("-or -or or", "!'or':* & !'or':* & 'or':*"), 23 | ("or", "'or':*"), 24 | ("star wars", "'star':* & 'wars':*"), 25 | ("star!#", "'star':*"), 26 | ("123.14 or 12a4", "'123.14':* | '12a4':*"), 27 | ("john@example.com", "'john@example.com':*"), 28 | ("star organs", "'star':* & 'organs':*"), 29 | ("organs", "'organs':*"), 30 | ("star or wars", "'star':* | 'wars':*"), 31 | ("star or or wars", "'star':* | 'or':* & 'wars':*"), 32 | ('"star or or wars"', "star:* <-> or:* <-> or:* <-> wars:*"), 33 | ("star or or or wars", "'star':* | 'or':* | 'wars':*"), 34 | ("star oror wars", "'star':* & 'oror':* & 'wars':*"), 35 | pytest.param( 36 | "star-wars", 37 | "'star-wars':* & 'star':* & 'wars':*", 38 | marks=pytest.mark.postgresql_max_version(13), 39 | ), 40 | pytest.param( 41 | "star-wars", 42 | "'star-wars':* <-> 'star':* <-> 'wars':*", 43 | marks=pytest.mark.postgresql_min_version(14), 44 | ), 45 | pytest.param( 46 | "star----wars", 47 | "'star':* & 'wars':*", 48 | marks=pytest.mark.postgresql_max_version(13), 49 | ), 50 | pytest.param( 51 | "star----wars", 52 | "'star':* <-> 'wars':*", 53 | marks=pytest.mark.postgresql_min_version(14), 54 | ), 55 | ("star wars luke", "'star':* & 'wars':* & 'luke':*"), 56 | ("örrimöykky", "'örrimöykky':*"), 57 | ("-star", "!'star':*"), 58 | ("--star", "!!'star':*"), 59 | ("star or or", "'star':* | or:*"), 60 | ('star or -""', "'star':*"), 61 | ('star or ""', "'star':*"), 62 | ("star or -", "'star':*"), 63 | ("star or (", "'star':*"), 64 | ("- -star", "!!'star':*"), 65 | ("star -wars", "'star':* & !'wars':*"), 66 | ("'star'", "'star':*"), 67 | ("''star''", "'star':*"), 68 | ('"star wars"', "'star':* <-> 'wars':*"), 69 | ('-"star wars"', "!( 'star':* <-> 'wars':* )"), 70 | ('""star wars""', "'star':* & 'wars':*"), 71 | ("star!:*@@?`", "'star':*"), 72 | ('"star', "'star':*"), 73 | ("ähtäri", "'ähtäri':*"), 74 | ('test"', "'test':*"), 75 | ('"test""', "'test':*"), 76 | ( 77 | '"death star" -"star wars"', 78 | "'death':* <-> 'star':* & ! ('star':* <-> 'wars':*)", 79 | ), 80 | ( 81 | '"something fishy happened"', 82 | "'something':* <-> 'fishy':* <-> 'happened':*", 83 | ), 84 | ( 85 | '"star wars" "death star"', 86 | "'star':* <-> 'wars':* & 'death':* <-> 'star':*", 87 | ), 88 | ( 89 | '"star wars""death star"', 90 | "'star':* <-> 'wars':* & 'death':* <-> 'star':*", 91 | ), 92 | ("star or wars luke or solo", "'star':* | 'wars':* & 'luke':* | 'solo':*"), 93 | pytest.param( 94 | "-star#wars", 95 | "!( 'star':* & 'wars':* )", 96 | marks=pytest.mark.postgresql_max_version(13), 97 | ), 98 | pytest.param( 99 | "-star#wars", 100 | "!( 'star':* <-> 'wars':* )", 101 | marks=pytest.mark.postgresql_min_version(14), 102 | ), 103 | pytest.param( 104 | "-star#wars or -star#wars", 105 | "!( 'star':* & 'wars':* ) | !( 'star':* & 'wars':* )", 106 | marks=pytest.mark.postgresql_max_version(13), 107 | ), 108 | pytest.param( 109 | "-star#wars or -star#wars", 110 | "!( 'star':* <-> 'wars':* ) | !( 'star':* <-> 'wars':* )", 111 | marks=pytest.mark.postgresql_min_version(14), 112 | ), 113 | pytest.param( 114 | '"star#wars star_wars"', 115 | "( 'star':* & 'wars':* ) <-> ( 'star':* & 'wars':* )", 116 | marks=pytest.mark.postgresql_max_version(13), 117 | ), 118 | pytest.param( 119 | '"star#wars star_wars"', 120 | "'star':* <-> 'wars':* <-> 'star':* <-> 'wars':*", 121 | marks=pytest.mark.postgresql_min_version(14), 122 | ), 123 | ), 124 | ) 125 | def test_parse(self, session, input, output): 126 | assert ( 127 | session.execute( 128 | text("SELECT parse_websearch('pg_catalog.simple', :input)"), 129 | {"input": input}, 130 | ).scalar() 131 | == session.execute( 132 | text("SELECT CAST(:output AS tsquery)"), {"output": output} 133 | ).scalar() 134 | ) 135 | -------------------------------------------------------------------------------- /tests/test_sync_trigger.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | import sqlalchemy as sa 3 | from sqlalchemy import text 4 | 5 | from sqlalchemy_searchable import sync_trigger, vectorizer 6 | 7 | 8 | class TestSyncTrigger: 9 | @pytest.fixture( 10 | params=[ 11 | "{table}_{column}_trigger", 12 | "{table}_{column}_trg", 13 | ] 14 | ) 15 | def search_trigger_name(self, request): 16 | return request.param 17 | 18 | @pytest.fixture( 19 | params=[ 20 | "{table}_{column}_update_trigger", 21 | "{table}_{column}_update", 22 | ] 23 | ) 24 | def search_trigger_function_name(self, request): 25 | return request.param 26 | 27 | @pytest.fixture(autouse=True) 28 | def create_tables(self, engine): 29 | with engine.begin() as conn: 30 | conn.execute( 31 | text( 32 | """ 33 | CREATE TABLE article ( 34 | name TEXT, 35 | content TEXT, 36 | "current_user" TEXT, 37 | search_vector TSVECTOR 38 | ); 39 | 40 | CREATE SCHEMA another; 41 | 42 | CREATE TABLE another.article ( 43 | name TEXT, 44 | content TEXT, 45 | "current_user" TEXT, 46 | search_vector TSVECTOR 47 | ); 48 | """ 49 | ) 50 | ) 51 | 52 | yield 53 | 54 | with engine.begin() as conn: 55 | conn.execute( 56 | text( 57 | """ 58 | DROP TABLE article; 59 | DROP TABLE another.article; 60 | DROP SCHEMA another; 61 | """ 62 | ) 63 | ) 64 | 65 | def test_creates_triggers_and_functions(self, engine, ts_vector_options): 66 | with engine.begin() as conn: 67 | sync_trigger( 68 | conn, 69 | "article", 70 | "search_vector", 71 | ["name", "content"], 72 | options=ts_vector_options, 73 | ) 74 | conn.execute( 75 | text( 76 | """INSERT INTO article (name, content) 77 | VALUES ('some name', 'some content')""" 78 | ) 79 | ) 80 | vector = conn.execute(text("SELECT search_vector FROM article")).scalar() 81 | assert vector == "'content':4 'name':2" 82 | 83 | def test_different_schema(self, engine): 84 | with engine.begin() as conn: 85 | sync_trigger( 86 | conn, 87 | "article", 88 | "search_vector", 89 | ["name", "content"], 90 | schema="another", 91 | ) 92 | conn.execute( 93 | text( 94 | """INSERT INTO another.article (name, content) 95 | VALUES ('some name', 'some content')""" 96 | ) 97 | ) 98 | vector = conn.execute( 99 | text("SELECT search_vector FROM another.article") 100 | ).scalar() 101 | assert vector == "'content':4 'name':2" 102 | 103 | def test_updates_column_values(self, engine, ts_vector_options): 104 | with engine.begin() as conn: 105 | sync_trigger( 106 | conn, 107 | "article", 108 | "search_vector", 109 | ["name", "content"], 110 | options=ts_vector_options, 111 | ) 112 | conn.execute( 113 | text( 114 | """INSERT INTO article (name, content) 115 | VALUES ('some name', 'some content')""" 116 | ) 117 | ) 118 | conn.execute(text("ALTER TABLE article DROP COLUMN name")) 119 | sync_trigger( 120 | conn, 121 | "article", 122 | "search_vector", 123 | ["content"], 124 | options=ts_vector_options, 125 | ) 126 | vector = conn.execute(text("SELECT search_vector FROM article")).scalar() 127 | assert vector == "'content':2" 128 | 129 | def test_does_not_update_column_values_when_updating_rows_disabled( 130 | self, engine, ts_vector_options 131 | ): 132 | with engine.begin() as conn: 133 | sync_trigger( 134 | conn, 135 | "article", 136 | "search_vector", 137 | ["name", "content"], 138 | options=ts_vector_options, 139 | ) 140 | conn.execute( 141 | text( 142 | """INSERT INTO article (name, content) 143 | VALUES ('some name', 'some content')""" 144 | ) 145 | ) 146 | conn.execute(text("ALTER TABLE article DROP COLUMN name")) 147 | sync_trigger( 148 | conn, 149 | "article", 150 | "search_vector", 151 | ["content"], 152 | options=ts_vector_options, 153 | update_rows=False, 154 | ) 155 | vector = conn.execute(text("SELECT search_vector FROM article")).scalar() 156 | assert vector == "'content':4 'name':2" 157 | 158 | def test_custom_vectorizers(sel, Base, engine, session, ts_vector_options): 159 | articles = sa.Table( 160 | "article", 161 | Base.metadata, 162 | autoload_with=session.bind, 163 | ) 164 | 165 | @vectorizer(articles.c.content) 166 | def vectorize_content(column): 167 | return sa.func.replace(column, "bad", "good") 168 | 169 | with engine.begin() as conn: 170 | sync_trigger( 171 | conn, 172 | "article", 173 | "search_vector", 174 | ["name", "content"], 175 | metadata=Base.metadata, 176 | options=ts_vector_options, 177 | ) 178 | conn.execute( 179 | text( 180 | """INSERT INTO article (name, content) 181 | VALUES ('some name', 'some bad content')""" 182 | ) 183 | ) 184 | vector = conn.execute(text("SELECT search_vector FROM article")).scalar() 185 | assert vector == "'content':5 'good':4 'name':2" 186 | 187 | def test_trigger_with_reserved_word(self, engine, ts_vector_options): 188 | with engine.begin() as conn: 189 | conn.execute( 190 | text( 191 | """INSERT INTO article (name, content, "current_user") 192 | VALUES ('some name', 'some bad content', now())""" 193 | ) 194 | ) 195 | 196 | sync_trigger( 197 | conn, 198 | "article", 199 | "search_vector", 200 | ["name", "content", "current_user"], 201 | options=ts_vector_options, 202 | ) 203 | # raises ProgrammingError without reserved_words: 204 | conn.execute(text("UPDATE article SET name=name")) 205 | -------------------------------------------------------------------------------- /tests/test_vectorizers.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | import sqlalchemy as sa 3 | from sqlalchemy.dialects.postgresql import HSTORE 4 | from sqlalchemy_utils import TSVectorType 5 | 6 | from sqlalchemy_searchable import vectorizer 7 | 8 | 9 | class TestTypeVectorizers: 10 | @pytest.fixture 11 | def models(self, Article): 12 | pass 13 | 14 | @pytest.fixture 15 | def Article(self, Base): 16 | class Article(Base): 17 | __tablename__ = "textitem" 18 | 19 | id = sa.Column(sa.Integer, primary_key=True, autoincrement=True) 20 | 21 | name = sa.Column(HSTORE) 22 | 23 | search_vector = sa.Column( 24 | TSVectorType("name", "content", regconfig="simple") 25 | ) 26 | 27 | content = sa.Column(sa.Text) 28 | 29 | @vectorizer(HSTORE) 30 | def hstore_vectorizer(column): 31 | return sa.cast(sa.func.avals(column), sa.Text) 32 | 33 | return Article 34 | 35 | def test_uses_type_vectorizer(self, Article, session): 36 | article = Article(name={"fi": "Joku artikkeli", "en": "Some article"}) 37 | session.add(article) 38 | session.commit() 39 | session.refresh(article) 40 | assert "article" in article.search_vector 41 | assert "joku" in article.search_vector 42 | assert "some" in article.search_vector 43 | assert "artikkeli" in article.search_vector 44 | assert "fi" not in article.search_vector 45 | assert "en" not in article.search_vector 46 | 47 | 48 | class TestColumnVectorizer: 49 | @pytest.fixture 50 | def models(self, Article): 51 | pass 52 | 53 | @pytest.fixture 54 | def Article(self, Base): 55 | class Article(Base): 56 | __tablename__ = "textitem" 57 | 58 | id = sa.Column(sa.Integer, primary_key=True, autoincrement=True) 59 | 60 | name = sa.Column(HSTORE) 61 | 62 | search_vector = sa.Column( 63 | TSVectorType("name", "content", regconfig="simple") 64 | ) 65 | 66 | content = sa.Column(sa.String) 67 | 68 | @vectorizer(Article.content) 69 | def vectorize_content(column): 70 | return sa.func.replace(column, "bad", "good") 71 | 72 | @vectorizer(HSTORE) 73 | def hstore_vectorizer(column): 74 | return sa.cast(sa.func.avals(column), sa.Text) 75 | 76 | return Article 77 | 78 | def test_column_vectorizer_has_priority_over_type_vectorizer( 79 | self, Article, session 80 | ): 81 | article = Article( 82 | name={"fi": "Joku artikkeli", "en": "Some article"}, content="bad" 83 | ) 84 | session.add(article) 85 | session.commit() 86 | session.refresh(article) 87 | for word in ["article", "artikkeli", "good", "joku", "some"]: 88 | assert word in article.search_vector 89 | 90 | def test_unknown_vectorizable_type(self): 91 | with pytest.raises(TypeError): 92 | 93 | @vectorizer("some unknown type") 94 | def my_vectorizer(column): 95 | pass 96 | -------------------------------------------------------------------------------- /tests/test_weighted_search_vector.py: -------------------------------------------------------------------------------- 1 | import re 2 | 3 | import pytest 4 | import sqlalchemy as sa 5 | from sqlalchemy import text 6 | from sqlalchemy_utils import TSVectorType 7 | 8 | from sqlalchemy_searchable import search 9 | from tests.schema_test_case import SchemaTestCase 10 | 11 | 12 | @pytest.fixture 13 | def models(WeightedTextItem): 14 | pass 15 | 16 | 17 | @pytest.fixture 18 | def WeightedTextItem(Base): 19 | class WeightedTextItem(Base): 20 | __tablename__ = "textitem" 21 | 22 | id = sa.Column(sa.Integer, primary_key=True, autoincrement=True) 23 | 24 | name = sa.Column(sa.String(255)) 25 | content = sa.Column(sa.Text) 26 | search_vector = sa.Column( 27 | TSVectorType("name", "content", weights={"name": "A", "content": "B"}) 28 | ) 29 | 30 | return WeightedTextItem 31 | 32 | 33 | class TestCreateWeightedSearchVector(SchemaTestCase): 34 | @pytest.fixture 35 | def should_create_indexes(self): 36 | return ["ix_textitem_search_vector"] 37 | 38 | @pytest.fixture 39 | def should_create_triggers(self): 40 | return ["textitem_search_vector_trigger"] 41 | 42 | def test_search_function_weights(self, session): 43 | func_name = "textitem_search_vector_update" 44 | sql = text("SELECT proname,prosrc FROM pg_proc WHERE proname=:name") 45 | name, src = session.execute(sql, {"name": func_name}).fetchone() 46 | pattern = ( 47 | r"setweight\(to_tsvector\(.+?" 48 | r"coalesce\(NEW.(\w+).+?" 49 | r"\)\), '([A-D])'\)" 50 | ) 51 | first, second = (match.groups() for match in re.finditer(pattern, src)) 52 | assert first == ("name", "A") 53 | assert second == ("content", "B") 54 | 55 | 56 | class TestWeightedSearchFunction: 57 | @pytest.fixture(autouse=True) 58 | def items(self, session, WeightedTextItem): 59 | session.add(WeightedTextItem(name="Gort", content="Klaatu barada nikto")) 60 | session.add(WeightedTextItem(name="Klaatu", content="barada nikto")) 61 | session.commit() 62 | 63 | def test_weighted_search_results(self, session, WeightedTextItem): 64 | first, second = session.scalars( 65 | search(sa.select(WeightedTextItem), "klaatu", sort=True) 66 | ).all() 67 | assert first.search_vector == "'barada':2B 'klaatu':1A 'nikto':3B" 68 | assert second.search_vector == "'barada':3B 'gort':1A 'klaatu':2B 'nikto':4B" 69 | -------------------------------------------------------------------------------- /tox.ini: -------------------------------------------------------------------------------- 1 | [tox] 2 | envlist = {py38,py39,310,311,312,pypy3}-sqla{1.4,2.0}, lint 3 | 4 | [testenv] 5 | deps= 6 | pytest>=2.2.3 7 | psycopg2cffi>=2.6.1; platform_python_implementation == 'PyPy' 8 | psycopg2>=2.4.6; platform_python_implementation == 'CPython' 9 | sqla1.4: SQLAlchemy>=1.4,<1.5 10 | sqla2.0: SQLAlchemy>=2.0,<2.1 11 | passenv = 12 | SQLALCHEMY_SEARCHABLE_TEST_USER 13 | SQLALCHEMY_SEARCHABLE_TEST_PASSWORD 14 | SQLALCHEMY_SEARCHABLE_TEST_DB 15 | setenv = 16 | SQLALCHEMY_WARN_20=1 17 | commands=py.test {posargs} 18 | 19 | [testenv:lint] 20 | deps = 21 | ruff==0.2.2 22 | commands = 23 | ruff check . 24 | ruff format --check 25 | --------------------------------------------------------------------------------