├── .github └── workflows │ ├── backport.yml │ └── ci.yml ├── .gitignore ├── .readthedocs.yaml ├── AUTHORS ├── CONTRIBUTING.rst ├── Changelog.rst ├── LICENSE ├── MANIFEST.in ├── README ├── README.rst ├── docs ├── CONTRIBUTING.rst ├── Changelog.rst ├── Makefile ├── api.rst ├── async_api.rst ├── asyncio.rst ├── conf.py ├── configuration.rst ├── faceted_search.rst ├── index.rst ├── persistence.rst ├── search_dsl.rst ├── tutorials.rst └── update_by_query.rst ├── elasticsearch_dsl └── __init__.py ├── examples ├── README.rst ├── alias_migration.py ├── async │ ├── alias_migration.py │ ├── completion.py │ ├── composite_agg.py │ ├── parent_child.py │ ├── percolate.py │ ├── search_as_you_type.py │ ├── semantic_text.py │ ├── sparse_vectors.py │ └── vectors.py ├── completion.py ├── composite_agg.py ├── parent_child.py ├── percolate.py ├── search_as_you_type.py ├── semantic_text.py ├── sparse_vectors.py └── vectors.py ├── noxfile.py ├── setup.cfg ├── setup.py ├── tests ├── __init__.py ├── _async │ ├── __init__.py │ ├── test_document.py │ ├── test_faceted_search.py │ ├── test_index.py │ ├── test_mapping.py │ ├── test_search.py │ └── test_update_by_query.py ├── _sync │ ├── __init__.py │ ├── test_document.py │ ├── test_faceted_search.py │ ├── test_index.py │ ├── test_mapping.py │ ├── test_search.py │ └── test_update_by_query.py ├── async_sleep.py ├── conftest.py ├── sleep.py ├── test_aggs.py ├── test_analysis.py ├── test_connections.py ├── test_field.py ├── test_integration │ ├── __init__.py │ ├── _async │ │ ├── __init__.py │ │ ├── test_analysis.py │ │ ├── test_document.py │ │ ├── test_faceted_search.py │ │ ├── test_index.py │ │ ├── test_mapping.py │ │ ├── test_search.py │ │ └── test_update_by_query.py │ ├── _sync │ │ ├── __init__.py │ │ ├── test_analysis.py │ │ ├── test_document.py │ │ ├── test_faceted_search.py │ │ ├── test_index.py │ │ ├── test_mapping.py │ │ ├── test_search.py │ │ └── test_update_by_query.py │ ├── test_count.py │ ├── test_data.py │ └── test_examples │ │ ├── __init__.py │ │ ├── _async │ │ ├── __init__.py │ │ ├── test_alias_migration.py │ │ ├── test_completion.py │ │ ├── test_composite_aggs.py │ │ ├── test_parent_child.py │ │ ├── test_percolate.py │ │ └── test_vectors.py │ │ ├── _sync │ │ ├── __init__.py │ │ ├── test_alias_migration.py │ │ ├── test_completion.py │ │ ├── test_composite_aggs.py │ │ ├── test_parent_child.py │ │ ├── test_percolate.py │ │ └── test_vectors.py │ │ ├── async_examples │ │ └── examples ├── test_package.py ├── test_query.py ├── test_result.py ├── test_utils.py ├── test_validation.py └── test_wrappers.py └── utils ├── build-dists.py ├── generator.py ├── license-headers.py ├── run-unasync.py └── templates ├── aggs.py.tpl ├── query.py.tpl ├── response.__init__.py.tpl └── types.py.tpl /.github/workflows/backport.yml: -------------------------------------------------------------------------------- 1 | name: Backport 2 | on: 3 | pull_request_target: 4 | types: 5 | - closed 6 | - labeled 7 | 8 | jobs: 9 | backport: 10 | name: Backport 11 | runs-on: ubuntu-latest 12 | # Only react to merged PRs for security reasons. 13 | # See https://docs.github.com/en/actions/using-workflows/events-that-trigger-workflows#pull_request_target. 14 | if: > 15 | github.event.pull_request.merged 16 | && ( 17 | github.event.action == 'closed' 18 | || ( 19 | github.event.action == 'labeled' 20 | && contains(github.event.label.name, 'backport') 21 | ) 22 | ) 23 | steps: 24 | - uses: tibdex/backport@9565281eda0731b1d20c4025c43339fb0a23812e # v2.0.4 25 | with: 26 | github_token: ${{ secrets.GITHUB_TOKEN }} 27 | -------------------------------------------------------------------------------- /.github/workflows/ci.yml: -------------------------------------------------------------------------------- 1 | --- 2 | name: CI 3 | 4 | on: [push, pull_request] 5 | 6 | jobs: 7 | package: 8 | runs-on: ubuntu-latest 9 | steps: 10 | - name: Checkout Repository 11 | uses: actions/checkout@v3 12 | - name: Set up Python 13 | uses: actions/setup-python@v4 14 | with: 15 | python-version: "3.12" 16 | - name: Install dependencies 17 | run: | 18 | python3 -m pip install setuptools wheel twine 19 | - name: Build packages 20 | run: | 21 | python3 utils/build-dists.py 22 | - name: Check packages 23 | run: | 24 | set -exo pipefail; 25 | if [ $(python3 -m twine check dist/* | grep -c 'warning') != 0 ]; then exit 1; fi 26 | 27 | lint: 28 | runs-on: ubuntu-latest 29 | steps: 30 | - name: Checkout Repository 31 | uses: actions/checkout@v3 32 | - name: Set up Python 33 | uses: actions/setup-python@v4 34 | with: 35 | python-version: "3.13" 36 | - name: Install dependencies 37 | run: | 38 | python3 -m pip install nox 39 | - name: Lint the code 40 | run: nox -s lint 41 | 42 | type_check: 43 | runs-on: ubuntu-latest 44 | steps: 45 | - name: Checkout Repository 46 | uses: actions/checkout@v3 47 | - name: Set up Python 48 | uses: actions/setup-python@v4 49 | with: 50 | python-version: "3.8" 51 | - name: Install dependencies 52 | run: | 53 | python3 -m pip install nox 54 | - name: Lint the code 55 | run: nox -s type_check 56 | 57 | docs: 58 | runs-on: ubuntu-latest 59 | steps: 60 | - name: Checkout Repository 61 | uses: actions/checkout@v3 62 | - name: Set up Python 63 | uses: actions/setup-python@v4 64 | with: 65 | python-version: "3.x" 66 | - name: Install dependencies 67 | run: | 68 | python3 -m pip install nox 69 | - name: Build the docs 70 | run: nox -s docs 71 | 72 | test-linux: 73 | runs-on: ubuntu-latest 74 | 75 | strategy: 76 | fail-fast: false 77 | matrix: 78 | python-version: [ 79 | "3.8", 80 | "3.9", 81 | "3.10", 82 | "3.11", 83 | "3.12", 84 | "3.13", 85 | ] 86 | es-version: [8.0.0, 8.16.0] 87 | 88 | steps: 89 | - name: Remove irrelevant software to free up disk space 90 | run: | 91 | df -h 92 | sudo rm -rf /opt/ghc 93 | sudo rm -rf /opt/hostedtoolcache/CodeQL 94 | sudo rm -rf /usr/local/lib/android 95 | sudo rm -rf /usr/share/dotnet 96 | df -h 97 | - name: Checkout Repository 98 | uses: actions/checkout@v3 99 | - name: Setup Elasticsearch 100 | run: | 101 | mkdir /tmp/elasticsearch 102 | wget -O - https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-${{ matrix.es-version }}-linux-x86_64.tar.gz | tar xz --directory=/tmp/elasticsearch --strip-components=1 103 | /tmp/elasticsearch/bin/elasticsearch -E xpack.security.enabled=false -E discovery.type=single-node -d 104 | - name: Setup Python - ${{ matrix.python-version }} 105 | uses: actions/setup-python@v4 106 | with: 107 | python-version: ${{ matrix.python-version }} 108 | - name: Install dependencies 109 | run: | 110 | python3 -m pip install nox 111 | - name: Run Tests 112 | run: | 113 | nox -rs test-${{ matrix.python-version }} 114 | env: 115 | WAIT_FOR_ES: "1" 116 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .*.swp 2 | *~ 3 | *.py[co] 4 | .coverage 5 | *.egg-info 6 | dist 7 | build 8 | *.egg 9 | coverage.xml 10 | junit.xml 11 | test_elasticsearch_dsl/htmlcov 12 | docs/_build 13 | .cache 14 | venv 15 | .idea 16 | .pytest_cache 17 | 18 | # sample code for GitHub issues 19 | issues 20 | .direnv 21 | .envrc 22 | -------------------------------------------------------------------------------- /.readthedocs.yaml: -------------------------------------------------------------------------------- 1 | version: 2 2 | 3 | build: 4 | os: ubuntu-22.04 5 | tools: 6 | python: "3.12" 7 | 8 | python: 9 | install: 10 | - method: pip 11 | path: . 12 | extra_requirements: 13 | - develop 14 | 15 | sphinx: 16 | configuration: docs/conf.py 17 | fail_on_warning: true 18 | -------------------------------------------------------------------------------- /AUTHORS: -------------------------------------------------------------------------------- 1 | For a list of all our amazing authors please see the contributors page: 2 | https://github.com/elastic/elasticsearch-py/graphs/contributors 3 | -------------------------------------------------------------------------------- /CONTRIBUTING.rst: -------------------------------------------------------------------------------- 1 | Contribution Guide 2 | ================== 3 | 4 | If you have a bugfix or new feature that you would like to contribute to 5 | elasticsearch-dsl-py, please find or open an issue about it first. Talk about what 6 | you would like to do. It may be that somebody is already working on it, or that 7 | there are particular issues that you should know about before implementing the 8 | change. 9 | 10 | If you want to be rewarded for your contributions, sign up for the 11 | `Elastic Contributor Program `_. 12 | Each time you make a valid contribution, you’ll earn points that increase your 13 | chances of winning prizes and being recognized as a top contributor. 14 | 15 | We enjoy working with contributors to get their code accepted. There are many 16 | approaches to fixing a problem and it is important to find the best approach 17 | before writing too much code. 18 | 19 | The process for contributing to any of the Elasticsearch repositories is similar. 20 | 21 | 1. Please make sure you have signed the `Contributor License 22 | Agreement `_. We are not 23 | asking you to assign copyright to us, but to give us the right to distribute 24 | your code without restriction. We ask this of all contributors in order to 25 | assure our users of the origin and continuing existence of the code. You only 26 | need to sign the CLA once. 27 | 28 | 2. Many classes included in this library are offered in two versions, for 29 | asynchronous and synchronous Python. When working with these classes, you only 30 | need to make changes to the asynchronous code, located in *_async* 31 | subdirectories in the source and tests trees. Once you've made your changes, 32 | run the following command to automatically generate the corresponding 33 | synchronous code: 34 | 35 | .. code:: bash 36 | 37 | $ nox -rs format 38 | 39 | 3. Run the test suite to ensure your changes do not break existing code: 40 | 41 | .. code:: bash 42 | 43 | $ nox -rs lint test 44 | 45 | 4. Rebase your changes. 46 | Update your local repository with the most recent code from the main 47 | elasticsearch-dsl-py repository, and rebase your branch on top of the latest master 48 | branch. We prefer your changes to be squashed into a single commit. 49 | 50 | 5. Submit a pull request. Push your local changes to your forked copy of the 51 | repository and submit a pull request. In the pull request, describe what your 52 | changes do and mention the number of the issue where discussion has taken 53 | place, eg “Closes #123″. Please consider adding or modifying tests related to 54 | your changes. Include any generated files in the *_sync* subdirectory in your 55 | pull request. 56 | 57 | Then sit back and wait. There will probably be discussion about the pull 58 | request and, if any changes are needed, we would love to work with you to get 59 | your pull request merged into elasticsearch-dsl-py. 60 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include AUTHORS 2 | include LICENSE 3 | include README 4 | include CONTRIBUTING.rst 5 | include Changelog.rst 6 | -------------------------------------------------------------------------------- /README.rst: -------------------------------------------------------------------------------- 1 | README -------------------------------------------------------------------------------- /docs/CONTRIBUTING.rst: -------------------------------------------------------------------------------- 1 | ../CONTRIBUTING.rst -------------------------------------------------------------------------------- /docs/Changelog.rst: -------------------------------------------------------------------------------- 1 | ../Changelog.rst -------------------------------------------------------------------------------- /docs/api.rst: -------------------------------------------------------------------------------- 1 | .. _api: 2 | 3 | API Documentation 4 | ================= 5 | 6 | Below please find the documentation for the public classes and functions of ``elasticsearch_dsl``. 7 | The :ref:`Asynchronous API ` classes are documented separately. 8 | 9 | .. py:module:: elasticsearch_dsl 10 | 11 | .. autoclass:: Search 12 | :inherited-members: 13 | :members: 14 | 15 | .. autoclass:: MultiSearch 16 | :inherited-members: 17 | :members: 18 | 19 | .. autoclass:: Document 20 | :inherited-members: 21 | :members: 22 | 23 | .. autoclass:: Index 24 | :inherited-members: 25 | :members: 26 | 27 | .. autoclass:: FacetedSearch 28 | :inherited-members: 29 | :members: 30 | 31 | .. autoclass:: UpdateByQuery 32 | :inherited-members: 33 | :members: 34 | 35 | Mappings 36 | -------- 37 | 38 | If you wish to create mappings manually you can use the ``Mapping`` class, for 39 | more advanced use cases, however, we recommend you use the :ref:`doc_type` 40 | abstraction in combination with :ref:`index` (or :ref:`index-template`) to define 41 | index-level settings and properties. The mapping definition follows a similar 42 | pattern to the query dsl: 43 | 44 | .. code:: python 45 | 46 | from elasticsearch_dsl import Keyword, Mapping, Nested, Text 47 | 48 | # name your type 49 | m = Mapping() 50 | 51 | # add fields 52 | m.field('title', 'text') 53 | 54 | # you can use multi-fields easily 55 | m.field('category', 'text', fields={'raw': Keyword()}) 56 | 57 | # you can also create a field manually 58 | comment = Nested( 59 | properties={ 60 | 'author': Text(), 61 | 'created_at': Date() 62 | }) 63 | 64 | # and attach it to the mapping 65 | m.field('comments', comment) 66 | 67 | # you can also define mappings for the meta fields 68 | m.meta('_all', enabled=False) 69 | 70 | # save the mapping into index 'my-index' 71 | m.save('my-index') 72 | 73 | .. note:: 74 | 75 | By default all fields (with the exception of ``Nested``) will expect single 76 | values. You can always override this expectation during the field 77 | creation/definition by passing in ``multi=True`` into the constructor 78 | (``m.field('tags', Keyword(multi=True))``). Then the 79 | value of the field, even if the field hasn't been set, will be an empty 80 | list enabling you to write ``doc.tags.append('search')``. 81 | 82 | Especially if you are using dynamic mappings it might be useful to update the 83 | mapping based on an existing type in Elasticsearch, or create the mapping 84 | directly from an existing type: 85 | 86 | .. code:: python 87 | 88 | # get the mapping from our production cluster 89 | m = Mapping.from_es('my-index', using='prod') 90 | 91 | # update based on data in QA cluster 92 | m.update_from_es('my-index', using='qa') 93 | 94 | # update the mapping on production 95 | m.save('my-index', using='prod') 96 | 97 | Common field options: 98 | 99 | ``multi`` 100 | If set to ``True`` the field's value will be set to ``[]`` at first access. 101 | 102 | ``required`` 103 | Indicates if a field requires a value for the document to be valid. 104 | -------------------------------------------------------------------------------- /docs/async_api.rst: -------------------------------------------------------------------------------- 1 | .. _async_api: 2 | 3 | Asynchronous API Documentation 4 | ============================== 5 | 6 | Below please find the documentation for the asychronous classes of ``elasticsearch_dsl``. 7 | 8 | .. py:module:: elasticsearch_dsl 9 | :no-index: 10 | 11 | .. autoclass:: AsyncSearch 12 | :inherited-members: 13 | :members: 14 | 15 | .. autoclass:: AsyncMultiSearch 16 | :inherited-members: 17 | :members: 18 | 19 | .. autoclass:: AsyncDocument 20 | :inherited-members: 21 | :members: 22 | 23 | .. autoclass:: AsyncIndex 24 | :inherited-members: 25 | :members: 26 | 27 | .. autoclass:: AsyncFacetedSearch 28 | :inherited-members: 29 | :members: 30 | 31 | .. autoclass:: AsyncUpdateByQuery 32 | :inherited-members: 33 | :members: 34 | -------------------------------------------------------------------------------- /docs/asyncio.rst: -------------------------------------------------------------------------------- 1 | .. _asyncio: 2 | 3 | Using asyncio with Elasticsearch DSL 4 | ==================================== 5 | 6 | The ``elasticsearch-dsl`` package supports async/await with `asyncio `__. 7 | To ensure that you have all the required dependencies, install the ``[async]`` extra: 8 | 9 | .. code:: bash 10 | 11 | $ python -m pip install elasticsearch-dsl[async] 12 | 13 | Connections 14 | ----------- 15 | 16 | Use the ``async_connections`` module to manage your asynchronous connections. 17 | 18 | .. code:: python 19 | 20 | from elasticsearch_dsl import async_connections 21 | 22 | async_connections.create_connection(hosts=['localhost'], timeout=20) 23 | 24 | All the options available in the ``connections`` module can be used with ``async_connections``. 25 | 26 | How to avoid 'Unclosed client session / connector' warnings on exit 27 | ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 28 | 29 | These warnings come from the ``aiohttp`` package, which is used internally by the 30 | ``AsyncElasticsearch`` client. They appear often when the application exits and 31 | are caused by HTTP connections that are open when they are garbage collected. To 32 | avoid these warnings, make sure that you close your connections. 33 | 34 | .. code:: python 35 | 36 | es = async_connections.get_connection() 37 | await es.close() 38 | 39 | Search DSL 40 | ---------- 41 | 42 | Use the ``AsyncSearch`` class to perform asynchronous searches. 43 | 44 | .. code:: python 45 | 46 | from elasticsearch_dsl import AsyncSearch 47 | 48 | s = AsyncSearch().query("match", title="python") 49 | async for hit in s: 50 | print(hit.title) 51 | 52 | Instead of using the ``AsyncSearch`` object as an asynchronous iterator, you can 53 | explicitly call the ``execute()`` method to get a ``Response`` object. 54 | 55 | .. code:: python 56 | 57 | s = AsyncSearch().query("match", title="python") 58 | response = await s.execute() 59 | for hit in response: 60 | print(hit.title) 61 | 62 | An ``AsyncMultiSearch`` is available as well. 63 | 64 | .. code:: python 65 | 66 | from elasticsearch_dsl import AsyncMultiSearch 67 | 68 | ms = AsyncMultiSearch(index='blogs') 69 | 70 | ms = ms.add(AsyncSearch().filter('term', tags='python')) 71 | ms = ms.add(AsyncSearch().filter('term', tags='elasticsearch')) 72 | 73 | responses = await ms.execute() 74 | 75 | for response in responses: 76 | print("Results for query %r." % response.search.query) 77 | for hit in response: 78 | print(hit.title) 79 | 80 | Asynchronous Documents, Indexes, and more 81 | ----------------------------------------- 82 | 83 | The ``Document``, ``Index``, ``IndexTemplate``, ``Mapping``, ``UpdateByQuery`` and 84 | ``FacetedSearch`` classes all have asynchronous versions that use the same name 85 | with an ``Async`` prefix. These classes expose the same interfaces as the 86 | synchronous versions, but any methods that perform I/O are defined as coroutines. 87 | 88 | Auxiliary classes that do not perform I/O do not have asynchronous versions. The 89 | same classes can be used in synchronous and asynchronous applications. 90 | 91 | When using a :ref:`custom analyzer ` in an asynchronous application, use 92 | the ``async_simulate()`` method to invoke the Analyze API on it. 93 | 94 | Consult the :ref:`api` section for details about each specific method. 95 | -------------------------------------------------------------------------------- /docs/configuration.rst: -------------------------------------------------------------------------------- 1 | .. _configuration: 2 | 3 | Configuration 4 | ============= 5 | 6 | There are several ways to configure connections for the library. The easiest 7 | and most useful approach is to define one default connection that can be 8 | used every time an API call is made without explicitly passing in other 9 | connections. 10 | 11 | .. note:: 12 | 13 | Unless you want to access multiple clusters from your application, it is 14 | highly recommended that you use the ``create_connection`` method and all 15 | operations will use that connection automatically. 16 | 17 | .. _default connection: 18 | 19 | Default connection 20 | ------------------ 21 | 22 | To define a default connection that can be used globally, use the 23 | ``connections`` module and the ``create_connection`` method like this: 24 | 25 | .. code:: python 26 | 27 | from elasticsearch_dsl import connections 28 | 29 | connections.create_connection(hosts=['localhost'], timeout=20) 30 | 31 | Single connection with an alias 32 | ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 33 | 34 | You can define the ``alias`` or name of a connection so you can easily 35 | refer to it later. The default value for ``alias`` is ``default``. 36 | 37 | .. code:: python 38 | 39 | from elasticsearch_dsl import connections 40 | 41 | connections.create_connection(alias='my_new_connection', hosts=['localhost'], timeout=60) 42 | 43 | Additional keyword arguments (``hosts`` and ``timeout`` in our example) will be passed 44 | to the ``Elasticsearch`` class from ``elasticsearch-py``. 45 | 46 | To see all 47 | possible configuration options refer to the `documentation 48 | `_. 49 | 50 | Multiple clusters 51 | ----------------- 52 | 53 | You can define multiple connections to multiple clusters at the same 54 | time using the ``configure`` method: 55 | 56 | .. code:: python 57 | 58 | from elasticsearch_dsl import connections 59 | 60 | connections.configure( 61 | default={'hosts': 'localhost'}, 62 | dev={ 63 | 'hosts': ['esdev1.example.com:9200'], 64 | 'sniff_on_start': True 65 | } 66 | ) 67 | 68 | Such connections will be constructed lazily when requested for the first time. 69 | 70 | You can alternatively define multiple connections by adding them one by one 71 | as shown in the following example: 72 | 73 | .. code:: python 74 | 75 | # if you have configuration options to be passed to Elasticsearch.__init__ 76 | # this also shows creating a connection with the alias 'qa' 77 | connections.create_connection('qa', hosts=['esqa1.example.com'], sniff_on_start=True) 78 | 79 | # if you already have an Elasticsearch instance ready 80 | connections.add_connection('another_qa', my_client) 81 | 82 | Using aliases 83 | ~~~~~~~~~~~~~ 84 | 85 | When using multiple connections, you can refer to them using the string 86 | alias specified when you created the connection. 87 | 88 | This example shows how to use an alias to a connection: 89 | 90 | .. code:: python 91 | 92 | s = Search(using='qa') 93 | 94 | A ``KeyError`` will be raised if there is no connection registered with that 95 | alias. 96 | 97 | Manual 98 | ------ 99 | 100 | If you don't want to supply a global configuration, you can always pass in your 101 | own connection as an instance of ``elasticsearch.Elasticsearch`` with the parameter 102 | ``using`` wherever it is accepted like this: 103 | 104 | .. code:: python 105 | 106 | s = Search(using=Elasticsearch('localhost')) 107 | 108 | You can even use this approach to override any connection the object might be 109 | already associated with: 110 | 111 | .. code:: python 112 | 113 | s = s.using(Elasticsearch('otherhost:9200')) 114 | 115 | .. note:: 116 | 117 | When using ``elasticsearch_dsl``, it is highly recommended that you use the built-in 118 | serializer (``elasticsearch_dsl.serializer.serializer``) to ensure 119 | your objects are correctly serialized into ``JSON`` every time. The 120 | ``create_connection`` method that is described here (and that the ``configure`` 121 | method uses under the hood) will do that automatically for you, unless you 122 | explicitly specify your own serializer. The built-in serializer also allows 123 | you to serialize your own objects - just define a ``to_dict()`` method on your 124 | objects and that method will be automatically called when serializing your custom 125 | objects to ``JSON``. 126 | -------------------------------------------------------------------------------- /docs/faceted_search.rst: -------------------------------------------------------------------------------- 1 | .. _faceted_search: 2 | 3 | Faceted Search 4 | ============== 5 | 6 | The library comes with a simple abstraction aimed at helping you develop 7 | faceted navigation for your data. 8 | 9 | .. note:: 10 | 11 | This API is experimental and will be subject to change. Any feedback is 12 | welcome. 13 | 14 | Configuration 15 | ------------- 16 | 17 | You can provide several configuration options (as class attributes) when 18 | declaring a ``FacetedSearch`` subclass: 19 | 20 | ``index`` 21 | the name of the index (as string) to search through, defaults to ``'_all'``. 22 | 23 | ``doc_types`` 24 | list of ``Document`` subclasses or strings to be used, defaults to 25 | ``['_all']``. 26 | 27 | ``fields`` 28 | list of fields on the document type to search through. The list will be 29 | passes to ``MultiMatch`` query so can contain boost values (``'title^5'``), 30 | defaults to ``['*']``. 31 | 32 | ``facets`` 33 | dictionary of facets to display/filter on. The key is the name displayed and 34 | values should be instances of any ``Facet`` subclass, for example: ``{'tags': 35 | TermsFacet(field='tags')}`` 36 | 37 | 38 | Facets 39 | ~~~~~~ 40 | 41 | There are several different facets available: 42 | 43 | ``TermsFacet`` 44 | provides an option to split documents into groups based on a value of a field, for example ``TermsFacet(field='category')`` 45 | 46 | ``DateHistogramFacet`` 47 | split documents into time intervals, example: ``DateHistogramFacet(field="published_date", calendar_interval="day")`` 48 | 49 | ``HistogramFacet`` 50 | similar to ``DateHistogramFacet`` but for numerical values: ``HistogramFacet(field="rating", interval=2)`` 51 | 52 | ``RangeFacet`` 53 | allows you to define your own ranges for a numerical fields: 54 | ``RangeFacet(field="comment_count", ranges=[("few", (None, 2)), ("lots", (2, None))])`` 55 | 56 | ``NestedFacet`` 57 | is just a simple facet that wraps another to provide access to nested documents: 58 | ``NestedFacet('variants', TermsFacet(field='variants.color'))`` 59 | 60 | 61 | By default facet results will only calculate document count, if you wish for 62 | a different metric you can pass in any single value metric aggregation as the 63 | ``metric`` kwarg (``TermsFacet(field='tags', metric=A('max', 64 | field=timestamp))``). When specifying ``metric`` the results will be, by 65 | default, sorted in descending order by that metric. To change it to ascending 66 | specify ``metric_sort="asc"`` and to just sort by document count use 67 | ``metric_sort=False``. 68 | 69 | Advanced 70 | ~~~~~~~~ 71 | 72 | If you require any custom behavior or modifications simply override one or more 73 | of the methods responsible for the class' functions: 74 | 75 | ``search(self)`` 76 | is responsible for constructing the ``Search`` object used. Override this if 77 | you want to customize the search object (for example by adding a global 78 | filter for published articles only). 79 | 80 | ``query(self, search)`` 81 | adds the query position of the search (if search input specified), by default 82 | using ``MultiField`` query. Override this if you want to modify the query type used. 83 | 84 | ``highlight(self, search)`` 85 | defines the highlighting on the ``Search`` object and returns a new one. 86 | Default behavior is to highlight on all fields specified for search. 87 | 88 | 89 | Usage 90 | ----- 91 | 92 | The custom subclass can be instantiated empty to provide an empty search 93 | (matching everything) or with ``query``, ``filters`` and ``sort``. 94 | 95 | ``query`` 96 | is used to pass in the text of the query to be performed. If ``None`` is 97 | passed in (default) a ``MatchAll`` query will be used. For example ``'python 98 | web'`` 99 | 100 | ``filters`` 101 | is a dictionary containing all the facet filters that you wish to apply. Use 102 | the name of the facet (from ``.facets`` attribute) as the key and one of the 103 | possible values as value. For example ``{'tags': 'python'}``. 104 | 105 | ``sort`` 106 | is a tuple or list of fields on which the results should be sorted. The format 107 | of the individual fields are to be the same as those passed to 108 | :meth:`~elasticsearch_dsl.Search.sort`. 109 | 110 | 111 | Response 112 | ~~~~~~~~ 113 | 114 | the response returned from the ``FacetedSearch`` object (by calling 115 | ``.execute()``) is a subclass of the standard ``Response`` class that adds a 116 | property called ``facets`` which contains a dictionary with lists of buckets - 117 | each represented by a tuple of key, document count and a flag indicating 118 | whether this value has been filtered on. 119 | 120 | Example 121 | ------- 122 | 123 | .. code:: python 124 | 125 | from datetime import date 126 | 127 | from elasticsearch_dsl import FacetedSearch, TermsFacet, DateHistogramFacet 128 | 129 | class BlogSearch(FacetedSearch): 130 | doc_types = [Article, ] 131 | # fields that should be searched 132 | fields = ['tags', 'title', 'body'] 133 | 134 | facets = { 135 | # use bucket aggregations to define facets 136 | 'tags': TermsFacet(field='tags'), 137 | 'publishing_frequency': DateHistogramFacet(field='published_from', interval='month') 138 | } 139 | 140 | def search(self): 141 | # override methods to add custom pieces 142 | s = super().search() 143 | return s.filter('range', publish_from={'lte': 'now/h'}) 144 | 145 | bs = BlogSearch('python web', {'publishing_frequency': date(2015, 6)}) 146 | response = bs.execute() 147 | 148 | # access hits and other attributes as usual 149 | total = response.hits.total 150 | print('total hits', total.relation, total.value) 151 | for hit in response: 152 | print(hit.meta.score, hit.title) 153 | 154 | for (tag, count, selected) in response.facets.tags: 155 | print(tag, ' (SELECTED):' if selected else ':', count) 156 | 157 | for (month, count, selected) in response.facets.publishing_frequency: 158 | print(month.strftime('%B %Y'), ' (SELECTED):' if selected else ':', count) 159 | 160 | 161 | -------------------------------------------------------------------------------- /docs/index.rst: -------------------------------------------------------------------------------- 1 | Elasticsearch DSL 2 | ================= 3 | 4 | .. note:: 5 | 6 | As of release 8.18.0, the Elasticsearch DSL package is part of the official 7 | `Elasticsearch Python client `_, 8 | so a separate install is not needed anymore. To migrate, follow these steps: 9 | 10 | - Uninstall the ``elasticsearch-dsl`` package 11 | - Make sure you have version 8.18.0 or newer of the ``elasticsearch`` package 12 | installed 13 | - Replace ``elasticsearch_dsl`` with ``elasticsearch.dsl`` in your imports 14 | 15 | To prevent applications from breaking unexpectedly due to this change, the 16 | 8.18.0 release of the ``elasticsearch-dsl`` package automatically redirects 17 | all imports to the corresponding modules of the Python client package. 18 | 19 | This documentation site applies to releases before 8.18.0. For newer releases 20 | use the following links: 21 | 22 | - `8.x releases starting with 8.18.0 `_ 23 | - `9.x releases `_ 24 | 25 | Elasticsearch DSL is a high-level library whose aim is to help with writing and 26 | running queries against Elasticsearch. It is built on top of the official 27 | low-level client (`elasticsearch-py `_). 28 | 29 | It provides a more convenient and idiomatic way to write and manipulate 30 | queries. It stays close to the Elasticsearch JSON DSL, mirroring its 31 | terminology and structure. It exposes the whole range of the DSL from Python 32 | either directly using defined classes or a queryset-like expressions. Here is 33 | an example:: 34 | 35 | from elasticsearch_dsl import Search 36 | 37 | s = Search(index="my-index") \ 38 | .filter("term", category="search") \ 39 | .query("match", title="python") \ 40 | .exclude("match", description="beta") 41 | for hit in s: 42 | print(hit.title) 43 | 44 | Or with asynchronous Python:: 45 | 46 | from elasticsearch_dsl import AsyncSearch 47 | 48 | async def run_query(): 49 | s = AsyncSearch(index="my-index") \ 50 | .filter("term", category="search") \ 51 | .query("match", title="python") \ 52 | .exclude("match", description="beta") 53 | async for hit in s: 54 | print(hit.title) 55 | 56 | It also provides an optional wrapper for working with documents as Python 57 | objects: defining mappings, retrieving and saving documents, wrapping the 58 | document data in user-defined classes. 59 | 60 | To use the other Elasticsearch APIs (eg. cluster health) just use the 61 | underlying client. 62 | 63 | Installation 64 | ------------ 65 | 66 | Install the ``elasticsearch-dsl`` package with `pip `_:: 67 | 68 | pip install elasticsearch-dsl 69 | 70 | For asynchronous applications, install with the ``async`` extra:: 71 | 72 | pip install elasticsearch-dsl[async] 73 | 74 | Read more about :ref:`how to use asyncio with this project `. 75 | 76 | Examples 77 | -------- 78 | 79 | Please see the `examples 80 | `_ 81 | directory to see some complex examples using ``elasticsearch-dsl``. 82 | 83 | Compatibility 84 | ------------- 85 | 86 | The library is compatible with all Elasticsearch versions since ``2.x`` but you 87 | **have to use a matching major version**: 88 | 89 | For **Elasticsearch 8.0** and later, use the major version 8 (``8.x.y``) of the 90 | library. 91 | 92 | For **Elasticsearch 7.0** and later, use the major version 7 (``7.x.y``) of the 93 | library. 94 | 95 | For **Elasticsearch 6.0** and later, use the major version 6 (``6.x.y``) of the 96 | library. 97 | 98 | For **Elasticsearch 5.0** and later, use the major version 5 (``5.x.y``) of the 99 | library. 100 | 101 | For **Elasticsearch 2.0** and later, use the major version 2 (``2.x.y``) of the 102 | library. 103 | 104 | The recommended way to set your requirements in your `setup.py` or 105 | `requirements.txt` is:: 106 | 107 | # Elasticsearch 8.x 108 | elasticsearch-dsl>=8.0.0,<9.0.0 109 | 110 | # Elasticsearch 7.x 111 | elasticsearch-dsl>=7.0.0,<8.0.0 112 | 113 | # Elasticsearch 6.x 114 | elasticsearch-dsl>=6.0.0,<7.0.0 115 | 116 | # Elasticsearch 5.x 117 | elasticsearch-dsl>=5.0.0,<6.0.0 118 | 119 | # Elasticsearch 2.x 120 | elasticsearch-dsl>=2.0.0,<3.0.0 121 | 122 | 123 | The development is happening on ``main``, older branches only get bugfix releases 124 | 125 | License 126 | ------- 127 | 128 | Copyright 2013 Elasticsearch 129 | 130 | Licensed under the Apache License, Version 2.0 (the "License"); 131 | you may not use this file except in compliance with the License. 132 | You may obtain a copy of the License at 133 | 134 | http://www.apache.org/licenses/LICENSE-2.0 135 | 136 | Unless required by applicable law or agreed to in writing, software 137 | distributed under the License is distributed on an "AS IS" BASIS, 138 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 139 | See the License for the specific language governing permissions and 140 | limitations under the License. 141 | 142 | Contents 143 | -------- 144 | 145 | .. toctree:: 146 | :caption: About 147 | :maxdepth: 2 148 | 149 | self 150 | configuration 151 | 152 | .. toctree:: 153 | :caption: Tutorials 154 | :maxdepth: 2 155 | 156 | tutorials 157 | 158 | .. toctree:: 159 | :caption: How-To Guides 160 | :maxdepth: 2 161 | 162 | search_dsl 163 | persistence 164 | faceted_search 165 | update_by_query 166 | asyncio 167 | 168 | .. toctree:: 169 | :caption: Reference 170 | :maxdepth: 2 171 | 172 | api 173 | async_api 174 | 175 | .. toctree:: 176 | :caption: Community 177 | :maxdepth: 2 178 | 179 | CONTRIBUTING 180 | Changelog 181 | -------------------------------------------------------------------------------- /docs/update_by_query.rst: -------------------------------------------------------------------------------- 1 | .. _update_by_query: 2 | 3 | Update By Query 4 | ================ 5 | 6 | The ``Update By Query`` object 7 | ------------------------------- 8 | 9 | The ``Update By Query`` object enables the use of the 10 | `_update_by_query `_ 11 | endpoint to perform an update on documents that match a search query. 12 | 13 | The object is implemented as a modification of the ``Search`` object, containing a 14 | subset of its query methods, as well as a script method, which is used to make updates. 15 | 16 | The ``Update By Query`` object implements the following ``Search`` query types: 17 | 18 | * queries 19 | 20 | * filters 21 | 22 | * excludes 23 | 24 | For more information on queries, see the :ref:`search_dsl` chapter. 25 | 26 | Like the ``Search`` object, the API is designed to be chainable. This means that the ``Update By Query`` object 27 | is immutable: all changes to the object will result in a shallow copy being created which 28 | contains the changes. This means you can safely pass the ``Update By Query`` object to 29 | foreign code without fear of it modifying your objects as long as it sticks to 30 | the ``Update By Query`` object APIs. 31 | 32 | You can define your client in a number of ways, but the preferred method is to use a global configuration. 33 | For more information on defining a client, see the :ref:`configuration` chapter. 34 | 35 | Once your client is defined, you can instantiate a copy of the ``Update By Query`` object as seen below: 36 | 37 | .. code:: python 38 | 39 | from elasticsearch_dsl import UpdateByQuery 40 | 41 | ubq = UpdateByQuery().using(client) 42 | # or 43 | ubq = UpdateByQuery(using=client) 44 | 45 | .. note:: 46 | 47 | All methods return a *copy* of the object, making it safe to pass to 48 | outside code. 49 | 50 | The API is chainable, allowing you to combine multiple method calls in one 51 | statement: 52 | 53 | .. code:: python 54 | 55 | ubq = UpdateByQuery().using(client).query("match", title="python") 56 | 57 | To send the request to Elasticsearch: 58 | 59 | .. code:: python 60 | 61 | response = ubq.execute() 62 | 63 | It should be noted, that there are limits to the chaining using the script method: calling script multiple times will 64 | overwrite the previous value. That is, only a single script can be sent with a call. An attempt to use two scripts will 65 | result in only the second script being stored. 66 | 67 | Given the below example: 68 | 69 | .. code:: python 70 | 71 | ubq = UpdateByQuery().using(client).script(source="ctx._source.likes++").script(source="ctx._source.likes+=2") 72 | 73 | This means that the stored script by this client will be ``'source': 'ctx._source.likes+=2'`` and the previous call 74 | will not be stored. 75 | 76 | For debugging purposes you can serialize the ``Update By Query`` object to a ``dict`` 77 | explicitly: 78 | 79 | .. code:: python 80 | 81 | print(ubq.to_dict()) 82 | 83 | Also, to use variables in script see below example: 84 | 85 | .. code:: python 86 | 87 | ubq.script( 88 | source="ctx._source.messages.removeIf(x -> x.somefield == params.some_var)", 89 | params={ 90 | 'some_var': 'some_string_val' 91 | } 92 | ) 93 | 94 | Serialization and Deserialization 95 | ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 96 | 97 | The search object can be serialized into a dictionary by using the 98 | ``.to_dict()`` method. 99 | 100 | You can also create a ``Update By Query`` object from a ``dict`` using the ``from_dict`` 101 | class method. This will create a new ``Update By Query`` object and populate it using 102 | the data from the dict: 103 | 104 | .. code:: python 105 | 106 | ubq = UpdateByQuery.from_dict({"query": {"match": {"title": "python"}}}) 107 | 108 | If you wish to modify an existing ``Update By Query`` object, overriding it's 109 | properties, instead use the ``update_from_dict`` method that alters an instance 110 | **in-place**: 111 | 112 | .. code:: python 113 | 114 | ubq = UpdateByQuery(index='i') 115 | ubq.update_from_dict({"query": {"match": {"title": "python"}}, "size": 42}) 116 | 117 | Extra properties and parameters 118 | ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 119 | 120 | To set extra properties of the search request, use the ``.extra()`` method. 121 | This can be used to define keys in the body that cannot be defined via a 122 | specific API method like ``explain``: 123 | 124 | .. code:: python 125 | 126 | ubq = ubq.extra(explain=True) 127 | 128 | To set query parameters, use the ``.params()`` method: 129 | 130 | .. code:: python 131 | 132 | ubq = ubq.params(routing="42") 133 | 134 | Response 135 | -------- 136 | 137 | You can execute your search by calling the ``.execute()`` method that will return 138 | a ``Response`` object. The ``Response`` object allows you access to any key 139 | from the response dictionary via attribute access. It also provides some 140 | convenient helpers: 141 | 142 | .. code:: python 143 | 144 | response = ubq.execute() 145 | 146 | print(response.success()) 147 | # True 148 | 149 | print(response.took) 150 | # 12 151 | 152 | If you want to inspect the contents of the ``response`` objects, just use its 153 | ``to_dict`` method to get access to the raw data for pretty printing. 154 | -------------------------------------------------------------------------------- /elasticsearch_dsl/__init__.py: -------------------------------------------------------------------------------- 1 | # Licensed to Elasticsearch B.V. under one or more contributor 2 | # license agreements. See the NOTICE file distributed with 3 | # this work for additional information regarding copyright 4 | # ownership. Elasticsearch B.V. licenses this file to you under 5 | # the Apache License, Version 2.0 (the "License"); you may 6 | # not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | import sys 19 | 20 | from elasticsearch import __version__, dsl # noqa: F401 21 | 22 | modules = [mod for mod in sys.modules.keys() if mod.startswith("elasticsearch.dsl")] 23 | for mod in modules: 24 | sys.modules[mod.replace("elasticsearch.dsl", "elasticsearch_dsl")] = sys.modules[ 25 | mod 26 | ] 27 | sys.modules["elasticsearch_dsl"].VERSION = __version__ 28 | sys.modules["elasticsearch_dsl"].__versionstr__ = ".".join(map(str, __version__)) 29 | -------------------------------------------------------------------------------- /examples/README.rst: -------------------------------------------------------------------------------- 1 | Elasticsearch DSL Examples 2 | ========================== 3 | 4 | In this directory you can see several complete examples demonstrating key 5 | concepts and patterns exposed by ``elasticsearch-dsl``. 6 | 7 | ``alias_migration.py`` 8 | ---------------------- 9 | 10 | The alias migration example shows a useful pattern where we use versioned 11 | indices (``test-blog-0``, ``test-blog-1``, ...) to manage schema changes and 12 | hides that behind an alias so that the application doesn't have to be aware of 13 | the versions and just refer to the ``test-blog`` alias for both read and write 14 | operations. 15 | 16 | For simplicity we use a timestamp as version in the index name. 17 | 18 | ``parent_child.py`` 19 | ------------------- 20 | 21 | More complex example highlighting the possible relationships available in 22 | elasticsearch - `parent/child 23 | `_ and 24 | `nested 25 | `_. 26 | 27 | ``composite_agg.py`` 28 | -------------------- 29 | 30 | A helper function using the `composite aggregation 31 | `_ 32 | to paginate over aggregation results. 33 | 34 | ``percolate.py`` 35 | ---------------- 36 | 37 | A ``BlogPost`` document with automatic classification using the `percolator 38 | `_ 39 | functionality. 40 | 41 | ``completion.py`` 42 | ----------------- 43 | 44 | As example using `completion suggester 45 | `_ 46 | to auto complete people's names. 47 | 48 | -------------------------------------------------------------------------------- /examples/alias_migration.py: -------------------------------------------------------------------------------- 1 | # Licensed to Elasticsearch B.V. under one or more contributor 2 | # license agreements. See the NOTICE file distributed with 3 | # this work for additional information regarding copyright 4 | # ownership. Elasticsearch B.V. licenses this file to you under 5 | # the Apache License, Version 2.0 (the "License"); you may 6 | # not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | """ 19 | Simple example with a single Document demonstrating how schema can be managed, 20 | including upgrading with reindexing. 21 | 22 | Key concepts: 23 | 24 | * setup() function to first initialize the schema (as index template) in 25 | elasticsearch. Can be called any time (recommended with every deploy of 26 | your app). 27 | 28 | * migrate() function to be called any time when the schema changes - it 29 | will create a new index (by incrementing the version) and update the alias. 30 | By default it will also (before flipping the alias) move the data from the 31 | previous index to the new one. 32 | 33 | * BlogPost._matches() class method is required for this code to work since 34 | otherwise BlogPost will not be used to deserialize the documents as those 35 | will have index set to the concrete index whereas the class refers to the 36 | alias. 37 | """ 38 | import os 39 | from datetime import datetime 40 | from fnmatch import fnmatch 41 | from typing import TYPE_CHECKING, Any, Dict, List, Optional 42 | 43 | from elasticsearch_dsl import Document, Keyword, connections, mapped_field 44 | 45 | ALIAS = "test-blog" 46 | PATTERN = ALIAS + "-*" 47 | PRIORITY = 100 48 | 49 | 50 | class BlogPost(Document): 51 | if TYPE_CHECKING: 52 | # definitions here help type checkers understand additional arguments 53 | # that are allowed in the constructor 54 | _id: int 55 | 56 | title: str 57 | tags: List[str] = mapped_field(Keyword()) 58 | content: str 59 | published: Optional[datetime] = mapped_field(default=None) 60 | 61 | def is_published(self) -> bool: 62 | return bool(self.published and datetime.now() > self.published) 63 | 64 | @classmethod 65 | def _matches(cls, hit: Dict[str, Any]) -> bool: 66 | # override _matches to match indices in a pattern instead of just ALIAS 67 | # hit is the raw dict as returned by elasticsearch 68 | return fnmatch(hit["_index"], PATTERN) 69 | 70 | class Index: 71 | # we will use an alias instead of the index 72 | name = ALIAS 73 | # set settings and possibly other attributes of the index like 74 | # analyzers 75 | settings = {"number_of_shards": 1, "number_of_replicas": 0} 76 | 77 | 78 | def setup() -> None: 79 | """ 80 | Create the index template in elasticsearch specifying the mappings and any 81 | settings to be used. This can be run at any time, ideally at every new code 82 | deploy. 83 | """ 84 | # create an index template 85 | index_template = BlogPost._index.as_composable_template( 86 | ALIAS, PATTERN, priority=PRIORITY 87 | ) 88 | # upload the template into elasticsearch 89 | # potentially overriding the one already there 90 | index_template.save() 91 | 92 | # create the first index if it doesn't exist 93 | if not BlogPost._index.exists(): 94 | migrate(move_data=False) 95 | 96 | 97 | def migrate(move_data: bool = True, update_alias: bool = True) -> None: 98 | """ 99 | Upgrade function that creates a new index for the data. Optionally it also can 100 | (and by default will) reindex previous copy of the data into the new index 101 | (specify ``move_data=False`` to skip this step) and update the alias to 102 | point to the latest index (set ``update_alias=False`` to skip). 103 | 104 | Note that while this function is running the application can still perform 105 | any and all searches without any loss of functionality. It should, however, 106 | not perform any writes at this time as those might be lost. 107 | """ 108 | # construct a new index name by appending current timestamp 109 | next_index = PATTERN.replace("*", datetime.now().strftime("%Y%m%d%H%M%S%f")) 110 | 111 | # get the low level connection 112 | es = connections.get_connection() 113 | 114 | # create new index, it will use the settings from the template 115 | es.indices.create(index=next_index) 116 | 117 | if move_data: 118 | # move data from current alias to the new index 119 | es.options(request_timeout=3600).reindex( 120 | body={"source": {"index": ALIAS}, "dest": {"index": next_index}} 121 | ) 122 | # refresh the index to make the changes visible 123 | es.indices.refresh(index=next_index) 124 | 125 | if update_alias: 126 | # repoint the alias to point to the newly created index 127 | es.indices.update_aliases( 128 | body={ 129 | "actions": [ 130 | {"remove": {"alias": ALIAS, "index": PATTERN}}, 131 | {"add": {"alias": ALIAS, "index": next_index}}, 132 | ] 133 | } 134 | ) 135 | 136 | 137 | def main() -> None: 138 | # initiate the default connection to elasticsearch 139 | connections.create_connection(hosts=[os.environ["ELASTICSEARCH_URL"]]) 140 | 141 | # create the empty index 142 | setup() 143 | 144 | # create a new document 145 | bp = BlogPost( 146 | _id=0, 147 | title="Hello World!", 148 | tags=["testing", "dummy"], 149 | content=open(__file__).read(), 150 | ) 151 | bp.save(refresh=True) 152 | 153 | # create new index 154 | migrate() 155 | 156 | # close the connection 157 | connections.get_connection().close() 158 | 159 | 160 | if __name__ == "__main__": 161 | main() 162 | -------------------------------------------------------------------------------- /examples/async/alias_migration.py: -------------------------------------------------------------------------------- 1 | # Licensed to Elasticsearch B.V. under one or more contributor 2 | # license agreements. See the NOTICE file distributed with 3 | # this work for additional information regarding copyright 4 | # ownership. Elasticsearch B.V. licenses this file to you under 5 | # the Apache License, Version 2.0 (the "License"); you may 6 | # not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | """ 19 | Simple example with a single Document demonstrating how schema can be managed, 20 | including upgrading with reindexing. 21 | 22 | Key concepts: 23 | 24 | * setup() function to first initialize the schema (as index template) in 25 | elasticsearch. Can be called any time (recommended with every deploy of 26 | your app). 27 | 28 | * migrate() function to be called any time when the schema changes - it 29 | will create a new index (by incrementing the version) and update the alias. 30 | By default it will also (before flipping the alias) move the data from the 31 | previous index to the new one. 32 | 33 | * BlogPost._matches() class method is required for this code to work since 34 | otherwise BlogPost will not be used to deserialize the documents as those 35 | will have index set to the concrete index whereas the class refers to the 36 | alias. 37 | """ 38 | import asyncio 39 | import os 40 | from datetime import datetime 41 | from fnmatch import fnmatch 42 | from typing import TYPE_CHECKING, Any, Dict, List, Optional 43 | 44 | from elasticsearch_dsl import AsyncDocument, Keyword, async_connections, mapped_field 45 | 46 | ALIAS = "test-blog" 47 | PATTERN = ALIAS + "-*" 48 | PRIORITY = 100 49 | 50 | 51 | class BlogPost(AsyncDocument): 52 | if TYPE_CHECKING: 53 | # definitions here help type checkers understand additional arguments 54 | # that are allowed in the constructor 55 | _id: int 56 | 57 | title: str 58 | tags: List[str] = mapped_field(Keyword()) 59 | content: str 60 | published: Optional[datetime] = mapped_field(default=None) 61 | 62 | def is_published(self) -> bool: 63 | return bool(self.published and datetime.now() > self.published) 64 | 65 | @classmethod 66 | def _matches(cls, hit: Dict[str, Any]) -> bool: 67 | # override _matches to match indices in a pattern instead of just ALIAS 68 | # hit is the raw dict as returned by elasticsearch 69 | return fnmatch(hit["_index"], PATTERN) 70 | 71 | class Index: 72 | # we will use an alias instead of the index 73 | name = ALIAS 74 | # set settings and possibly other attributes of the index like 75 | # analyzers 76 | settings = {"number_of_shards": 1, "number_of_replicas": 0} 77 | 78 | 79 | async def setup() -> None: 80 | """ 81 | Create the index template in elasticsearch specifying the mappings and any 82 | settings to be used. This can be run at any time, ideally at every new code 83 | deploy. 84 | """ 85 | # create an index template 86 | index_template = BlogPost._index.as_composable_template( 87 | ALIAS, PATTERN, priority=PRIORITY 88 | ) 89 | # upload the template into elasticsearch 90 | # potentially overriding the one already there 91 | await index_template.save() 92 | 93 | # create the first index if it doesn't exist 94 | if not await BlogPost._index.exists(): 95 | await migrate(move_data=False) 96 | 97 | 98 | async def migrate(move_data: bool = True, update_alias: bool = True) -> None: 99 | """ 100 | Upgrade function that creates a new index for the data. Optionally it also can 101 | (and by default will) reindex previous copy of the data into the new index 102 | (specify ``move_data=False`` to skip this step) and update the alias to 103 | point to the latest index (set ``update_alias=False`` to skip). 104 | 105 | Note that while this function is running the application can still perform 106 | any and all searches without any loss of functionality. It should, however, 107 | not perform any writes at this time as those might be lost. 108 | """ 109 | # construct a new index name by appending current timestamp 110 | next_index = PATTERN.replace("*", datetime.now().strftime("%Y%m%d%H%M%S%f")) 111 | 112 | # get the low level connection 113 | es = async_connections.get_connection() 114 | 115 | # create new index, it will use the settings from the template 116 | await es.indices.create(index=next_index) 117 | 118 | if move_data: 119 | # move data from current alias to the new index 120 | await es.options(request_timeout=3600).reindex( 121 | body={"source": {"index": ALIAS}, "dest": {"index": next_index}} 122 | ) 123 | # refresh the index to make the changes visible 124 | await es.indices.refresh(index=next_index) 125 | 126 | if update_alias: 127 | # repoint the alias to point to the newly created index 128 | await es.indices.update_aliases( 129 | body={ 130 | "actions": [ 131 | {"remove": {"alias": ALIAS, "index": PATTERN}}, 132 | {"add": {"alias": ALIAS, "index": next_index}}, 133 | ] 134 | } 135 | ) 136 | 137 | 138 | async def main() -> None: 139 | # initiate the default connection to elasticsearch 140 | async_connections.create_connection(hosts=[os.environ["ELASTICSEARCH_URL"]]) 141 | 142 | # create the empty index 143 | await setup() 144 | 145 | # create a new document 146 | bp = BlogPost( 147 | _id=0, 148 | title="Hello World!", 149 | tags=["testing", "dummy"], 150 | content=open(__file__).read(), 151 | ) 152 | await bp.save(refresh=True) 153 | 154 | # create new index 155 | await migrate() 156 | 157 | # close the connection 158 | await async_connections.get_connection().close() 159 | 160 | 161 | if __name__ == "__main__": 162 | asyncio.run(main()) 163 | -------------------------------------------------------------------------------- /examples/async/completion.py: -------------------------------------------------------------------------------- 1 | # Licensed to Elasticsearch B.V. under one or more contributor 2 | # license agreements. See the NOTICE file distributed with 3 | # this work for additional information regarding copyright 4 | # ownership. Elasticsearch B.V. licenses this file to you under 5 | # the Apache License, Version 2.0 (the "License"); you may 6 | # not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | """ 19 | Example ``Document`` with completion suggester. 20 | 21 | In the ``Person`` class we index the person's name to allow auto completing in 22 | any order ("first last", "middle last first", ...). For the weight we use a 23 | value from the ``popularity`` field which is a long. 24 | 25 | To make the suggestions work in different languages we added a custom analyzer 26 | that does ascii folding. 27 | """ 28 | 29 | import asyncio 30 | import os 31 | from itertools import permutations 32 | from typing import TYPE_CHECKING, Any, Dict, Optional 33 | 34 | from elasticsearch_dsl import ( 35 | AsyncDocument, 36 | Completion, 37 | Keyword, 38 | Long, 39 | Text, 40 | analyzer, 41 | async_connections, 42 | mapped_field, 43 | token_filter, 44 | ) 45 | 46 | # custom analyzer for names 47 | ascii_fold = analyzer( 48 | "ascii_fold", 49 | # we don't want to split O'Brian or Toulouse-Lautrec 50 | tokenizer="whitespace", 51 | filter=["lowercase", token_filter("ascii_fold", "asciifolding")], 52 | ) 53 | 54 | 55 | class Person(AsyncDocument): 56 | if TYPE_CHECKING: 57 | # definitions here help type checkers understand additional arguments 58 | # that are allowed in the constructor 59 | _id: Optional[int] = mapped_field(default=None) 60 | 61 | name: str = mapped_field(Text(fields={"keyword": Keyword()}), default="") 62 | popularity: int = mapped_field(Long(), default=0) 63 | 64 | # completion field with a custom analyzer 65 | suggest: Dict[str, Any] = mapped_field(Completion(analyzer=ascii_fold), init=False) 66 | 67 | def clean(self) -> None: 68 | """ 69 | Automatically construct the suggestion input and weight by taking all 70 | possible permutations of Person's name as ``input`` and taking their 71 | popularity as ``weight``. 72 | """ 73 | self.suggest = { 74 | "input": [" ".join(p) for p in permutations(self.name.split())], 75 | "weight": self.popularity, 76 | } 77 | 78 | class Index: 79 | name = "test-suggest" 80 | settings = {"number_of_shards": 1, "number_of_replicas": 0} 81 | 82 | 83 | async def main() -> None: 84 | # initiate the default connection to elasticsearch 85 | async_connections.create_connection(hosts=[os.environ["ELASTICSEARCH_URL"]]) 86 | 87 | # create the empty index 88 | await Person.init() 89 | 90 | # index some sample data 91 | for id, (name, popularity) in enumerate( 92 | [("Henri de Toulouse-Lautrec", 42), ("Jára Cimrman", 124)] 93 | ): 94 | await Person(_id=id, name=name, popularity=popularity).save() 95 | 96 | # refresh index manually to make changes live 97 | await Person._index.refresh() 98 | 99 | # run some suggestions 100 | for text in ("já", "Jara Cimr", "tou", "de hen"): 101 | s = Person.search() 102 | s = s.suggest("auto_complete", text, completion={"field": "suggest"}) 103 | response = await s.execute() 104 | 105 | # print out all the options we got 106 | for option in response.suggest["auto_complete"][0].options: 107 | print("%10s: %25s (%d)" % (text, option._source.name, option._score)) 108 | 109 | # close the connection 110 | await async_connections.get_connection().close() 111 | 112 | 113 | if __name__ == "__main__": 114 | asyncio.run(main()) 115 | -------------------------------------------------------------------------------- /examples/async/composite_agg.py: -------------------------------------------------------------------------------- 1 | # Licensed to Elasticsearch B.V. under one or more contributor 2 | # license agreements. See the NOTICE file distributed with 3 | # this work for additional information regarding copyright 4 | # ownership. Elasticsearch B.V. licenses this file to you under 5 | # the Apache License, Version 2.0 (the "License"); you may 6 | # not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | import asyncio 19 | import os 20 | from typing import Any, AsyncIterator, Dict, Mapping, Sequence, cast 21 | 22 | from elasticsearch.helpers import async_bulk 23 | 24 | from elasticsearch_dsl import Agg, AsyncSearch, Response, aggs, async_connections 25 | from elasticsearch_dsl.types import CompositeAggregate 26 | from tests.test_integration.test_data import DATA, GIT_INDEX 27 | 28 | 29 | async def scan_aggs( 30 | search: AsyncSearch, 31 | source_aggs: Sequence[Mapping[str, Agg]], 32 | inner_aggs: Dict[str, Agg] = {}, 33 | size: int = 10, 34 | ) -> AsyncIterator[CompositeAggregate]: 35 | """ 36 | Helper function used to iterate over all possible bucket combinations of 37 | ``source_aggs``, returning results of ``inner_aggs`` for each. Uses the 38 | ``composite`` aggregation under the hood to perform this. 39 | """ 40 | 41 | async def run_search(**kwargs: Any) -> Response: 42 | s = search[:0] 43 | bucket = s.aggs.bucket( 44 | "comp", 45 | aggs.Composite( 46 | sources=source_aggs, 47 | size=size, 48 | **kwargs, 49 | ), 50 | ) 51 | for agg_name, agg in inner_aggs.items(): 52 | bucket[agg_name] = agg 53 | return await s.execute() 54 | 55 | response = await run_search() 56 | while response.aggregations["comp"].buckets: 57 | for b in response.aggregations["comp"].buckets: 58 | yield cast(CompositeAggregate, b) 59 | if "after_key" in response.aggregations["comp"]: 60 | after = response.aggregations["comp"].after_key 61 | else: 62 | after = response.aggregations["comp"].buckets[-1].key 63 | response = await run_search(after=after) 64 | 65 | 66 | async def main() -> None: 67 | # initiate the default connection to elasticsearch 68 | client = async_connections.create_connection( 69 | hosts=[os.environ["ELASTICSEARCH_URL"]] 70 | ) 71 | 72 | # create the index and populate it with some data 73 | # note that the dataset is imported from the library's test suite 74 | await client.indices.delete(index="git", ignore_unavailable=True) 75 | await client.indices.create(index="git", **GIT_INDEX) 76 | await async_bulk(client, DATA, raise_on_error=True, refresh=True) 77 | 78 | # run some aggregations on the data 79 | async for b in scan_aggs( 80 | AsyncSearch(index="git"), 81 | [{"files": aggs.Terms(field="files")}], 82 | {"first_seen": aggs.Min(field="committed_date")}, 83 | ): 84 | print( 85 | "File %s has been modified %d times, first seen at %s." 86 | % (b.key.files, b.doc_count, b.first_seen.value_as_string) 87 | ) 88 | 89 | # close the connection 90 | await async_connections.get_connection().close() 91 | 92 | 93 | if __name__ == "__main__": 94 | asyncio.run(main()) 95 | -------------------------------------------------------------------------------- /examples/async/percolate.py: -------------------------------------------------------------------------------- 1 | # Licensed to Elasticsearch B.V. under one or more contributor 2 | # license agreements. See the NOTICE file distributed with 3 | # this work for additional information regarding copyright 4 | # ownership. Elasticsearch B.V. licenses this file to you under 5 | # the Apache License, Version 2.0 (the "License"); you may 6 | # not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | import asyncio 19 | import os 20 | from typing import TYPE_CHECKING, Any, List, Optional 21 | 22 | from elasticsearch_dsl import ( 23 | AsyncDocument, 24 | AsyncSearch, 25 | Keyword, 26 | Percolator, 27 | Q, 28 | Query, 29 | async_connections, 30 | mapped_field, 31 | ) 32 | 33 | 34 | class BlogPost(AsyncDocument): 35 | """ 36 | Blog posts that will be automatically tagged based on percolation queries. 37 | """ 38 | 39 | if TYPE_CHECKING: 40 | # definitions here help type checkers understand additional arguments 41 | # that are allowed in the constructor 42 | _id: int 43 | 44 | content: Optional[str] 45 | tags: List[str] = mapped_field(Keyword(), default_factory=list) 46 | 47 | class Index: 48 | name = "test-blogpost" 49 | 50 | async def add_tags(self) -> None: 51 | # run a percolation to automatically tag the blog post. 52 | s = AsyncSearch(index="test-percolator") 53 | s = s.query( 54 | "percolate", field="query", index=self._get_index(), document=self.to_dict() 55 | ) 56 | 57 | # collect all the tags from matched percolators 58 | async for percolator in s: 59 | self.tags.extend(percolator.tags) 60 | 61 | # make sure tags are unique 62 | self.tags = list(set(self.tags)) 63 | 64 | async def save(self, **kwargs: Any) -> None: # type: ignore[override] 65 | await self.add_tags() 66 | await super().save(**kwargs) 67 | 68 | 69 | class PercolatorDoc(AsyncDocument): 70 | """ 71 | Document class used for storing the percolation queries. 72 | """ 73 | 74 | if TYPE_CHECKING: 75 | _id: str 76 | 77 | # relevant fields from BlogPost must be also present here for the queries 78 | # to be able to use them. Another option would be to use document 79 | # inheritance but save() would have to be reset to normal behavior. 80 | content: Optional[str] 81 | 82 | # the percolator query to be run against the doc 83 | query: Query = mapped_field(Percolator()) 84 | # list of tags to append to a document 85 | tags: List[str] = mapped_field(Keyword(multi=True)) 86 | 87 | class Index: 88 | name = "test-percolator" 89 | settings = {"number_of_shards": 1, "number_of_replicas": 0} 90 | 91 | 92 | async def setup() -> None: 93 | # create the percolator index if it doesn't exist 94 | if not await PercolatorDoc._index.exists(): 95 | await PercolatorDoc.init() 96 | 97 | # register a percolation query looking for documents about python 98 | await PercolatorDoc( 99 | _id="python", 100 | tags=["programming", "development", "python"], 101 | content="", 102 | query=Q("match", content="python"), 103 | ).save(refresh=True) 104 | 105 | 106 | async def main() -> None: 107 | # initiate the default connection to elasticsearch 108 | async_connections.create_connection(hosts=[os.environ["ELASTICSEARCH_URL"]]) 109 | 110 | await setup() 111 | 112 | # close the connection 113 | await async_connections.get_connection().close() 114 | 115 | 116 | if __name__ == "__main__": 117 | asyncio.run(main()) 118 | -------------------------------------------------------------------------------- /examples/async/search_as_you_type.py: -------------------------------------------------------------------------------- 1 | # Licensed to Elasticsearch B.V. under one or more contributor 2 | # license agreements. See the NOTICE file distributed with 3 | # this work for additional information regarding copyright 4 | # ownership. Elasticsearch B.V. licenses this file to you under 5 | # the Apache License, Version 2.0 (the "License"); you may 6 | # not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | """ 19 | Example ``Document`` with search_as_you_type field datatype and how to search it. 20 | 21 | When creating a field with search_as_you_type datatype ElasticSearch creates additional 22 | subfields to enable efficient as-you-type completion, matching terms at any position 23 | within the input. 24 | 25 | To custom analyzer with ascii folding allow search to work in different languages. 26 | """ 27 | 28 | import asyncio 29 | import os 30 | from typing import TYPE_CHECKING, Optional 31 | 32 | from elasticsearch_dsl import ( 33 | AsyncDocument, 34 | SearchAsYouType, 35 | async_connections, 36 | mapped_field, 37 | ) 38 | from elasticsearch_dsl.query import MultiMatch 39 | 40 | 41 | class Person(AsyncDocument): 42 | if TYPE_CHECKING: 43 | # definitions here help type checkers understand additional arguments 44 | # that are allowed in the constructor 45 | _id: Optional[int] = mapped_field(default=None) 46 | 47 | name: str = mapped_field(SearchAsYouType(max_shingle_size=3), default="") 48 | 49 | class Index: 50 | name = "test-search-as-you-type" 51 | settings = {"number_of_shards": 1, "number_of_replicas": 0} 52 | 53 | 54 | async def main() -> None: 55 | # initiate the default connection to elasticsearch 56 | async_connections.create_connection(hosts=[os.environ["ELASTICSEARCH_URL"]]) 57 | 58 | # create the empty index 59 | await Person.init() 60 | 61 | import pprint 62 | 63 | pprint.pprint(Person().to_dict(), indent=2) 64 | 65 | # index some sample data 66 | names = [ 67 | "Andy Warhol", 68 | "Alphonse Mucha", 69 | "Henri de Toulouse-Lautrec", 70 | "Jára Cimrman", 71 | ] 72 | for id, name in enumerate(names): 73 | await Person(_id=id, name=name).save() 74 | 75 | # refresh index manually to make changes live 76 | await Person._index.refresh() 77 | 78 | # run some suggestions 79 | for text in ("já", "Cimr", "toulouse", "Henri Tou", "a"): 80 | s = Person.search() 81 | 82 | s.query = MultiMatch( # type: ignore[assignment] 83 | query=text, 84 | type="bool_prefix", 85 | fields=["name", "name._2gram", "name._3gram"], 86 | ) 87 | 88 | response = await s.execute() 89 | 90 | # print out all the options we got 91 | for h in response: 92 | print("%15s: %25s" % (text, h.name)) 93 | 94 | # close the connection 95 | await async_connections.get_connection().close() 96 | 97 | 98 | if __name__ == "__main__": 99 | asyncio.run(main()) 100 | -------------------------------------------------------------------------------- /examples/async/semantic_text.py: -------------------------------------------------------------------------------- 1 | # Licensed to Elasticsearch B.V. under one or more contributor 2 | # license agreements. See the NOTICE file distributed with 3 | # this work for additional information regarding copyright 4 | # ownership. Elasticsearch B.V. licenses this file to you under 5 | # the Apache License, Version 2.0 (the "License"); you may 6 | # not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | 19 | """ 20 | # Semantic Text example 21 | 22 | Requirements: 23 | 24 | $ pip install "elasticsearch-dsl[async]" tqdm 25 | 26 | Before running this example, an ELSER inference endpoint must be created in the 27 | Elasticsearch cluster. This can be done manually from Kibana, or with the 28 | following curl command from a terminal: 29 | 30 | curl -X PUT \ 31 | "$ELASTICSEARCH_URL/_inference/sparse_embedding/my-elser-endpoint" \ 32 | -H "Content-Type: application/json" \ 33 | -d '{"service":"elser","service_settings":{"num_allocations":1,"num_threads":1}}' 34 | 35 | To run the example: 36 | 37 | $ python semantic_text.py "text to search" 38 | 39 | The index will be created automatically if it does not exist. Add 40 | `--recreate-index` to the command to regenerate it. 41 | 42 | The example dataset includes a selection of workplace documents. The 43 | following are good example queries to try out with this dataset: 44 | 45 | $ python semantic_text.py "work from home" 46 | $ python semantic_text.py "vacation time" 47 | $ python semantic_text.py "can I bring a bird to work?" 48 | 49 | When the index is created, the inference service will split the documents into 50 | short passages, and for each passage a sparse embedding will be generated using 51 | Elastic's ELSER v2 model. 52 | """ 53 | 54 | import argparse 55 | import asyncio 56 | import json 57 | import os 58 | from datetime import datetime 59 | from typing import Any, Optional 60 | from urllib.request import urlopen 61 | 62 | from tqdm import tqdm 63 | 64 | import elasticsearch_dsl as dsl 65 | 66 | DATASET_URL = "https://raw.githubusercontent.com/elastic/elasticsearch-labs/main/datasets/workplace-documents.json" 67 | 68 | 69 | class WorkplaceDoc(dsl.AsyncDocument): 70 | class Index: 71 | name = "workplace_documents_semantic" 72 | 73 | name: str 74 | summary: str 75 | content: Any = dsl.mapped_field( 76 | dsl.field.SemanticText(inference_id="my-elser-endpoint") 77 | ) 78 | created: datetime 79 | updated: Optional[datetime] 80 | url: str = dsl.mapped_field(dsl.Keyword()) 81 | category: str = dsl.mapped_field(dsl.Keyword()) 82 | 83 | 84 | async def create() -> None: 85 | 86 | # create the index 87 | await WorkplaceDoc._index.delete(ignore_unavailable=True) 88 | await WorkplaceDoc.init() 89 | 90 | # download the data 91 | dataset = json.loads(urlopen(DATASET_URL).read()) 92 | 93 | # import the dataset 94 | for data in tqdm(dataset, desc="Indexing documents..."): 95 | doc = WorkplaceDoc( 96 | name=data["name"], 97 | summary=data["summary"], 98 | content=data["content"], 99 | created=data.get("created_on"), 100 | updated=data.get("updated_at"), 101 | url=data["url"], 102 | category=data["category"], 103 | ) 104 | await doc.save() 105 | 106 | # refresh the index 107 | await WorkplaceDoc._index.refresh() 108 | 109 | 110 | async def search(query: str) -> dsl.AsyncSearch[WorkplaceDoc]: 111 | search = WorkplaceDoc.search() 112 | search = search[:5] 113 | return search.query(dsl.query.Semantic(field=WorkplaceDoc.content, query=query)) 114 | 115 | 116 | def parse_args() -> argparse.Namespace: 117 | parser = argparse.ArgumentParser(description="Vector database with Elasticsearch") 118 | parser.add_argument( 119 | "--recreate-index", action="store_true", help="Recreate and populate the index" 120 | ) 121 | parser.add_argument("query", action="store", help="The search query") 122 | return parser.parse_args() 123 | 124 | 125 | async def main() -> None: 126 | args = parse_args() 127 | 128 | # initiate the default connection to elasticsearch 129 | dsl.async_connections.create_connection(hosts=[os.environ["ELASTICSEARCH_URL"]]) 130 | 131 | if args.recreate_index or not await WorkplaceDoc._index.exists(): 132 | await create() 133 | 134 | results = await search(args.query) 135 | 136 | async for hit in results: 137 | print( 138 | f"Document: {hit.name} [Category: {hit.category}] [Score: {hit.meta.score}]" 139 | ) 140 | print(f"Content: {hit.content.text}") 141 | print("--------------------\n") 142 | 143 | # close the connection 144 | await dsl.async_connections.get_connection().close() 145 | 146 | 147 | if __name__ == "__main__": 148 | asyncio.run(main()) 149 | -------------------------------------------------------------------------------- /examples/completion.py: -------------------------------------------------------------------------------- 1 | # Licensed to Elasticsearch B.V. under one or more contributor 2 | # license agreements. See the NOTICE file distributed with 3 | # this work for additional information regarding copyright 4 | # ownership. Elasticsearch B.V. licenses this file to you under 5 | # the Apache License, Version 2.0 (the "License"); you may 6 | # not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | """ 19 | Example ``Document`` with completion suggester. 20 | 21 | In the ``Person`` class we index the person's name to allow auto completing in 22 | any order ("first last", "middle last first", ...). For the weight we use a 23 | value from the ``popularity`` field which is a long. 24 | 25 | To make the suggestions work in different languages we added a custom analyzer 26 | that does ascii folding. 27 | """ 28 | 29 | import os 30 | from itertools import permutations 31 | from typing import TYPE_CHECKING, Any, Dict, Optional 32 | 33 | from elasticsearch_dsl import ( 34 | Completion, 35 | Document, 36 | Keyword, 37 | Long, 38 | Text, 39 | analyzer, 40 | connections, 41 | mapped_field, 42 | token_filter, 43 | ) 44 | 45 | # custom analyzer for names 46 | ascii_fold = analyzer( 47 | "ascii_fold", 48 | # we don't want to split O'Brian or Toulouse-Lautrec 49 | tokenizer="whitespace", 50 | filter=["lowercase", token_filter("ascii_fold", "asciifolding")], 51 | ) 52 | 53 | 54 | class Person(Document): 55 | if TYPE_CHECKING: 56 | # definitions here help type checkers understand additional arguments 57 | # that are allowed in the constructor 58 | _id: Optional[int] = mapped_field(default=None) 59 | 60 | name: str = mapped_field(Text(fields={"keyword": Keyword()}), default="") 61 | popularity: int = mapped_field(Long(), default=0) 62 | 63 | # completion field with a custom analyzer 64 | suggest: Dict[str, Any] = mapped_field(Completion(analyzer=ascii_fold), init=False) 65 | 66 | def clean(self) -> None: 67 | """ 68 | Automatically construct the suggestion input and weight by taking all 69 | possible permutations of Person's name as ``input`` and taking their 70 | popularity as ``weight``. 71 | """ 72 | self.suggest = { 73 | "input": [" ".join(p) for p in permutations(self.name.split())], 74 | "weight": self.popularity, 75 | } 76 | 77 | class Index: 78 | name = "test-suggest" 79 | settings = {"number_of_shards": 1, "number_of_replicas": 0} 80 | 81 | 82 | def main() -> None: 83 | # initiate the default connection to elasticsearch 84 | connections.create_connection(hosts=[os.environ["ELASTICSEARCH_URL"]]) 85 | 86 | # create the empty index 87 | Person.init() 88 | 89 | # index some sample data 90 | for id, (name, popularity) in enumerate( 91 | [("Henri de Toulouse-Lautrec", 42), ("Jára Cimrman", 124)] 92 | ): 93 | Person(_id=id, name=name, popularity=popularity).save() 94 | 95 | # refresh index manually to make changes live 96 | Person._index.refresh() 97 | 98 | # run some suggestions 99 | for text in ("já", "Jara Cimr", "tou", "de hen"): 100 | s = Person.search() 101 | s = s.suggest("auto_complete", text, completion={"field": "suggest"}) 102 | response = s.execute() 103 | 104 | # print out all the options we got 105 | for option in response.suggest["auto_complete"][0].options: 106 | print("%10s: %25s (%d)" % (text, option._source.name, option._score)) 107 | 108 | # close the connection 109 | connections.get_connection().close() 110 | 111 | 112 | if __name__ == "__main__": 113 | main() 114 | -------------------------------------------------------------------------------- /examples/composite_agg.py: -------------------------------------------------------------------------------- 1 | # Licensed to Elasticsearch B.V. under one or more contributor 2 | # license agreements. See the NOTICE file distributed with 3 | # this work for additional information regarding copyright 4 | # ownership. Elasticsearch B.V. licenses this file to you under 5 | # the Apache License, Version 2.0 (the "License"); you may 6 | # not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | import os 19 | from typing import Any, Dict, Iterator, Mapping, Sequence, cast 20 | 21 | from elasticsearch.helpers import bulk 22 | 23 | from elasticsearch_dsl import Agg, Response, Search, aggs, connections 24 | from elasticsearch_dsl.types import CompositeAggregate 25 | from tests.test_integration.test_data import DATA, GIT_INDEX 26 | 27 | 28 | def scan_aggs( 29 | search: Search, 30 | source_aggs: Sequence[Mapping[str, Agg]], 31 | inner_aggs: Dict[str, Agg] = {}, 32 | size: int = 10, 33 | ) -> Iterator[CompositeAggregate]: 34 | """ 35 | Helper function used to iterate over all possible bucket combinations of 36 | ``source_aggs``, returning results of ``inner_aggs`` for each. Uses the 37 | ``composite`` aggregation under the hood to perform this. 38 | """ 39 | 40 | def run_search(**kwargs: Any) -> Response: 41 | s = search[:0] 42 | bucket = s.aggs.bucket( 43 | "comp", 44 | aggs.Composite( 45 | sources=source_aggs, 46 | size=size, 47 | **kwargs, 48 | ), 49 | ) 50 | for agg_name, agg in inner_aggs.items(): 51 | bucket[agg_name] = agg 52 | return s.execute() 53 | 54 | response = run_search() 55 | while response.aggregations["comp"].buckets: 56 | for b in response.aggregations["comp"].buckets: 57 | yield cast(CompositeAggregate, b) 58 | if "after_key" in response.aggregations["comp"]: 59 | after = response.aggregations["comp"].after_key 60 | else: 61 | after = response.aggregations["comp"].buckets[-1].key 62 | response = run_search(after=after) 63 | 64 | 65 | def main() -> None: 66 | # initiate the default connection to elasticsearch 67 | client = connections.create_connection(hosts=[os.environ["ELASTICSEARCH_URL"]]) 68 | 69 | # create the index and populate it with some data 70 | # note that the dataset is imported from the library's test suite 71 | client.indices.delete(index="git", ignore_unavailable=True) 72 | client.indices.create(index="git", **GIT_INDEX) 73 | bulk(client, DATA, raise_on_error=True, refresh=True) 74 | 75 | # run some aggregations on the data 76 | for b in scan_aggs( 77 | Search(index="git"), 78 | [{"files": aggs.Terms(field="files")}], 79 | {"first_seen": aggs.Min(field="committed_date")}, 80 | ): 81 | print( 82 | "File %s has been modified %d times, first seen at %s." 83 | % (b.key.files, b.doc_count, b.first_seen.value_as_string) 84 | ) 85 | 86 | # close the connection 87 | connections.get_connection().close() 88 | 89 | 90 | if __name__ == "__main__": 91 | main() 92 | -------------------------------------------------------------------------------- /examples/percolate.py: -------------------------------------------------------------------------------- 1 | # Licensed to Elasticsearch B.V. under one or more contributor 2 | # license agreements. See the NOTICE file distributed with 3 | # this work for additional information regarding copyright 4 | # ownership. Elasticsearch B.V. licenses this file to you under 5 | # the Apache License, Version 2.0 (the "License"); you may 6 | # not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | import os 19 | from typing import TYPE_CHECKING, Any, List, Optional 20 | 21 | from elasticsearch_dsl import ( 22 | Document, 23 | Keyword, 24 | Percolator, 25 | Q, 26 | Query, 27 | Search, 28 | connections, 29 | mapped_field, 30 | ) 31 | 32 | 33 | class BlogPost(Document): 34 | """ 35 | Blog posts that will be automatically tagged based on percolation queries. 36 | """ 37 | 38 | if TYPE_CHECKING: 39 | # definitions here help type checkers understand additional arguments 40 | # that are allowed in the constructor 41 | _id: int 42 | 43 | content: Optional[str] 44 | tags: List[str] = mapped_field(Keyword(), default_factory=list) 45 | 46 | class Index: 47 | name = "test-blogpost" 48 | 49 | def add_tags(self) -> None: 50 | # run a percolation to automatically tag the blog post. 51 | s = Search(index="test-percolator") 52 | s = s.query( 53 | "percolate", field="query", index=self._get_index(), document=self.to_dict() 54 | ) 55 | 56 | # collect all the tags from matched percolators 57 | for percolator in s: 58 | self.tags.extend(percolator.tags) 59 | 60 | # make sure tags are unique 61 | self.tags = list(set(self.tags)) 62 | 63 | def save(self, **kwargs: Any) -> None: # type: ignore[override] 64 | self.add_tags() 65 | super().save(**kwargs) 66 | 67 | 68 | class PercolatorDoc(Document): 69 | """ 70 | Document class used for storing the percolation queries. 71 | """ 72 | 73 | if TYPE_CHECKING: 74 | _id: str 75 | 76 | # relevant fields from BlogPost must be also present here for the queries 77 | # to be able to use them. Another option would be to use document 78 | # inheritance but save() would have to be reset to normal behavior. 79 | content: Optional[str] 80 | 81 | # the percolator query to be run against the doc 82 | query: Query = mapped_field(Percolator()) 83 | # list of tags to append to a document 84 | tags: List[str] = mapped_field(Keyword(multi=True)) 85 | 86 | class Index: 87 | name = "test-percolator" 88 | settings = {"number_of_shards": 1, "number_of_replicas": 0} 89 | 90 | 91 | def setup() -> None: 92 | # create the percolator index if it doesn't exist 93 | if not PercolatorDoc._index.exists(): 94 | PercolatorDoc.init() 95 | 96 | # register a percolation query looking for documents about python 97 | PercolatorDoc( 98 | _id="python", 99 | tags=["programming", "development", "python"], 100 | content="", 101 | query=Q("match", content="python"), 102 | ).save(refresh=True) 103 | 104 | 105 | def main() -> None: 106 | # initiate the default connection to elasticsearch 107 | connections.create_connection(hosts=[os.environ["ELASTICSEARCH_URL"]]) 108 | 109 | setup() 110 | 111 | # close the connection 112 | connections.get_connection().close() 113 | 114 | 115 | if __name__ == "__main__": 116 | main() 117 | -------------------------------------------------------------------------------- /examples/search_as_you_type.py: -------------------------------------------------------------------------------- 1 | # Licensed to Elasticsearch B.V. under one or more contributor 2 | # license agreements. See the NOTICE file distributed with 3 | # this work for additional information regarding copyright 4 | # ownership. Elasticsearch B.V. licenses this file to you under 5 | # the Apache License, Version 2.0 (the "License"); you may 6 | # not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | """ 19 | Example ``Document`` with search_as_you_type field datatype and how to search it. 20 | 21 | When creating a field with search_as_you_type datatype ElasticSearch creates additional 22 | subfields to enable efficient as-you-type completion, matching terms at any position 23 | within the input. 24 | 25 | To custom analyzer with ascii folding allow search to work in different languages. 26 | """ 27 | 28 | import os 29 | from typing import TYPE_CHECKING, Optional 30 | 31 | from elasticsearch_dsl import Document, SearchAsYouType, connections, mapped_field 32 | from elasticsearch_dsl.query import MultiMatch 33 | 34 | 35 | class Person(Document): 36 | if TYPE_CHECKING: 37 | # definitions here help type checkers understand additional arguments 38 | # that are allowed in the constructor 39 | _id: Optional[int] = mapped_field(default=None) 40 | 41 | name: str = mapped_field(SearchAsYouType(max_shingle_size=3), default="") 42 | 43 | class Index: 44 | name = "test-search-as-you-type" 45 | settings = {"number_of_shards": 1, "number_of_replicas": 0} 46 | 47 | 48 | def main() -> None: 49 | # initiate the default connection to elasticsearch 50 | connections.create_connection(hosts=[os.environ["ELASTICSEARCH_URL"]]) 51 | 52 | # create the empty index 53 | Person.init() 54 | 55 | import pprint 56 | 57 | pprint.pprint(Person().to_dict(), indent=2) 58 | 59 | # index some sample data 60 | names = [ 61 | "Andy Warhol", 62 | "Alphonse Mucha", 63 | "Henri de Toulouse-Lautrec", 64 | "Jára Cimrman", 65 | ] 66 | for id, name in enumerate(names): 67 | Person(_id=id, name=name).save() 68 | 69 | # refresh index manually to make changes live 70 | Person._index.refresh() 71 | 72 | # run some suggestions 73 | for text in ("já", "Cimr", "toulouse", "Henri Tou", "a"): 74 | s = Person.search() 75 | 76 | s.query = MultiMatch( # type: ignore[assignment] 77 | query=text, 78 | type="bool_prefix", 79 | fields=["name", "name._2gram", "name._3gram"], 80 | ) 81 | 82 | response = s.execute() 83 | 84 | # print out all the options we got 85 | for h in response: 86 | print("%15s: %25s" % (text, h.name)) 87 | 88 | # close the connection 89 | connections.get_connection().close() 90 | 91 | 92 | if __name__ == "__main__": 93 | main() 94 | -------------------------------------------------------------------------------- /examples/semantic_text.py: -------------------------------------------------------------------------------- 1 | # Licensed to Elasticsearch B.V. under one or more contributor 2 | # license agreements. See the NOTICE file distributed with 3 | # this work for additional information regarding copyright 4 | # ownership. Elasticsearch B.V. licenses this file to you under 5 | # the Apache License, Version 2.0 (the "License"); you may 6 | # not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | 19 | """ 20 | # Semantic Text example 21 | 22 | Requirements: 23 | 24 | $ pip install "elasticsearch-dsl" tqdm 25 | 26 | Before running this example, an ELSER inference endpoint must be created in the 27 | Elasticsearch cluster. This can be done manually from Kibana, or with the 28 | following curl command from a terminal: 29 | 30 | curl -X PUT \ 31 | "$ELASTICSEARCH_URL/_inference/sparse_embedding/my-elser-endpoint" \ 32 | -H "Content-Type: application/json" \ 33 | -d '{"service":"elser","service_settings":{"num_allocations":1,"num_threads":1}}' 34 | 35 | To run the example: 36 | 37 | $ python semantic_text.py "text to search" 38 | 39 | The index will be created automatically if it does not exist. Add 40 | `--recreate-index` to the command to regenerate it. 41 | 42 | The example dataset includes a selection of workplace documents. The 43 | following are good example queries to try out with this dataset: 44 | 45 | $ python semantic_text.py "work from home" 46 | $ python semantic_text.py "vacation time" 47 | $ python semantic_text.py "can I bring a bird to work?" 48 | 49 | When the index is created, the inference service will split the documents into 50 | short passages, and for each passage a sparse embedding will be generated using 51 | Elastic's ELSER v2 model. 52 | """ 53 | 54 | import argparse 55 | import json 56 | import os 57 | from datetime import datetime 58 | from typing import Any, Optional 59 | from urllib.request import urlopen 60 | 61 | from tqdm import tqdm 62 | 63 | import elasticsearch_dsl as dsl 64 | 65 | DATASET_URL = "https://raw.githubusercontent.com/elastic/elasticsearch-labs/main/datasets/workplace-documents.json" 66 | 67 | 68 | class WorkplaceDoc(dsl.Document): 69 | class Index: 70 | name = "workplace_documents_semantic" 71 | 72 | name: str 73 | summary: str 74 | content: Any = dsl.mapped_field( 75 | dsl.field.SemanticText(inference_id="my-elser-endpoint") 76 | ) 77 | created: datetime 78 | updated: Optional[datetime] 79 | url: str = dsl.mapped_field(dsl.Keyword()) 80 | category: str = dsl.mapped_field(dsl.Keyword()) 81 | 82 | 83 | def create() -> None: 84 | 85 | # create the index 86 | WorkplaceDoc._index.delete(ignore_unavailable=True) 87 | WorkplaceDoc.init() 88 | 89 | # download the data 90 | dataset = json.loads(urlopen(DATASET_URL).read()) 91 | 92 | # import the dataset 93 | for data in tqdm(dataset, desc="Indexing documents..."): 94 | doc = WorkplaceDoc( 95 | name=data["name"], 96 | summary=data["summary"], 97 | content=data["content"], 98 | created=data.get("created_on"), 99 | updated=data.get("updated_at"), 100 | url=data["url"], 101 | category=data["category"], 102 | ) 103 | doc.save() 104 | 105 | # refresh the index 106 | WorkplaceDoc._index.refresh() 107 | 108 | 109 | def search(query: str) -> dsl.Search[WorkplaceDoc]: 110 | search = WorkplaceDoc.search() 111 | search = search[:5] 112 | return search.query(dsl.query.Semantic(field=WorkplaceDoc.content, query=query)) 113 | 114 | 115 | def parse_args() -> argparse.Namespace: 116 | parser = argparse.ArgumentParser(description="Vector database with Elasticsearch") 117 | parser.add_argument( 118 | "--recreate-index", action="store_true", help="Recreate and populate the index" 119 | ) 120 | parser.add_argument("query", action="store", help="The search query") 121 | return parser.parse_args() 122 | 123 | 124 | def main() -> None: 125 | args = parse_args() 126 | 127 | # initiate the default connection to elasticsearch 128 | dsl.connections.create_connection(hosts=[os.environ["ELASTICSEARCH_URL"]]) 129 | 130 | if args.recreate_index or not WorkplaceDoc._index.exists(): 131 | create() 132 | 133 | results = search(args.query) 134 | 135 | for hit in results: 136 | print( 137 | f"Document: {hit.name} [Category: {hit.category}] [Score: {hit.meta.score}]" 138 | ) 139 | print(f"Content: {hit.content.text}") 140 | print("--------------------\n") 141 | 142 | # close the connection 143 | dsl.connections.get_connection().close() 144 | 145 | 146 | if __name__ == "__main__": 147 | main() 148 | -------------------------------------------------------------------------------- /noxfile.py: -------------------------------------------------------------------------------- 1 | # Licensed to Elasticsearch B.V. under one or more contributor 2 | # license agreements. See the NOTICE file distributed with 3 | # this work for additional information regarding copyright 4 | # ownership. Elasticsearch B.V. licenses this file to you under 5 | # the Apache License, Version 2.0 (the "License"); you may 6 | # not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | import nox 19 | 20 | SOURCE_FILES = ( 21 | "setup.py", 22 | "noxfile.py", 23 | "docs/", 24 | "elasticsearch_dsl/", 25 | "examples/", 26 | "tests/", 27 | "utils/", 28 | ) 29 | 30 | 31 | @nox.session( 32 | python=[ 33 | "3.8", 34 | "3.9", 35 | "3.10", 36 | "3.11", 37 | "3.12", 38 | "3.13", 39 | ] 40 | ) 41 | def test(session): 42 | session.install(".[develop]") 43 | session.install("elasticsearch<9") # tests run against 8.x servers 44 | 45 | if session.posargs: 46 | argv = session.posargs 47 | else: 48 | argv = ( 49 | "-vvv", 50 | "--cov=elasticsearch_dsl", 51 | "--cov=tests.test_integration.test_examples", 52 | "--cov-report=term-missing", 53 | "--cov-branch", 54 | "--cov-report=html", 55 | "tests/", 56 | ) 57 | session.run("pytest", *argv) 58 | 59 | 60 | @nox.session(python="3.13") 61 | def format(session): 62 | session.install("black~=24.0", "isort", "setuptools", ".[develop]") 63 | session.run("black", "--target-version=py38", *SOURCE_FILES) 64 | session.run("isort", *SOURCE_FILES) 65 | session.run("python", "utils/license-headers.py", "fix", *SOURCE_FILES) 66 | 67 | lint(session) 68 | 69 | 70 | @nox.session(python="3.13") 71 | def lint(session): 72 | session.install("flake8", "black~=24.0", "isort", "setuptools") 73 | session.run("black", "--check", "--target-version=py38", *SOURCE_FILES) 74 | session.run("isort", "--check", *SOURCE_FILES) 75 | session.run("flake8", "--ignore=E501,E741,W503,E704", *SOURCE_FILES) 76 | session.run("python", "utils/license-headers.py", "check", *SOURCE_FILES) 77 | 78 | 79 | @nox.session(python="3.8") 80 | def type_check(session): 81 | # type checking is done by the unified client now 82 | pass 83 | 84 | 85 | @nox.session() 86 | def docs(session): 87 | session.install(".[develop]") 88 | 89 | session.run("sphinx-build", "docs/", "docs/_build", "-b", "html", "-W") 90 | -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [build_sphinx] 2 | source-dir = docs/ 3 | build-dir = docs/_build 4 | all_files = 1 5 | 6 | [isort] 7 | profile = black 8 | 9 | [tool:pytest] 10 | filterwarnings = 11 | error 12 | ignore:Legacy index templates are deprecated in favor of composable templates.:elasticsearch.exceptions.ElasticsearchWarning 13 | ignore:datetime.datetime.utcfromtimestamp\(\) is deprecated and scheduled for removal in a future version..*:DeprecationWarning 14 | default:enable_cleanup_closed ignored.*:DeprecationWarning 15 | markers = 16 | sync: mark a test as performing I/O without asyncio. 17 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | # Licensed to Elasticsearch B.V. under one or more contributor 2 | # license agreements. See the NOTICE file distributed with 3 | # this work for additional information regarding copyright 4 | # ownership. Elasticsearch B.V. licenses this file to you under 5 | # the Apache License, Version 2.0 (the "License"); you may 6 | # not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | from os.path import dirname, join 19 | 20 | from setuptools import find_packages, setup 21 | 22 | VERSION = (8, 18, 0) 23 | __version__ = VERSION 24 | __versionstr__ = ".".join(map(str, VERSION)) 25 | 26 | f = open(join(dirname(__file__), "README")) 27 | long_description = f.read().strip() 28 | f.close() 29 | 30 | install_requires = [ 31 | "python-dateutil", 32 | "typing-extensions", 33 | "elasticsearch>=8.0.0,<9.0.0", 34 | "elastic-transport>=8.0.0,<9.0.0", 35 | ] 36 | 37 | async_requires = [ 38 | "elasticsearch[async]>=8.0.0,<9.0.0", 39 | ] 40 | 41 | develop_requires = [ 42 | "elasticsearch[async]", 43 | "unasync", 44 | "jinja2", 45 | "pytest", 46 | "pytest-cov", 47 | "pytest-mock", 48 | "pytest-asyncio", 49 | "pytz", 50 | "coverage", 51 | # the following three are used by the vectors example and its tests 52 | "nltk", 53 | "sentence_transformers", 54 | "tqdm", 55 | # Override Read the Docs default (sphinx<2 and sphinx-rtd-theme<0.5) 56 | "sphinx>2", 57 | "sphinx-rtd-theme>0.5", 58 | # typing support 59 | "mypy", 60 | "pyright", 61 | "types-python-dateutil", 62 | "types-pytz", 63 | "types-tqdm", 64 | ] 65 | 66 | setup( 67 | name="elasticsearch-dsl", 68 | description="Python client for Elasticsearch", 69 | license="Apache-2.0", 70 | url="https://github.com/elasticsearch/elasticsearch-dsl-py", 71 | long_description=long_description, 72 | long_description_content_type="text/x-rst", 73 | version=__versionstr__, 74 | author="Elastic Client Library Maintainers", 75 | author_email="client-libs@elastic.co", 76 | packages=find_packages(where=".", exclude=("tests*",)), 77 | python_requires=">=3.8", 78 | classifiers=[ 79 | "Development Status :: 4 - Beta", 80 | "License :: OSI Approved :: Apache Software License", 81 | "Intended Audience :: Developers", 82 | "Operating System :: OS Independent", 83 | "Programming Language :: Python", 84 | "Programming Language :: Python :: 3", 85 | "Programming Language :: Python :: 3 :: Only", 86 | "Programming Language :: Python :: 3.8", 87 | "Programming Language :: Python :: 3.9", 88 | "Programming Language :: Python :: 3.10", 89 | "Programming Language :: Python :: 3.11", 90 | "Programming Language :: Python :: 3.12", 91 | "Programming Language :: Python :: 3.13", 92 | "Programming Language :: Python :: Implementation :: CPython", 93 | "Programming Language :: Python :: Implementation :: PyPy", 94 | ], 95 | install_requires=install_requires, 96 | extras_require={"async": async_requires, "develop": develop_requires}, 97 | ) 98 | -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- 1 | # Licensed to Elasticsearch B.V. under one or more contributor 2 | # license agreements. See the NOTICE file distributed with 3 | # this work for additional information regarding copyright 4 | # ownership. Elasticsearch B.V. licenses this file to you under 5 | # the Apache License, Version 2.0 (the "License"); you may 6 | # not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | -------------------------------------------------------------------------------- /tests/_async/__init__.py: -------------------------------------------------------------------------------- 1 | # Licensed to Elasticsearch B.V. under one or more contributor 2 | # license agreements. See the NOTICE file distributed with 3 | # this work for additional information regarding copyright 4 | # ownership. Elasticsearch B.V. licenses this file to you under 5 | # the Apache License, Version 2.0 (the "License"); you may 6 | # not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | -------------------------------------------------------------------------------- /tests/_async/test_update_by_query.py: -------------------------------------------------------------------------------- 1 | # Licensed to Elasticsearch B.V. under one or more contributor 2 | # license agreements. See the NOTICE file distributed with 3 | # this work for additional information regarding copyright 4 | # ownership. Elasticsearch B.V. licenses this file to you under 5 | # the Apache License, Version 2.0 (the "License"); you may 6 | # not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | from copy import deepcopy 19 | from typing import Any 20 | 21 | import pytest 22 | 23 | from elasticsearch_dsl import AsyncUpdateByQuery, Q 24 | from elasticsearch_dsl.response import UpdateByQueryResponse 25 | from elasticsearch_dsl.search_base import SearchBase 26 | 27 | 28 | def test_ubq_starts_with_no_query() -> None: 29 | ubq = AsyncUpdateByQuery() 30 | 31 | assert ubq.query._proxied is None 32 | 33 | 34 | def test_ubq_to_dict() -> None: 35 | ubq = AsyncUpdateByQuery() 36 | assert {} == ubq.to_dict() 37 | 38 | ubq = ubq.query("match", f=42) 39 | assert {"query": {"match": {"f": 42}}} == ubq.to_dict() 40 | 41 | assert {"query": {"match": {"f": 42}}, "size": 10} == ubq.to_dict(size=10) 42 | 43 | ubq = AsyncUpdateByQuery(extra={"size": 5}) 44 | assert {"size": 5} == ubq.to_dict() 45 | 46 | ubq = AsyncUpdateByQuery(extra={"extra_q": Q("term", category="conference")}) 47 | assert {"extra_q": {"term": {"category": "conference"}}} == ubq.to_dict() 48 | 49 | 50 | def test_complex_example() -> None: 51 | ubq = AsyncUpdateByQuery() 52 | ubq = ( 53 | ubq.query("match", title="python") 54 | .query(~Q("match", title="ruby")) 55 | .filter(Q("term", category="meetup") | Q("term", category="conference")) 56 | .script( 57 | source="ctx._source.likes += params.f", lang="painless", params={"f": 3} 58 | ) 59 | ) 60 | 61 | ubq.query.minimum_should_match = 2 62 | assert { 63 | "query": { 64 | "bool": { 65 | "filter": [ 66 | { 67 | "bool": { 68 | "should": [ 69 | {"term": {"category": "meetup"}}, 70 | {"term": {"category": "conference"}}, 71 | ] 72 | } 73 | } 74 | ], 75 | "must": [{"match": {"title": "python"}}], 76 | "must_not": [{"match": {"title": "ruby"}}], 77 | "minimum_should_match": 2, 78 | } 79 | }, 80 | "script": { 81 | "source": "ctx._source.likes += params.f", 82 | "lang": "painless", 83 | "params": {"f": 3}, 84 | }, 85 | } == ubq.to_dict() 86 | 87 | 88 | def test_exclude() -> None: 89 | ubq = AsyncUpdateByQuery() 90 | ubq = ubq.exclude("match", title="python") 91 | 92 | assert { 93 | "query": { 94 | "bool": { 95 | "filter": [{"bool": {"must_not": [{"match": {"title": "python"}}]}}] 96 | } 97 | } 98 | } == ubq.to_dict() 99 | 100 | 101 | def test_reverse() -> None: 102 | d = { 103 | "query": { 104 | "bool": { 105 | "filter": [ 106 | { 107 | "bool": { 108 | "should": [ 109 | {"term": {"category": "meetup"}}, 110 | {"term": {"category": "conference"}}, 111 | ] 112 | } 113 | } 114 | ], 115 | "must": [ 116 | { 117 | "bool": { 118 | "must": [{"match": {"title": "python"}}], 119 | "must_not": [{"match": {"title": "ruby"}}], 120 | "minimum_should_match": 2, 121 | } 122 | } 123 | ], 124 | } 125 | }, 126 | "script": { 127 | "source": "ctx._source.likes += params.f", 128 | "lang": "painless", 129 | "params": {"f": 3}, 130 | }, 131 | } 132 | 133 | d2 = deepcopy(d) 134 | 135 | ubq = AsyncUpdateByQuery.from_dict(d) 136 | 137 | assert d == d2 138 | assert d == ubq.to_dict() 139 | 140 | 141 | def test_from_dict_doesnt_need_query() -> None: 142 | ubq = AsyncUpdateByQuery.from_dict({"script": {"source": "test"}}) 143 | 144 | assert {"script": {"source": "test"}} == ubq.to_dict() 145 | 146 | 147 | @pytest.mark.asyncio 148 | async def test_params_being_passed_to_search(async_mock_client: Any) -> None: 149 | ubq = AsyncUpdateByQuery(using="mock", index="i") 150 | ubq = ubq.params(routing="42") 151 | await ubq.execute() 152 | 153 | async_mock_client.update_by_query.assert_called_once_with(index=["i"], routing="42") 154 | 155 | 156 | def test_overwrite_script() -> None: 157 | ubq = AsyncUpdateByQuery() 158 | ubq = ubq.script( 159 | source="ctx._source.likes += params.f", lang="painless", params={"f": 3} 160 | ) 161 | assert { 162 | "script": { 163 | "source": "ctx._source.likes += params.f", 164 | "lang": "painless", 165 | "params": {"f": 3}, 166 | } 167 | } == ubq.to_dict() 168 | ubq = ubq.script(source="ctx._source.likes++") 169 | assert {"script": {"source": "ctx._source.likes++"}} == ubq.to_dict() 170 | 171 | 172 | def test_update_by_query_response_success() -> None: 173 | ubqr = UpdateByQueryResponse(SearchBase(), {"timed_out": False, "failures": []}) 174 | assert ubqr.success() 175 | 176 | ubqr = UpdateByQueryResponse(SearchBase(), {"timed_out": True, "failures": []}) 177 | assert not ubqr.success() 178 | 179 | ubqr = UpdateByQueryResponse(SearchBase(), {"timed_out": False, "failures": [{}]}) 180 | assert not ubqr.success() 181 | -------------------------------------------------------------------------------- /tests/_sync/__init__.py: -------------------------------------------------------------------------------- 1 | # Licensed to Elasticsearch B.V. under one or more contributor 2 | # license agreements. See the NOTICE file distributed with 3 | # this work for additional information regarding copyright 4 | # ownership. Elasticsearch B.V. licenses this file to you under 5 | # the Apache License, Version 2.0 (the "License"); you may 6 | # not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | -------------------------------------------------------------------------------- /tests/_sync/test_update_by_query.py: -------------------------------------------------------------------------------- 1 | # Licensed to Elasticsearch B.V. under one or more contributor 2 | # license agreements. See the NOTICE file distributed with 3 | # this work for additional information regarding copyright 4 | # ownership. Elasticsearch B.V. licenses this file to you under 5 | # the Apache License, Version 2.0 (the "License"); you may 6 | # not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | from copy import deepcopy 19 | from typing import Any 20 | 21 | import pytest 22 | 23 | from elasticsearch_dsl import Q, UpdateByQuery 24 | from elasticsearch_dsl.response import UpdateByQueryResponse 25 | from elasticsearch_dsl.search_base import SearchBase 26 | 27 | 28 | def test_ubq_starts_with_no_query() -> None: 29 | ubq = UpdateByQuery() 30 | 31 | assert ubq.query._proxied is None 32 | 33 | 34 | def test_ubq_to_dict() -> None: 35 | ubq = UpdateByQuery() 36 | assert {} == ubq.to_dict() 37 | 38 | ubq = ubq.query("match", f=42) 39 | assert {"query": {"match": {"f": 42}}} == ubq.to_dict() 40 | 41 | assert {"query": {"match": {"f": 42}}, "size": 10} == ubq.to_dict(size=10) 42 | 43 | ubq = UpdateByQuery(extra={"size": 5}) 44 | assert {"size": 5} == ubq.to_dict() 45 | 46 | ubq = UpdateByQuery(extra={"extra_q": Q("term", category="conference")}) 47 | assert {"extra_q": {"term": {"category": "conference"}}} == ubq.to_dict() 48 | 49 | 50 | def test_complex_example() -> None: 51 | ubq = UpdateByQuery() 52 | ubq = ( 53 | ubq.query("match", title="python") 54 | .query(~Q("match", title="ruby")) 55 | .filter(Q("term", category="meetup") | Q("term", category="conference")) 56 | .script( 57 | source="ctx._source.likes += params.f", lang="painless", params={"f": 3} 58 | ) 59 | ) 60 | 61 | ubq.query.minimum_should_match = 2 62 | assert { 63 | "query": { 64 | "bool": { 65 | "filter": [ 66 | { 67 | "bool": { 68 | "should": [ 69 | {"term": {"category": "meetup"}}, 70 | {"term": {"category": "conference"}}, 71 | ] 72 | } 73 | } 74 | ], 75 | "must": [{"match": {"title": "python"}}], 76 | "must_not": [{"match": {"title": "ruby"}}], 77 | "minimum_should_match": 2, 78 | } 79 | }, 80 | "script": { 81 | "source": "ctx._source.likes += params.f", 82 | "lang": "painless", 83 | "params": {"f": 3}, 84 | }, 85 | } == ubq.to_dict() 86 | 87 | 88 | def test_exclude() -> None: 89 | ubq = UpdateByQuery() 90 | ubq = ubq.exclude("match", title="python") 91 | 92 | assert { 93 | "query": { 94 | "bool": { 95 | "filter": [{"bool": {"must_not": [{"match": {"title": "python"}}]}}] 96 | } 97 | } 98 | } == ubq.to_dict() 99 | 100 | 101 | def test_reverse() -> None: 102 | d = { 103 | "query": { 104 | "bool": { 105 | "filter": [ 106 | { 107 | "bool": { 108 | "should": [ 109 | {"term": {"category": "meetup"}}, 110 | {"term": {"category": "conference"}}, 111 | ] 112 | } 113 | } 114 | ], 115 | "must": [ 116 | { 117 | "bool": { 118 | "must": [{"match": {"title": "python"}}], 119 | "must_not": [{"match": {"title": "ruby"}}], 120 | "minimum_should_match": 2, 121 | } 122 | } 123 | ], 124 | } 125 | }, 126 | "script": { 127 | "source": "ctx._source.likes += params.f", 128 | "lang": "painless", 129 | "params": {"f": 3}, 130 | }, 131 | } 132 | 133 | d2 = deepcopy(d) 134 | 135 | ubq = UpdateByQuery.from_dict(d) 136 | 137 | assert d == d2 138 | assert d == ubq.to_dict() 139 | 140 | 141 | def test_from_dict_doesnt_need_query() -> None: 142 | ubq = UpdateByQuery.from_dict({"script": {"source": "test"}}) 143 | 144 | assert {"script": {"source": "test"}} == ubq.to_dict() 145 | 146 | 147 | @pytest.mark.sync 148 | def test_params_being_passed_to_search(mock_client: Any) -> None: 149 | ubq = UpdateByQuery(using="mock", index="i") 150 | ubq = ubq.params(routing="42") 151 | ubq.execute() 152 | 153 | mock_client.update_by_query.assert_called_once_with(index=["i"], routing="42") 154 | 155 | 156 | def test_overwrite_script() -> None: 157 | ubq = UpdateByQuery() 158 | ubq = ubq.script( 159 | source="ctx._source.likes += params.f", lang="painless", params={"f": 3} 160 | ) 161 | assert { 162 | "script": { 163 | "source": "ctx._source.likes += params.f", 164 | "lang": "painless", 165 | "params": {"f": 3}, 166 | } 167 | } == ubq.to_dict() 168 | ubq = ubq.script(source="ctx._source.likes++") 169 | assert {"script": {"source": "ctx._source.likes++"}} == ubq.to_dict() 170 | 171 | 172 | def test_update_by_query_response_success() -> None: 173 | ubqr = UpdateByQueryResponse(SearchBase(), {"timed_out": False, "failures": []}) 174 | assert ubqr.success() 175 | 176 | ubqr = UpdateByQueryResponse(SearchBase(), {"timed_out": True, "failures": []}) 177 | assert not ubqr.success() 178 | 179 | ubqr = UpdateByQueryResponse(SearchBase(), {"timed_out": False, "failures": [{}]}) 180 | assert not ubqr.success() 181 | -------------------------------------------------------------------------------- /tests/async_sleep.py: -------------------------------------------------------------------------------- 1 | # Licensed to Elasticsearch B.V. under one or more contributor 2 | # license agreements. See the NOTICE file distributed with 3 | # this work for additional information regarding copyright 4 | # ownership. Elasticsearch B.V. licenses this file to you under 5 | # the Apache License, Version 2.0 (the "License"); you may 6 | # not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | import asyncio 19 | from typing import Union 20 | 21 | 22 | async def sleep(secs: Union[int, float]) -> None: 23 | """Tests can use this function to sleep.""" 24 | await asyncio.sleep(secs) 25 | -------------------------------------------------------------------------------- /tests/sleep.py: -------------------------------------------------------------------------------- 1 | # Licensed to Elasticsearch B.V. under one or more contributor 2 | # license agreements. See the NOTICE file distributed with 3 | # this work for additional information regarding copyright 4 | # ownership. Elasticsearch B.V. licenses this file to you under 5 | # the Apache License, Version 2.0 (the "License"); you may 6 | # not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | import time 19 | from typing import Union 20 | 21 | 22 | def sleep(secs: Union[int, float]) -> None: 23 | """Tests can use this function to sleep.""" 24 | time.sleep(secs) 25 | -------------------------------------------------------------------------------- /tests/test_connections.py: -------------------------------------------------------------------------------- 1 | # Licensed to Elasticsearch B.V. under one or more contributor 2 | # license agreements. See the NOTICE file distributed with 3 | # this work for additional information regarding copyright 4 | # ownership. Elasticsearch B.V. licenses this file to you under 5 | # the Apache License, Version 2.0 (the "License"); you may 6 | # not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | from typing import Any, List 19 | 20 | from elasticsearch import Elasticsearch 21 | from pytest import raises 22 | 23 | from elasticsearch_dsl import connections, serializer 24 | 25 | 26 | class DummyElasticsearch: 27 | def __init__(self, *args: Any, hosts: List[str], **kwargs: Any): 28 | self.hosts = hosts 29 | 30 | 31 | def test_default_connection_is_returned_by_default() -> None: 32 | c = connections.Connections[object](elasticsearch_class=object) 33 | 34 | con, con2 = object(), object() 35 | c.add_connection("default", con) 36 | 37 | c.add_connection("not-default", con2) 38 | 39 | assert c.get_connection() is con 40 | 41 | 42 | def test_get_connection_created_connection_if_needed() -> None: 43 | c = connections.Connections[DummyElasticsearch]( 44 | elasticsearch_class=DummyElasticsearch 45 | ) 46 | c.configure( 47 | default={"hosts": ["https://es.com:9200"]}, 48 | local={"hosts": ["https://localhost:9200"]}, 49 | ) 50 | 51 | default = c.get_connection() 52 | local = c.get_connection("local") 53 | 54 | assert isinstance(default, DummyElasticsearch) 55 | assert isinstance(local, DummyElasticsearch) 56 | 57 | assert default.hosts == ["https://es.com:9200"] 58 | assert local.hosts == ["https://localhost:9200"] 59 | 60 | 61 | def test_configure_preserves_unchanged_connections() -> None: 62 | c = connections.Connections[DummyElasticsearch]( 63 | elasticsearch_class=DummyElasticsearch 64 | ) 65 | 66 | c.configure( 67 | default={"hosts": ["https://es.com:9200"]}, 68 | local={"hosts": ["https://localhost:9200"]}, 69 | ) 70 | default = c.get_connection() 71 | local = c.get_connection("local") 72 | 73 | c.configure( 74 | default={"hosts": ["https://not-es.com:9200"]}, 75 | local={"hosts": ["https://localhost:9200"]}, 76 | ) 77 | new_default = c.get_connection() 78 | new_local = c.get_connection("local") 79 | 80 | assert new_local is local 81 | assert new_default is not default 82 | 83 | 84 | def test_remove_connection_removes_both_conn_and_conf() -> None: 85 | c = connections.Connections[object](elasticsearch_class=DummyElasticsearch) 86 | 87 | c.configure( 88 | default={"hosts": ["https://es.com:9200"]}, 89 | local={"hosts": ["https://localhost:9200"]}, 90 | ) 91 | c.add_connection("local2", object()) 92 | 93 | c.remove_connection("default") 94 | c.get_connection("local2") 95 | c.remove_connection("local2") 96 | 97 | with raises(Exception): 98 | c.get_connection("local2") 99 | c.get_connection("default") 100 | 101 | 102 | def test_create_connection_constructs_client() -> None: 103 | c = connections.Connections[DummyElasticsearch]( 104 | elasticsearch_class=DummyElasticsearch 105 | ) 106 | c.create_connection("testing", hosts=["https://es.com:9200"]) 107 | 108 | con = c.get_connection("testing") 109 | assert con.hosts == ["https://es.com:9200"] 110 | 111 | 112 | def test_create_connection_adds_our_serializer() -> None: 113 | c = connections.Connections[Elasticsearch](elasticsearch_class=Elasticsearch) 114 | c.create_connection("testing", hosts=["https://es.com:9200"]) 115 | 116 | c_serializers = c.get_connection("testing").transport.serializers 117 | assert c_serializers.serializers["application/json"] is serializer.serializer 118 | 119 | 120 | def test_connection_has_correct_user_agent() -> None: 121 | c = connections.Connections[Elasticsearch](elasticsearch_class=Elasticsearch) 122 | 123 | c.create_connection("testing", hosts=["https://es.com:9200"]) 124 | assert ( 125 | c.get_connection("testing") 126 | ._headers["user-agent"] 127 | .startswith("elasticsearch-dsl-py/") 128 | ) 129 | 130 | my_client = Elasticsearch(hosts=["http://localhost:9200"]) 131 | my_client = my_client.options(headers={"user-agent": "my-user-agent/1.0"}) 132 | c.add_connection("default", my_client) 133 | assert c.get_connection()._headers["user-agent"].startswith("elasticsearch-dsl-py/") 134 | 135 | my_client = Elasticsearch(hosts=["http://localhost:9200"]) 136 | assert ( 137 | c.get_connection(my_client) 138 | ._headers["user-agent"] 139 | .startswith("elasticsearch-dsl-py/") 140 | ) 141 | 142 | not_a_client = object() 143 | assert c.get_connection(not_a_client) == not_a_client # type: ignore[arg-type] 144 | -------------------------------------------------------------------------------- /tests/test_integration/__init__.py: -------------------------------------------------------------------------------- 1 | # Licensed to Elasticsearch B.V. under one or more contributor 2 | # license agreements. See the NOTICE file distributed with 3 | # this work for additional information regarding copyright 4 | # ownership. Elasticsearch B.V. licenses this file to you under 5 | # the Apache License, Version 2.0 (the "License"); you may 6 | # not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | -------------------------------------------------------------------------------- /tests/test_integration/_async/__init__.py: -------------------------------------------------------------------------------- 1 | # Licensed to Elasticsearch B.V. under one or more contributor 2 | # license agreements. See the NOTICE file distributed with 3 | # this work for additional information regarding copyright 4 | # ownership. Elasticsearch B.V. licenses this file to you under 5 | # the Apache License, Version 2.0 (the "License"); you may 6 | # not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | -------------------------------------------------------------------------------- /tests/test_integration/_async/test_analysis.py: -------------------------------------------------------------------------------- 1 | # Licensed to Elasticsearch B.V. under one or more contributor 2 | # license agreements. See the NOTICE file distributed with 3 | # this work for additional information regarding copyright 4 | # ownership. Elasticsearch B.V. licenses this file to you under 5 | # the Apache License, Version 2.0 (the "License"); you may 6 | # not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | import pytest 19 | from elasticsearch import AsyncElasticsearch 20 | 21 | from elasticsearch_dsl import analyzer, token_filter, tokenizer 22 | 23 | 24 | @pytest.mark.asyncio 25 | async def test_simulate_with_just__builtin_tokenizer( 26 | async_client: AsyncElasticsearch, 27 | ) -> None: 28 | a = analyzer("my-analyzer", tokenizer="keyword") 29 | tokens = (await a.async_simulate("Hello World!", using=async_client)).tokens 30 | 31 | assert len(tokens) == 1 32 | assert tokens[0].token == "Hello World!" 33 | 34 | 35 | @pytest.mark.asyncio 36 | async def test_simulate_complex(async_client: AsyncElasticsearch) -> None: 37 | a = analyzer( 38 | "my-analyzer", 39 | tokenizer=tokenizer("split_words", "simple_pattern_split", pattern=":"), 40 | filter=["lowercase", token_filter("no-ifs", "stop", stopwords=["if"])], 41 | ) 42 | 43 | tokens = (await a.async_simulate("if:this:works", using=async_client)).tokens 44 | 45 | assert len(tokens) == 2 46 | assert ["this", "works"] == [t.token for t in tokens] 47 | 48 | 49 | @pytest.mark.asyncio 50 | async def test_simulate_builtin(async_client: AsyncElasticsearch) -> None: 51 | a = analyzer("my-analyzer", "english") 52 | tokens = (await a.async_simulate("fixes running")).tokens 53 | 54 | assert ["fix", "run"] == [t.token for t in tokens] 55 | -------------------------------------------------------------------------------- /tests/test_integration/_async/test_index.py: -------------------------------------------------------------------------------- 1 | # Licensed to Elasticsearch B.V. under one or more contributor 2 | # license agreements. See the NOTICE file distributed with 3 | # this work for additional information regarding copyright 4 | # ownership. Elasticsearch B.V. licenses this file to you under 5 | # the Apache License, Version 2.0 (the "License"); you may 6 | # not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | import pytest 19 | from elasticsearch import AsyncElasticsearch 20 | 21 | from elasticsearch_dsl import ( 22 | AsyncComposableIndexTemplate, 23 | AsyncDocument, 24 | AsyncIndex, 25 | AsyncIndexTemplate, 26 | Date, 27 | Text, 28 | analysis, 29 | ) 30 | 31 | 32 | class Post(AsyncDocument): 33 | title = Text(analyzer=analysis.analyzer("my_analyzer", tokenizer="keyword")) 34 | published_from = Date() 35 | 36 | 37 | @pytest.mark.asyncio 38 | async def test_index_template_works(async_write_client: AsyncElasticsearch) -> None: 39 | it = AsyncIndexTemplate("test-template", "test-legacy-*") 40 | it.document(Post) 41 | it.settings(number_of_replicas=0, number_of_shards=1) 42 | await it.save() 43 | 44 | i = AsyncIndex("test-legacy-blog") 45 | await i.create() 46 | 47 | assert { 48 | "test-legacy-blog": { 49 | "mappings": { 50 | "properties": { 51 | "title": {"type": "text", "analyzer": "my_analyzer"}, 52 | "published_from": {"type": "date"}, 53 | } 54 | } 55 | } 56 | } == await async_write_client.indices.get_mapping(index="test-legacy-blog") 57 | 58 | 59 | @pytest.mark.asyncio 60 | async def test_composable_index_template_works( 61 | async_write_client: AsyncElasticsearch, 62 | ) -> None: 63 | it = AsyncComposableIndexTemplate("test-template", "test-*") 64 | it.document(Post) 65 | it.settings(number_of_replicas=0, number_of_shards=1) 66 | await it.save() 67 | 68 | i = AsyncIndex("test-blog") 69 | await i.create() 70 | 71 | assert { 72 | "test-blog": { 73 | "mappings": { 74 | "properties": { 75 | "title": {"type": "text", "analyzer": "my_analyzer"}, 76 | "published_from": {"type": "date"}, 77 | } 78 | } 79 | } 80 | } == await async_write_client.indices.get_mapping(index="test-blog") 81 | 82 | 83 | @pytest.mark.asyncio 84 | async def test_index_can_be_saved_even_with_settings( 85 | async_write_client: AsyncElasticsearch, 86 | ) -> None: 87 | i = AsyncIndex("test-blog", using=async_write_client) 88 | i.settings(number_of_shards=3, number_of_replicas=0) 89 | await i.save() 90 | i.settings(number_of_replicas=1) 91 | await i.save() 92 | 93 | assert ( 94 | "1" 95 | == (await i.get_settings())["test-blog"]["settings"]["index"][ 96 | "number_of_replicas" 97 | ] 98 | ) 99 | 100 | 101 | @pytest.mark.asyncio 102 | async def test_index_exists(async_data_client: AsyncElasticsearch) -> None: 103 | assert await AsyncIndex("git").exists() 104 | assert not await AsyncIndex("not-there").exists() 105 | 106 | 107 | @pytest.mark.asyncio 108 | async def test_index_can_be_created_with_settings_and_mappings( 109 | async_write_client: AsyncElasticsearch, 110 | ) -> None: 111 | i = AsyncIndex("test-blog", using=async_write_client) 112 | i.document(Post) 113 | i.settings(number_of_replicas=0, number_of_shards=1) 114 | await i.create() 115 | 116 | assert { 117 | "test-blog": { 118 | "mappings": { 119 | "properties": { 120 | "title": {"type": "text", "analyzer": "my_analyzer"}, 121 | "published_from": {"type": "date"}, 122 | } 123 | } 124 | } 125 | } == await async_write_client.indices.get_mapping(index="test-blog") 126 | 127 | settings = await async_write_client.indices.get_settings(index="test-blog") 128 | assert settings["test-blog"]["settings"]["index"]["number_of_replicas"] == "0" 129 | assert settings["test-blog"]["settings"]["index"]["number_of_shards"] == "1" 130 | assert settings["test-blog"]["settings"]["index"]["analysis"] == { 131 | "analyzer": {"my_analyzer": {"type": "custom", "tokenizer": "keyword"}} 132 | } 133 | 134 | 135 | @pytest.mark.asyncio 136 | async def test_delete(async_write_client: AsyncElasticsearch) -> None: 137 | await async_write_client.indices.create( 138 | index="test-index", 139 | body={"settings": {"number_of_replicas": 0, "number_of_shards": 1}}, 140 | ) 141 | 142 | i = AsyncIndex("test-index", using=async_write_client) 143 | await i.delete() 144 | assert not await async_write_client.indices.exists(index="test-index") 145 | 146 | 147 | @pytest.mark.asyncio 148 | async def test_multiple_indices_with_same_doc_type_work( 149 | async_write_client: AsyncElasticsearch, 150 | ) -> None: 151 | i1 = AsyncIndex("test-index-1", using=async_write_client) 152 | i2 = AsyncIndex("test-index-2", using=async_write_client) 153 | 154 | for i in (i1, i2): 155 | i.document(Post) 156 | await i.create() 157 | 158 | for j in ("test-index-1", "test-index-2"): 159 | settings = await async_write_client.indices.get_settings(index=j) 160 | assert settings[j]["settings"]["index"]["analysis"] == { 161 | "analyzer": {"my_analyzer": {"type": "custom", "tokenizer": "keyword"}} 162 | } 163 | -------------------------------------------------------------------------------- /tests/test_integration/_async/test_update_by_query.py: -------------------------------------------------------------------------------- 1 | # Licensed to Elasticsearch B.V. under one or more contributor 2 | # license agreements. See the NOTICE file distributed with 3 | # this work for additional information regarding copyright 4 | # ownership. Elasticsearch B.V. licenses this file to you under 5 | # the Apache License, Version 2.0 (the "License"); you may 6 | # not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | import pytest 19 | from elasticsearch import AsyncElasticsearch 20 | 21 | from elasticsearch_dsl import AsyncUpdateByQuery 22 | from elasticsearch_dsl.search import Q 23 | 24 | 25 | @pytest.mark.asyncio 26 | async def test_update_by_query_no_script( 27 | async_write_client: AsyncElasticsearch, setup_ubq_tests: str 28 | ) -> None: 29 | index = setup_ubq_tests 30 | 31 | ubq = ( 32 | AsyncUpdateByQuery(using=async_write_client) 33 | .index(index) 34 | .filter(~Q("exists", field="is_public")) 35 | ) 36 | response = await ubq.execute() 37 | 38 | assert response.total == 52 39 | assert response["took"] > 0 40 | assert not response.timed_out 41 | assert response.updated == 52 42 | assert response.deleted == 0 43 | assert response.took > 0 44 | assert response.success() 45 | 46 | 47 | @pytest.mark.asyncio 48 | async def test_update_by_query_with_script( 49 | async_write_client: AsyncElasticsearch, setup_ubq_tests: str 50 | ) -> None: 51 | index = setup_ubq_tests 52 | 53 | ubq = ( 54 | AsyncUpdateByQuery(using=async_write_client) 55 | .index(index) 56 | .filter(~Q("exists", field="parent_shas")) 57 | .script(source="ctx._source.is_public = false") 58 | ) 59 | ubq = ubq.params(conflicts="proceed") 60 | 61 | response = await ubq.execute() 62 | assert response.total == 2 63 | assert response.updated == 2 64 | assert response.version_conflicts == 0 65 | 66 | 67 | @pytest.mark.asyncio 68 | async def test_delete_by_query_with_script( 69 | async_write_client: AsyncElasticsearch, setup_ubq_tests: str 70 | ) -> None: 71 | index = setup_ubq_tests 72 | 73 | ubq = ( 74 | AsyncUpdateByQuery(using=async_write_client) 75 | .index(index) 76 | .filter(Q("match", parent_shas="1dd19210b5be92b960f7db6f66ae526288edccc3")) 77 | .script(source='ctx.op = "delete"') 78 | ) 79 | ubq = ubq.params(conflicts="proceed") 80 | 81 | response = await ubq.execute() 82 | 83 | assert response.total == 1 84 | assert response.deleted == 1 85 | assert response.success() 86 | -------------------------------------------------------------------------------- /tests/test_integration/_sync/__init__.py: -------------------------------------------------------------------------------- 1 | # Licensed to Elasticsearch B.V. under one or more contributor 2 | # license agreements. See the NOTICE file distributed with 3 | # this work for additional information regarding copyright 4 | # ownership. Elasticsearch B.V. licenses this file to you under 5 | # the Apache License, Version 2.0 (the "License"); you may 6 | # not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | -------------------------------------------------------------------------------- /tests/test_integration/_sync/test_analysis.py: -------------------------------------------------------------------------------- 1 | # Licensed to Elasticsearch B.V. under one or more contributor 2 | # license agreements. See the NOTICE file distributed with 3 | # this work for additional information regarding copyright 4 | # ownership. Elasticsearch B.V. licenses this file to you under 5 | # the Apache License, Version 2.0 (the "License"); you may 6 | # not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | import pytest 19 | from elasticsearch import Elasticsearch 20 | 21 | from elasticsearch_dsl import analyzer, token_filter, tokenizer 22 | 23 | 24 | @pytest.mark.sync 25 | def test_simulate_with_just__builtin_tokenizer( 26 | client: Elasticsearch, 27 | ) -> None: 28 | a = analyzer("my-analyzer", tokenizer="keyword") 29 | tokens = (a.simulate("Hello World!", using=client)).tokens 30 | 31 | assert len(tokens) == 1 32 | assert tokens[0].token == "Hello World!" 33 | 34 | 35 | @pytest.mark.sync 36 | def test_simulate_complex(client: Elasticsearch) -> None: 37 | a = analyzer( 38 | "my-analyzer", 39 | tokenizer=tokenizer("split_words", "simple_pattern_split", pattern=":"), 40 | filter=["lowercase", token_filter("no-ifs", "stop", stopwords=["if"])], 41 | ) 42 | 43 | tokens = (a.simulate("if:this:works", using=client)).tokens 44 | 45 | assert len(tokens) == 2 46 | assert ["this", "works"] == [t.token for t in tokens] 47 | 48 | 49 | @pytest.mark.sync 50 | def test_simulate_builtin(client: Elasticsearch) -> None: 51 | a = analyzer("my-analyzer", "english") 52 | tokens = (a.simulate("fixes running")).tokens 53 | 54 | assert ["fix", "run"] == [t.token for t in tokens] 55 | -------------------------------------------------------------------------------- /tests/test_integration/_sync/test_index.py: -------------------------------------------------------------------------------- 1 | # Licensed to Elasticsearch B.V. under one or more contributor 2 | # license agreements. See the NOTICE file distributed with 3 | # this work for additional information regarding copyright 4 | # ownership. Elasticsearch B.V. licenses this file to you under 5 | # the Apache License, Version 2.0 (the "License"); you may 6 | # not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | import pytest 19 | from elasticsearch import Elasticsearch 20 | 21 | from elasticsearch_dsl import ( 22 | ComposableIndexTemplate, 23 | Date, 24 | Document, 25 | Index, 26 | IndexTemplate, 27 | Text, 28 | analysis, 29 | ) 30 | 31 | 32 | class Post(Document): 33 | title = Text(analyzer=analysis.analyzer("my_analyzer", tokenizer="keyword")) 34 | published_from = Date() 35 | 36 | 37 | @pytest.mark.sync 38 | def test_index_template_works(write_client: Elasticsearch) -> None: 39 | it = IndexTemplate("test-template", "test-legacy-*") 40 | it.document(Post) 41 | it.settings(number_of_replicas=0, number_of_shards=1) 42 | it.save() 43 | 44 | i = Index("test-legacy-blog") 45 | i.create() 46 | 47 | assert { 48 | "test-legacy-blog": { 49 | "mappings": { 50 | "properties": { 51 | "title": {"type": "text", "analyzer": "my_analyzer"}, 52 | "published_from": {"type": "date"}, 53 | } 54 | } 55 | } 56 | } == write_client.indices.get_mapping(index="test-legacy-blog") 57 | 58 | 59 | @pytest.mark.sync 60 | def test_composable_index_template_works( 61 | write_client: Elasticsearch, 62 | ) -> None: 63 | it = ComposableIndexTemplate("test-template", "test-*") 64 | it.document(Post) 65 | it.settings(number_of_replicas=0, number_of_shards=1) 66 | it.save() 67 | 68 | i = Index("test-blog") 69 | i.create() 70 | 71 | assert { 72 | "test-blog": { 73 | "mappings": { 74 | "properties": { 75 | "title": {"type": "text", "analyzer": "my_analyzer"}, 76 | "published_from": {"type": "date"}, 77 | } 78 | } 79 | } 80 | } == write_client.indices.get_mapping(index="test-blog") 81 | 82 | 83 | @pytest.mark.sync 84 | def test_index_can_be_saved_even_with_settings( 85 | write_client: Elasticsearch, 86 | ) -> None: 87 | i = Index("test-blog", using=write_client) 88 | i.settings(number_of_shards=3, number_of_replicas=0) 89 | i.save() 90 | i.settings(number_of_replicas=1) 91 | i.save() 92 | 93 | assert ( 94 | "1" 95 | == (i.get_settings())["test-blog"]["settings"]["index"]["number_of_replicas"] 96 | ) 97 | 98 | 99 | @pytest.mark.sync 100 | def test_index_exists(data_client: Elasticsearch) -> None: 101 | assert Index("git").exists() 102 | assert not Index("not-there").exists() 103 | 104 | 105 | @pytest.mark.sync 106 | def test_index_can_be_created_with_settings_and_mappings( 107 | write_client: Elasticsearch, 108 | ) -> None: 109 | i = Index("test-blog", using=write_client) 110 | i.document(Post) 111 | i.settings(number_of_replicas=0, number_of_shards=1) 112 | i.create() 113 | 114 | assert { 115 | "test-blog": { 116 | "mappings": { 117 | "properties": { 118 | "title": {"type": "text", "analyzer": "my_analyzer"}, 119 | "published_from": {"type": "date"}, 120 | } 121 | } 122 | } 123 | } == write_client.indices.get_mapping(index="test-blog") 124 | 125 | settings = write_client.indices.get_settings(index="test-blog") 126 | assert settings["test-blog"]["settings"]["index"]["number_of_replicas"] == "0" 127 | assert settings["test-blog"]["settings"]["index"]["number_of_shards"] == "1" 128 | assert settings["test-blog"]["settings"]["index"]["analysis"] == { 129 | "analyzer": {"my_analyzer": {"type": "custom", "tokenizer": "keyword"}} 130 | } 131 | 132 | 133 | @pytest.mark.sync 134 | def test_delete(write_client: Elasticsearch) -> None: 135 | write_client.indices.create( 136 | index="test-index", 137 | body={"settings": {"number_of_replicas": 0, "number_of_shards": 1}}, 138 | ) 139 | 140 | i = Index("test-index", using=write_client) 141 | i.delete() 142 | assert not write_client.indices.exists(index="test-index") 143 | 144 | 145 | @pytest.mark.sync 146 | def test_multiple_indices_with_same_doc_type_work( 147 | write_client: Elasticsearch, 148 | ) -> None: 149 | i1 = Index("test-index-1", using=write_client) 150 | i2 = Index("test-index-2", using=write_client) 151 | 152 | for i in (i1, i2): 153 | i.document(Post) 154 | i.create() 155 | 156 | for j in ("test-index-1", "test-index-2"): 157 | settings = write_client.indices.get_settings(index=j) 158 | assert settings[j]["settings"]["index"]["analysis"] == { 159 | "analyzer": {"my_analyzer": {"type": "custom", "tokenizer": "keyword"}} 160 | } 161 | -------------------------------------------------------------------------------- /tests/test_integration/_sync/test_mapping.py: -------------------------------------------------------------------------------- 1 | # Licensed to Elasticsearch B.V. under one or more contributor 2 | # license agreements. See the NOTICE file distributed with 3 | # this work for additional information regarding copyright 4 | # ownership. Elasticsearch B.V. licenses this file to you under 5 | # the Apache License, Version 2.0 (the "License"); you may 6 | # not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | import pytest 19 | from elasticsearch import Elasticsearch 20 | from pytest import raises 21 | 22 | from elasticsearch_dsl import Mapping, analysis, exceptions 23 | 24 | 25 | @pytest.mark.sync 26 | def test_mapping_saved_into_es(write_client: Elasticsearch) -> None: 27 | m = Mapping() 28 | m.field( 29 | "name", "text", analyzer=analysis.analyzer("my_analyzer", tokenizer="keyword") 30 | ) 31 | m.field("tags", "keyword") 32 | m.save("test-mapping", using=write_client) 33 | 34 | assert { 35 | "test-mapping": { 36 | "mappings": { 37 | "properties": { 38 | "name": {"type": "text", "analyzer": "my_analyzer"}, 39 | "tags": {"type": "keyword"}, 40 | } 41 | } 42 | } 43 | } == write_client.indices.get_mapping(index="test-mapping") 44 | 45 | 46 | @pytest.mark.sync 47 | def test_mapping_saved_into_es_when_index_already_exists_closed( 48 | write_client: Elasticsearch, 49 | ) -> None: 50 | m = Mapping() 51 | m.field( 52 | "name", "text", analyzer=analysis.analyzer("my_analyzer", tokenizer="keyword") 53 | ) 54 | write_client.indices.create(index="test-mapping") 55 | 56 | with raises(exceptions.IllegalOperation): 57 | m.save("test-mapping", using=write_client) 58 | 59 | write_client.cluster.health(index="test-mapping", wait_for_status="yellow") 60 | write_client.indices.close(index="test-mapping") 61 | m.save("test-mapping", using=write_client) 62 | 63 | assert { 64 | "test-mapping": { 65 | "mappings": { 66 | "properties": {"name": {"type": "text", "analyzer": "my_analyzer"}} 67 | } 68 | } 69 | } == write_client.indices.get_mapping(index="test-mapping") 70 | 71 | 72 | @pytest.mark.sync 73 | def test_mapping_saved_into_es_when_index_already_exists_with_analysis( 74 | write_client: Elasticsearch, 75 | ) -> None: 76 | m = Mapping() 77 | analyzer = analysis.analyzer("my_analyzer", tokenizer="keyword") 78 | m.field("name", "text", analyzer=analyzer) 79 | 80 | new_analysis = analyzer.get_analysis_definition() 81 | new_analysis["analyzer"]["other_analyzer"] = { 82 | "type": "custom", 83 | "tokenizer": "whitespace", 84 | } 85 | write_client.indices.create( 86 | index="test-mapping", body={"settings": {"analysis": new_analysis}} 87 | ) 88 | 89 | m.field("title", "text", analyzer=analyzer) 90 | m.save("test-mapping", using=write_client) 91 | 92 | assert { 93 | "test-mapping": { 94 | "mappings": { 95 | "properties": { 96 | "name": {"type": "text", "analyzer": "my_analyzer"}, 97 | "title": {"type": "text", "analyzer": "my_analyzer"}, 98 | } 99 | } 100 | } 101 | } == write_client.indices.get_mapping(index="test-mapping") 102 | 103 | 104 | @pytest.mark.sync 105 | def test_mapping_gets_updated_from_es( 106 | write_client: Elasticsearch, 107 | ) -> None: 108 | write_client.indices.create( 109 | index="test-mapping", 110 | body={ 111 | "settings": {"number_of_shards": 1, "number_of_replicas": 0}, 112 | "mappings": { 113 | "date_detection": False, 114 | "properties": { 115 | "title": { 116 | "type": "text", 117 | "analyzer": "snowball", 118 | "fields": {"raw": {"type": "keyword"}}, 119 | }, 120 | "created_at": {"type": "date"}, 121 | "comments": { 122 | "type": "nested", 123 | "properties": { 124 | "created": {"type": "date"}, 125 | "author": { 126 | "type": "text", 127 | "analyzer": "snowball", 128 | "fields": {"raw": {"type": "keyword"}}, 129 | }, 130 | }, 131 | }, 132 | }, 133 | }, 134 | }, 135 | ) 136 | 137 | m = Mapping.from_es("test-mapping", using=write_client) 138 | 139 | assert ["comments", "created_at", "title"] == list( 140 | sorted(m.properties.properties._d_.keys()) # type: ignore[attr-defined] 141 | ) 142 | assert { 143 | "date_detection": False, 144 | "properties": { 145 | "comments": { 146 | "type": "nested", 147 | "properties": { 148 | "created": {"type": "date"}, 149 | "author": { 150 | "analyzer": "snowball", 151 | "fields": {"raw": {"type": "keyword"}}, 152 | "type": "text", 153 | }, 154 | }, 155 | }, 156 | "created_at": {"type": "date"}, 157 | "title": { 158 | "analyzer": "snowball", 159 | "fields": {"raw": {"type": "keyword"}}, 160 | "type": "text", 161 | }, 162 | }, 163 | } == m.to_dict() 164 | 165 | # test same with alias 166 | write_client.indices.put_alias(index="test-mapping", name="test-alias") 167 | 168 | m2 = Mapping.from_es("test-alias", using=write_client) 169 | assert m2.to_dict() == m.to_dict() 170 | -------------------------------------------------------------------------------- /tests/test_integration/_sync/test_update_by_query.py: -------------------------------------------------------------------------------- 1 | # Licensed to Elasticsearch B.V. under one or more contributor 2 | # license agreements. See the NOTICE file distributed with 3 | # this work for additional information regarding copyright 4 | # ownership. Elasticsearch B.V. licenses this file to you under 5 | # the Apache License, Version 2.0 (the "License"); you may 6 | # not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | import pytest 19 | from elasticsearch import Elasticsearch 20 | 21 | from elasticsearch_dsl import UpdateByQuery 22 | from elasticsearch_dsl.search import Q 23 | 24 | 25 | @pytest.mark.sync 26 | def test_update_by_query_no_script( 27 | write_client: Elasticsearch, setup_ubq_tests: str 28 | ) -> None: 29 | index = setup_ubq_tests 30 | 31 | ubq = ( 32 | UpdateByQuery(using=write_client) 33 | .index(index) 34 | .filter(~Q("exists", field="is_public")) 35 | ) 36 | response = ubq.execute() 37 | 38 | assert response.total == 52 39 | assert response["took"] > 0 40 | assert not response.timed_out 41 | assert response.updated == 52 42 | assert response.deleted == 0 43 | assert response.took > 0 44 | assert response.success() 45 | 46 | 47 | @pytest.mark.sync 48 | def test_update_by_query_with_script( 49 | write_client: Elasticsearch, setup_ubq_tests: str 50 | ) -> None: 51 | index = setup_ubq_tests 52 | 53 | ubq = ( 54 | UpdateByQuery(using=write_client) 55 | .index(index) 56 | .filter(~Q("exists", field="parent_shas")) 57 | .script(source="ctx._source.is_public = false") 58 | ) 59 | ubq = ubq.params(conflicts="proceed") 60 | 61 | response = ubq.execute() 62 | assert response.total == 2 63 | assert response.updated == 2 64 | assert response.version_conflicts == 0 65 | 66 | 67 | @pytest.mark.sync 68 | def test_delete_by_query_with_script( 69 | write_client: Elasticsearch, setup_ubq_tests: str 70 | ) -> None: 71 | index = setup_ubq_tests 72 | 73 | ubq = ( 74 | UpdateByQuery(using=write_client) 75 | .index(index) 76 | .filter(Q("match", parent_shas="1dd19210b5be92b960f7db6f66ae526288edccc3")) 77 | .script(source='ctx.op = "delete"') 78 | ) 79 | ubq = ubq.params(conflicts="proceed") 80 | 81 | response = ubq.execute() 82 | 83 | assert response.total == 1 84 | assert response.deleted == 1 85 | assert response.success() 86 | -------------------------------------------------------------------------------- /tests/test_integration/test_count.py: -------------------------------------------------------------------------------- 1 | # Licensed to Elasticsearch B.V. under one or more contributor 2 | # license agreements. See the NOTICE file distributed with 3 | # this work for additional information regarding copyright 4 | # ownership. Elasticsearch B.V. licenses this file to you under 5 | # the Apache License, Version 2.0 (the "License"); you may 6 | # not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | from typing import Any 19 | 20 | from elasticsearch import Elasticsearch 21 | 22 | from elasticsearch_dsl.search import Q, Search 23 | 24 | 25 | def test_count_all(data_client: Elasticsearch) -> None: 26 | s = Search(using=data_client).index("git") 27 | assert 53 == s.count() 28 | 29 | 30 | def test_count_prefetch(data_client: Elasticsearch, mocker: Any) -> None: 31 | mocker.spy(data_client, "count") 32 | 33 | search = Search(using=data_client).index("git") 34 | search.execute() 35 | assert search.count() == 53 36 | assert data_client.count.call_count == 0 # type: ignore[attr-defined] 37 | 38 | search._response.hits.total.relation = "gte" # type: ignore[attr-defined] 39 | assert search.count() == 53 40 | assert data_client.count.call_count == 1 # type: ignore[attr-defined] 41 | 42 | 43 | def test_count_filter(data_client: Elasticsearch) -> None: 44 | s = Search(using=data_client).index("git").filter(~Q("exists", field="parent_shas")) 45 | # initial commit + repo document 46 | assert 2 == s.count() 47 | -------------------------------------------------------------------------------- /tests/test_integration/test_examples/__init__.py: -------------------------------------------------------------------------------- 1 | # Licensed to Elasticsearch B.V. under one or more contributor 2 | # license agreements. See the NOTICE file distributed with 3 | # this work for additional information regarding copyright 4 | # ownership. Elasticsearch B.V. licenses this file to you under 5 | # the Apache License, Version 2.0 (the "License"); you may 6 | # not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | -------------------------------------------------------------------------------- /tests/test_integration/test_examples/_async/__init__.py: -------------------------------------------------------------------------------- 1 | # Licensed to Elasticsearch B.V. under one or more contributor 2 | # license agreements. See the NOTICE file distributed with 3 | # this work for additional information regarding copyright 4 | # ownership. Elasticsearch B.V. licenses this file to you under 5 | # the Apache License, Version 2.0 (the "License"); you may 6 | # not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | -------------------------------------------------------------------------------- /tests/test_integration/test_examples/_async/test_alias_migration.py: -------------------------------------------------------------------------------- 1 | # Licensed to Elasticsearch B.V. under one or more contributor 2 | # license agreements. See the NOTICE file distributed with 3 | # this work for additional information regarding copyright 4 | # ownership. Elasticsearch B.V. licenses this file to you under 5 | # the Apache License, Version 2.0 (the "License"); you may 6 | # not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | import pytest 19 | from elasticsearch import AsyncElasticsearch 20 | 21 | from ..async_examples import alias_migration 22 | from ..async_examples.alias_migration import ALIAS, PATTERN, BlogPost, migrate 23 | 24 | 25 | @pytest.mark.asyncio 26 | async def test_alias_migration(async_write_client: AsyncElasticsearch) -> None: 27 | # create the index 28 | await alias_migration.setup() 29 | 30 | # verify that template, index, and alias has been set up 31 | assert await async_write_client.indices.exists_index_template(name=ALIAS) 32 | assert await async_write_client.indices.exists(index=PATTERN) 33 | assert await async_write_client.indices.exists_alias(name=ALIAS) 34 | 35 | indices = await async_write_client.indices.get(index=PATTERN) 36 | assert len(indices) == 1 37 | index_name, _ = indices.popitem() 38 | 39 | # which means we can now save a document 40 | with open(__file__) as f: 41 | bp = BlogPost( 42 | _id=0, 43 | title="Hello World!", 44 | tags=["testing", "dummy"], 45 | content=f.read(), 46 | published=None, 47 | ) 48 | await bp.save(refresh=True) 49 | 50 | assert await BlogPost.search().count() == 1 51 | 52 | # _matches work which means we get BlogPost instance 53 | bp = (await BlogPost.search().execute())[0] 54 | assert isinstance(bp, BlogPost) 55 | assert not bp.is_published() 56 | assert "0" == bp.meta.id 57 | 58 | # create new index 59 | await migrate() 60 | 61 | indices = await async_write_client.indices.get(index=PATTERN) 62 | assert 2 == len(indices) 63 | alias = await async_write_client.indices.get(index=ALIAS) 64 | assert 1 == len(alias) 65 | assert index_name not in alias 66 | 67 | # data has been moved properly 68 | assert await BlogPost.search().count() == 1 69 | 70 | # _matches work which means we get BlogPost instance 71 | bp = (await BlogPost.search().execute())[0] 72 | assert isinstance(bp, BlogPost) 73 | assert "0" == bp.meta.id 74 | -------------------------------------------------------------------------------- /tests/test_integration/test_examples/_async/test_completion.py: -------------------------------------------------------------------------------- 1 | # Licensed to Elasticsearch B.V. under one or more contributor 2 | # license agreements. See the NOTICE file distributed with 3 | # this work for additional information regarding copyright 4 | # ownership. Elasticsearch B.V. licenses this file to you under 5 | # the Apache License, Version 2.0 (the "License"); you may 6 | # not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | import pytest 19 | from elasticsearch import AsyncElasticsearch 20 | 21 | from ..async_examples.completion import Person 22 | 23 | 24 | @pytest.mark.asyncio 25 | async def test_person_suggests_on_all_variants_of_name( 26 | async_write_client: AsyncElasticsearch, 27 | ) -> None: 28 | await Person.init(using=async_write_client) 29 | 30 | await Person(_id=None, name="Honza Král", popularity=42).save(refresh=True) 31 | 32 | s = Person.search().suggest("t", "kra", completion={"field": "suggest"}) 33 | response = await s.execute() 34 | 35 | opts = response.suggest["t"][0].options 36 | 37 | assert 1 == len(opts) 38 | assert opts[0]._score == 42 39 | assert opts[0]._source.name == "Honza Král" 40 | -------------------------------------------------------------------------------- /tests/test_integration/test_examples/_async/test_composite_aggs.py: -------------------------------------------------------------------------------- 1 | # Licensed to Elasticsearch B.V. under one or more contributor 2 | # license agreements. See the NOTICE file distributed with 3 | # this work for additional information regarding copyright 4 | # ownership. Elasticsearch B.V. licenses this file to you under 5 | # the Apache License, Version 2.0 (the "License"); you may 6 | # not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | import pytest 19 | from elasticsearch import AsyncElasticsearch 20 | 21 | from elasticsearch_dsl import A, AsyncSearch 22 | 23 | from ..async_examples.composite_agg import scan_aggs 24 | 25 | 26 | @pytest.mark.asyncio 27 | async def test_scan_aggs_exhausts_all_files( 28 | async_data_client: AsyncElasticsearch, 29 | ) -> None: 30 | s = AsyncSearch(index="flat-git") 31 | key_aggs = [{"files": A("terms", field="files")}] 32 | file_list = [f async for f in scan_aggs(s, key_aggs)] 33 | 34 | assert len(file_list) == 26 35 | 36 | 37 | @pytest.mark.asyncio 38 | async def test_scan_aggs_with_multiple_aggs( 39 | async_data_client: AsyncElasticsearch, 40 | ) -> None: 41 | s = AsyncSearch(index="flat-git") 42 | key_aggs = [ 43 | {"files": A("terms", field="files")}, 44 | { 45 | "months": A( 46 | "date_histogram", field="committed_date", calendar_interval="month" 47 | ) 48 | }, 49 | ] 50 | file_list = [ 51 | f 52 | async for f in scan_aggs( 53 | s, key_aggs, {"first_seen": A("min", field="committed_date")} 54 | ) 55 | ] 56 | 57 | assert len(file_list) == 47 58 | -------------------------------------------------------------------------------- /tests/test_integration/test_examples/_async/test_parent_child.py: -------------------------------------------------------------------------------- 1 | # Licensed to Elasticsearch B.V. under one or more contributor 2 | # license agreements. See the NOTICE file distributed with 3 | # this work for additional information regarding copyright 4 | # ownership. Elasticsearch B.V. licenses this file to you under 5 | # the Apache License, Version 2.0 (the "License"); you may 6 | # not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | from datetime import datetime 19 | 20 | import pytest 21 | import pytest_asyncio 22 | from elasticsearch import AsyncElasticsearch 23 | 24 | from elasticsearch_dsl import Q 25 | 26 | from ..async_examples.parent_child import Answer, Comment, Question, User, setup 27 | 28 | honza = User( 29 | id=42, 30 | signed_up=datetime(2013, 4, 3), 31 | username="honzakral", 32 | email="honza@elastic.co", 33 | location="Prague", 34 | ) 35 | 36 | nick = User( 37 | id=47, 38 | signed_up=datetime(2017, 4, 3), 39 | username="fxdgear", 40 | email="nick.lang@elastic.co", 41 | location="Colorado", 42 | ) 43 | 44 | 45 | @pytest_asyncio.fixture 46 | async def question(async_write_client: AsyncElasticsearch) -> Question: 47 | await setup() 48 | assert await async_write_client.indices.exists_index_template(name="base") 49 | 50 | # create a question object 51 | q = Question( 52 | _id=1, 53 | author=nick, 54 | tags=["elasticsearch", "python"], 55 | title="How do I use elasticsearch from Python?", 56 | body=""" 57 | I want to use elasticsearch, how do I do it from Python? 58 | """, 59 | created=None, 60 | question_answer=None, 61 | comments=[], 62 | ) 63 | await q.save() 64 | return q 65 | 66 | 67 | @pytest.mark.asyncio 68 | async def test_comment( 69 | async_write_client: AsyncElasticsearch, question: Question 70 | ) -> None: 71 | await question.add_comment(nick, "Just use elasticsearch-py") 72 | 73 | q = await Question.get(1) # type: ignore[arg-type] 74 | assert isinstance(q, Question) 75 | assert 1 == len(q.comments) 76 | 77 | c = q.comments[0] 78 | assert isinstance(c, Comment) 79 | assert c.author.username == "fxdgear" 80 | 81 | 82 | @pytest.mark.asyncio 83 | async def test_question_answer( 84 | async_write_client: AsyncElasticsearch, question: Question 85 | ) -> None: 86 | a = await question.add_answer(honza, "Just use `elasticsearch-py`!") 87 | 88 | assert isinstance(a, Answer) 89 | 90 | # refresh the index so we can search right away 91 | await Question._index.refresh() 92 | 93 | # we can now fetch answers from elasticsearch 94 | answers = await question.get_answers() 95 | assert 1 == len(answers) 96 | assert isinstance(answers[0], Answer) 97 | 98 | search = Question.search().query( 99 | "has_child", 100 | type="answer", 101 | inner_hits={}, 102 | query=Q("term", author__username__keyword="honzakral"), 103 | ) 104 | response = await search.execute() 105 | 106 | assert 1 == len(response.hits) 107 | 108 | q = response.hits[0] 109 | assert isinstance(q, Question) 110 | assert 1 == len(q.meta.inner_hits.answer.hits) 111 | assert q.meta.inner_hits.answer.hits is await q.get_answers() 112 | 113 | a = q.meta.inner_hits.answer.hits[0] 114 | assert isinstance(a, Answer) 115 | assert isinstance(await a.get_question(), Question) 116 | assert (await a.get_question()).meta.id == "1" 117 | -------------------------------------------------------------------------------- /tests/test_integration/test_examples/_async/test_percolate.py: -------------------------------------------------------------------------------- 1 | # Licensed to Elasticsearch B.V. under one or more contributor 2 | # license agreements. See the NOTICE file distributed with 3 | # this work for additional information regarding copyright 4 | # ownership. Elasticsearch B.V. licenses this file to you under 5 | # the Apache License, Version 2.0 (the "License"); you may 6 | # not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | import pytest 19 | from elasticsearch import AsyncElasticsearch 20 | 21 | from ..async_examples.percolate import BlogPost, setup 22 | 23 | 24 | @pytest.mark.asyncio 25 | async def test_post_gets_tagged_automatically( 26 | async_write_client: AsyncElasticsearch, 27 | ) -> None: 28 | await setup() 29 | 30 | bp = BlogPost(_id=47, content="nothing about snakes here!") 31 | bp_py = BlogPost(_id=42, content="something about Python here!") 32 | 33 | await bp.save() 34 | await bp_py.save() 35 | 36 | assert [] == bp.tags 37 | assert {"programming", "development", "python"} == set(bp_py.tags) 38 | -------------------------------------------------------------------------------- /tests/test_integration/test_examples/_async/test_vectors.py: -------------------------------------------------------------------------------- 1 | # Licensed to Elasticsearch B.V. under one or more contributor 2 | # license agreements. See the NOTICE file distributed with 3 | # this work for additional information regarding copyright 4 | # ownership. Elasticsearch B.V. licenses this file to you under 5 | # the Apache License, Version 2.0 (the "License"); you may 6 | # not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | from hashlib import md5 19 | from typing import Any, List, Tuple 20 | from unittest import SkipTest 21 | 22 | import pytest 23 | from elasticsearch import AsyncElasticsearch 24 | 25 | from tests.async_sleep import sleep 26 | 27 | from ..async_examples import vectors 28 | 29 | 30 | @pytest.mark.asyncio 31 | async def test_vector_search( 32 | async_write_client: AsyncElasticsearch, es_version: Tuple[int, ...], mocker: Any 33 | ) -> None: 34 | # this test only runs on Elasticsearch >= 8.11 because the example uses 35 | # a dense vector without specifying an explicit size 36 | if es_version < (8, 11): 37 | raise SkipTest("This test requires Elasticsearch 8.11 or newer") 38 | 39 | class MockModel: 40 | def __init__(self, model: Any): 41 | pass 42 | 43 | def encode(self, text: str) -> List[float]: 44 | vector = [int(ch) for ch in md5(text.encode()).digest()] 45 | total = sum(vector) 46 | return [float(v) / total for v in vector] 47 | 48 | mocker.patch.object(vectors, "SentenceTransformer", new=MockModel) 49 | 50 | await vectors.create() 51 | for i in range(10): 52 | results = await (await vectors.search("Welcome to our team!")).execute() 53 | if len(results.hits) > 0: 54 | break 55 | await sleep(0.1) 56 | assert results[0].name == "New Employee Onboarding Guide" 57 | -------------------------------------------------------------------------------- /tests/test_integration/test_examples/_sync/__init__.py: -------------------------------------------------------------------------------- 1 | # Licensed to Elasticsearch B.V. under one or more contributor 2 | # license agreements. See the NOTICE file distributed with 3 | # this work for additional information regarding copyright 4 | # ownership. Elasticsearch B.V. licenses this file to you under 5 | # the Apache License, Version 2.0 (the "License"); you may 6 | # not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | -------------------------------------------------------------------------------- /tests/test_integration/test_examples/_sync/test_alias_migration.py: -------------------------------------------------------------------------------- 1 | # Licensed to Elasticsearch B.V. under one or more contributor 2 | # license agreements. See the NOTICE file distributed with 3 | # this work for additional information regarding copyright 4 | # ownership. Elasticsearch B.V. licenses this file to you under 5 | # the Apache License, Version 2.0 (the "License"); you may 6 | # not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | import pytest 19 | from elasticsearch import Elasticsearch 20 | 21 | from ..examples import alias_migration 22 | from ..examples.alias_migration import ALIAS, PATTERN, BlogPost, migrate 23 | 24 | 25 | @pytest.mark.sync 26 | def test_alias_migration(write_client: Elasticsearch) -> None: 27 | # create the index 28 | alias_migration.setup() 29 | 30 | # verify that template, index, and alias has been set up 31 | assert write_client.indices.exists_index_template(name=ALIAS) 32 | assert write_client.indices.exists(index=PATTERN) 33 | assert write_client.indices.exists_alias(name=ALIAS) 34 | 35 | indices = write_client.indices.get(index=PATTERN) 36 | assert len(indices) == 1 37 | index_name, _ = indices.popitem() 38 | 39 | # which means we can now save a document 40 | with open(__file__) as f: 41 | bp = BlogPost( 42 | _id=0, 43 | title="Hello World!", 44 | tags=["testing", "dummy"], 45 | content=f.read(), 46 | published=None, 47 | ) 48 | bp.save(refresh=True) 49 | 50 | assert BlogPost.search().count() == 1 51 | 52 | # _matches work which means we get BlogPost instance 53 | bp = (BlogPost.search().execute())[0] 54 | assert isinstance(bp, BlogPost) 55 | assert not bp.is_published() 56 | assert "0" == bp.meta.id 57 | 58 | # create new index 59 | migrate() 60 | 61 | indices = write_client.indices.get(index=PATTERN) 62 | assert 2 == len(indices) 63 | alias = write_client.indices.get(index=ALIAS) 64 | assert 1 == len(alias) 65 | assert index_name not in alias 66 | 67 | # data has been moved properly 68 | assert BlogPost.search().count() == 1 69 | 70 | # _matches work which means we get BlogPost instance 71 | bp = (BlogPost.search().execute())[0] 72 | assert isinstance(bp, BlogPost) 73 | assert "0" == bp.meta.id 74 | -------------------------------------------------------------------------------- /tests/test_integration/test_examples/_sync/test_completion.py: -------------------------------------------------------------------------------- 1 | # Licensed to Elasticsearch B.V. under one or more contributor 2 | # license agreements. See the NOTICE file distributed with 3 | # this work for additional information regarding copyright 4 | # ownership. Elasticsearch B.V. licenses this file to you under 5 | # the Apache License, Version 2.0 (the "License"); you may 6 | # not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | import pytest 19 | from elasticsearch import Elasticsearch 20 | 21 | from ..examples.completion import Person 22 | 23 | 24 | @pytest.mark.sync 25 | def test_person_suggests_on_all_variants_of_name( 26 | write_client: Elasticsearch, 27 | ) -> None: 28 | Person.init(using=write_client) 29 | 30 | Person(_id=None, name="Honza Král", popularity=42).save(refresh=True) 31 | 32 | s = Person.search().suggest("t", "kra", completion={"field": "suggest"}) 33 | response = s.execute() 34 | 35 | opts = response.suggest["t"][0].options 36 | 37 | assert 1 == len(opts) 38 | assert opts[0]._score == 42 39 | assert opts[0]._source.name == "Honza Král" 40 | -------------------------------------------------------------------------------- /tests/test_integration/test_examples/_sync/test_composite_aggs.py: -------------------------------------------------------------------------------- 1 | # Licensed to Elasticsearch B.V. under one or more contributor 2 | # license agreements. See the NOTICE file distributed with 3 | # this work for additional information regarding copyright 4 | # ownership. Elasticsearch B.V. licenses this file to you under 5 | # the Apache License, Version 2.0 (the "License"); you may 6 | # not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | import pytest 19 | from elasticsearch import Elasticsearch 20 | 21 | from elasticsearch_dsl import A, Search 22 | 23 | from ..examples.composite_agg import scan_aggs 24 | 25 | 26 | @pytest.mark.sync 27 | def test_scan_aggs_exhausts_all_files( 28 | data_client: Elasticsearch, 29 | ) -> None: 30 | s = Search(index="flat-git") 31 | key_aggs = [{"files": A("terms", field="files")}] 32 | file_list = [f for f in scan_aggs(s, key_aggs)] 33 | 34 | assert len(file_list) == 26 35 | 36 | 37 | @pytest.mark.sync 38 | def test_scan_aggs_with_multiple_aggs( 39 | data_client: Elasticsearch, 40 | ) -> None: 41 | s = Search(index="flat-git") 42 | key_aggs = [ 43 | {"files": A("terms", field="files")}, 44 | { 45 | "months": A( 46 | "date_histogram", field="committed_date", calendar_interval="month" 47 | ) 48 | }, 49 | ] 50 | file_list = [ 51 | f 52 | for f in scan_aggs( 53 | s, key_aggs, {"first_seen": A("min", field="committed_date")} 54 | ) 55 | ] 56 | 57 | assert len(file_list) == 47 58 | -------------------------------------------------------------------------------- /tests/test_integration/test_examples/_sync/test_parent_child.py: -------------------------------------------------------------------------------- 1 | # Licensed to Elasticsearch B.V. under one or more contributor 2 | # license agreements. See the NOTICE file distributed with 3 | # this work for additional information regarding copyright 4 | # ownership. Elasticsearch B.V. licenses this file to you under 5 | # the Apache License, Version 2.0 (the "License"); you may 6 | # not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | from datetime import datetime 19 | 20 | import pytest 21 | from elasticsearch import Elasticsearch 22 | 23 | from elasticsearch_dsl import Q 24 | 25 | from ..examples.parent_child import Answer, Comment, Question, User, setup 26 | 27 | honza = User( 28 | id=42, 29 | signed_up=datetime(2013, 4, 3), 30 | username="honzakral", 31 | email="honza@elastic.co", 32 | location="Prague", 33 | ) 34 | 35 | nick = User( 36 | id=47, 37 | signed_up=datetime(2017, 4, 3), 38 | username="fxdgear", 39 | email="nick.lang@elastic.co", 40 | location="Colorado", 41 | ) 42 | 43 | 44 | @pytest.fixture 45 | def question(write_client: Elasticsearch) -> Question: 46 | setup() 47 | assert write_client.indices.exists_index_template(name="base") 48 | 49 | # create a question object 50 | q = Question( 51 | _id=1, 52 | author=nick, 53 | tags=["elasticsearch", "python"], 54 | title="How do I use elasticsearch from Python?", 55 | body=""" 56 | I want to use elasticsearch, how do I do it from Python? 57 | """, 58 | created=None, 59 | question_answer=None, 60 | comments=[], 61 | ) 62 | q.save() 63 | return q 64 | 65 | 66 | @pytest.mark.sync 67 | def test_comment(write_client: Elasticsearch, question: Question) -> None: 68 | question.add_comment(nick, "Just use elasticsearch-py") 69 | 70 | q = Question.get(1) # type: ignore[arg-type] 71 | assert isinstance(q, Question) 72 | assert 1 == len(q.comments) 73 | 74 | c = q.comments[0] 75 | assert isinstance(c, Comment) 76 | assert c.author.username == "fxdgear" 77 | 78 | 79 | @pytest.mark.sync 80 | def test_question_answer(write_client: Elasticsearch, question: Question) -> None: 81 | a = question.add_answer(honza, "Just use `elasticsearch-py`!") 82 | 83 | assert isinstance(a, Answer) 84 | 85 | # refresh the index so we can search right away 86 | Question._index.refresh() 87 | 88 | # we can now fetch answers from elasticsearch 89 | answers = question.get_answers() 90 | assert 1 == len(answers) 91 | assert isinstance(answers[0], Answer) 92 | 93 | search = Question.search().query( 94 | "has_child", 95 | type="answer", 96 | inner_hits={}, 97 | query=Q("term", author__username__keyword="honzakral"), 98 | ) 99 | response = search.execute() 100 | 101 | assert 1 == len(response.hits) 102 | 103 | q = response.hits[0] 104 | assert isinstance(q, Question) 105 | assert 1 == len(q.meta.inner_hits.answer.hits) 106 | assert q.meta.inner_hits.answer.hits is q.get_answers() 107 | 108 | a = q.meta.inner_hits.answer.hits[0] 109 | assert isinstance(a, Answer) 110 | assert isinstance(a.get_question(), Question) 111 | assert (a.get_question()).meta.id == "1" 112 | -------------------------------------------------------------------------------- /tests/test_integration/test_examples/_sync/test_percolate.py: -------------------------------------------------------------------------------- 1 | # Licensed to Elasticsearch B.V. under one or more contributor 2 | # license agreements. See the NOTICE file distributed with 3 | # this work for additional information regarding copyright 4 | # ownership. Elasticsearch B.V. licenses this file to you under 5 | # the Apache License, Version 2.0 (the "License"); you may 6 | # not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | import pytest 19 | from elasticsearch import Elasticsearch 20 | 21 | from ..examples.percolate import BlogPost, setup 22 | 23 | 24 | @pytest.mark.sync 25 | def test_post_gets_tagged_automatically( 26 | write_client: Elasticsearch, 27 | ) -> None: 28 | setup() 29 | 30 | bp = BlogPost(_id=47, content="nothing about snakes here!") 31 | bp_py = BlogPost(_id=42, content="something about Python here!") 32 | 33 | bp.save() 34 | bp_py.save() 35 | 36 | assert [] == bp.tags 37 | assert {"programming", "development", "python"} == set(bp_py.tags) 38 | -------------------------------------------------------------------------------- /tests/test_integration/test_examples/_sync/test_vectors.py: -------------------------------------------------------------------------------- 1 | # Licensed to Elasticsearch B.V. under one or more contributor 2 | # license agreements. See the NOTICE file distributed with 3 | # this work for additional information regarding copyright 4 | # ownership. Elasticsearch B.V. licenses this file to you under 5 | # the Apache License, Version 2.0 (the "License"); you may 6 | # not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | from hashlib import md5 19 | from typing import Any, List, Tuple 20 | from unittest import SkipTest 21 | 22 | import pytest 23 | from elasticsearch import Elasticsearch 24 | 25 | from tests.sleep import sleep 26 | 27 | from ..examples import vectors 28 | 29 | 30 | @pytest.mark.sync 31 | def test_vector_search( 32 | write_client: Elasticsearch, es_version: Tuple[int, ...], mocker: Any 33 | ) -> None: 34 | # this test only runs on Elasticsearch >= 8.11 because the example uses 35 | # a dense vector without specifying an explicit size 36 | if es_version < (8, 11): 37 | raise SkipTest("This test requires Elasticsearch 8.11 or newer") 38 | 39 | class MockModel: 40 | def __init__(self, model: Any): 41 | pass 42 | 43 | def encode(self, text: str) -> List[float]: 44 | vector = [int(ch) for ch in md5(text.encode()).digest()] 45 | total = sum(vector) 46 | return [float(v) / total for v in vector] 47 | 48 | mocker.patch.object(vectors, "SentenceTransformer", new=MockModel) 49 | 50 | vectors.create() 51 | for i in range(10): 52 | results = (vectors.search("Welcome to our team!")).execute() 53 | if len(results.hits) > 0: 54 | break 55 | sleep(0.1) 56 | assert results[0].name == "New Employee Onboarding Guide" 57 | -------------------------------------------------------------------------------- /tests/test_integration/test_examples/async_examples: -------------------------------------------------------------------------------- 1 | ../../../examples/async -------------------------------------------------------------------------------- /tests/test_integration/test_examples/examples: -------------------------------------------------------------------------------- 1 | ../../../examples -------------------------------------------------------------------------------- /tests/test_package.py: -------------------------------------------------------------------------------- 1 | # Licensed to Elasticsearch B.V. under one or more contributor 2 | # license agreements. See the NOTICE file distributed with 3 | # this work for additional information regarding copyright 4 | # ownership. Elasticsearch B.V. licenses this file to you under 5 | # the Apache License, Version 2.0 (the "License"); you may 6 | # not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | import elasticsearch_dsl 19 | 20 | 21 | def test__all__is_sorted() -> None: 22 | assert elasticsearch_dsl.__all__ == sorted(elasticsearch_dsl.__all__) 23 | -------------------------------------------------------------------------------- /tests/test_utils.py: -------------------------------------------------------------------------------- 1 | # Licensed to Elasticsearch B.V. under one or more contributor 2 | # license agreements. See the NOTICE file distributed with 3 | # this work for additional information regarding copyright 4 | # ownership. Elasticsearch B.V. licenses this file to you under 5 | # the Apache License, Version 2.0 (the "License"); you may 6 | # not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | import pickle 19 | from typing import Any, Dict, Tuple 20 | 21 | from pytest import raises 22 | 23 | from elasticsearch_dsl import Q, serializer, utils 24 | 25 | 26 | def test_attrdict_pickle() -> None: 27 | ad: utils.AttrDict[str] = utils.AttrDict({}) 28 | 29 | pickled_ad = pickle.dumps(ad) 30 | assert ad == pickle.loads(pickled_ad) 31 | 32 | 33 | def test_attrlist_pickle() -> None: 34 | al = utils.AttrList[Any]([]) 35 | 36 | pickled_al = pickle.dumps(al) 37 | assert al == pickle.loads(pickled_al) 38 | 39 | 40 | def test_attrlist_slice() -> None: 41 | class MyAttrDict(utils.AttrDict[str]): 42 | pass 43 | 44 | l = utils.AttrList[Any]([{}, {}], obj_wrapper=MyAttrDict) 45 | assert isinstance(l[:][0], MyAttrDict) 46 | 47 | 48 | def test_attrlist_with_type_argument() -> None: 49 | a = utils.AttrList[str](["a", "b"]) 50 | assert list(a) == ["a", "b"] 51 | 52 | 53 | def test_attrdict_keys_items() -> None: 54 | a = utils.AttrDict({"a": {"b": 42, "c": 47}, "d": "e"}) 55 | assert list(a.keys()) == ["a", "d"] 56 | assert list(a.items()) == [("a", {"b": 42, "c": 47}), ("d", "e")] 57 | 58 | 59 | def test_attrdict_with_type_argument() -> None: 60 | a = utils.AttrDict[str]({"a": "b"}) 61 | assert list(a.keys()) == ["a"] 62 | assert list(a.items()) == [("a", "b")] 63 | 64 | 65 | def test_merge() -> None: 66 | a: utils.AttrDict[Any] = utils.AttrDict({"a": {"b": 42, "c": 47}}) 67 | b = {"a": {"b": 123, "d": -12}, "e": [1, 2, 3]} 68 | 69 | utils.merge(a, b) 70 | 71 | assert a == {"a": {"b": 123, "c": 47, "d": -12}, "e": [1, 2, 3]} 72 | 73 | 74 | def test_merge_conflict() -> None: 75 | data: Tuple[Dict[str, Any], ...] = ( 76 | {"a": 42}, 77 | {"a": {"b": 47}}, 78 | ) 79 | for d in data: 80 | utils.merge({"a": {"b": 42}}, d) 81 | with raises(ValueError): 82 | utils.merge({"a": {"b": 42}}, d, True) 83 | 84 | 85 | def test_attrdict_bool() -> None: 86 | d: utils.AttrDict[str] = utils.AttrDict({}) 87 | 88 | assert not d 89 | d.title = "Title" 90 | assert d 91 | 92 | 93 | def test_attrlist_items_get_wrapped_during_iteration() -> None: 94 | al = utils.AttrList([1, object(), [1], {}]) 95 | 96 | l = list(iter(al)) 97 | 98 | assert isinstance(l[2], utils.AttrList) 99 | assert isinstance(l[3], utils.AttrDict) 100 | 101 | 102 | def test_serializer_deals_with_Attr_versions() -> None: 103 | d = utils.AttrDict({"key": utils.AttrList([1, 2, 3])}) 104 | 105 | assert serializer.serializer.dumps(d) == serializer.serializer.dumps( 106 | {"key": [1, 2, 3]} 107 | ) 108 | 109 | 110 | def test_serializer_deals_with_objects_with_to_dict() -> None: 111 | class MyClass: 112 | def to_dict(self) -> int: 113 | return 42 114 | 115 | assert serializer.serializer.dumps(MyClass()) == b"42" 116 | 117 | 118 | def test_recursive_to_dict() -> None: 119 | assert utils.recursive_to_dict({"k": [1, (1.0, {"v": Q("match", key="val")})]}) == { 120 | "k": [1, (1.0, {"v": {"match": {"key": "val"}}})] 121 | } 122 | 123 | 124 | def test_attrlist_to_list() -> None: 125 | l = utils.AttrList[Any]([{}, {}]).to_list() 126 | assert isinstance(l, list) 127 | assert l == [{}, {}] 128 | 129 | 130 | def test_attrdict_with_reserved_keyword() -> None: 131 | d = utils.AttrDict({"from": 10, "size": 20}) 132 | assert d.from_ == 10 133 | assert d.size == 20 134 | d = utils.AttrDict({}) 135 | d.from_ = 10 136 | assert {"from": 10} == d.to_dict() 137 | -------------------------------------------------------------------------------- /tests/test_validation.py: -------------------------------------------------------------------------------- 1 | # Licensed to Elasticsearch B.V. under one or more contributor 2 | # license agreements. See the NOTICE file distributed with 3 | # this work for additional information regarding copyright 4 | # ownership. Elasticsearch B.V. licenses this file to you under 5 | # the Apache License, Version 2.0 (the "License"); you may 6 | # not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | from datetime import datetime 19 | from typing import Any 20 | 21 | from pytest import raises 22 | 23 | from elasticsearch_dsl import ( 24 | Date, 25 | Document, 26 | InnerDoc, 27 | Integer, 28 | Nested, 29 | Object, 30 | Text, 31 | mapped_field, 32 | ) 33 | from elasticsearch_dsl.exceptions import ValidationException 34 | 35 | 36 | class Author(InnerDoc): 37 | name: str 38 | email: str 39 | 40 | def clean(self) -> None: 41 | if not self.name: 42 | raise ValidationException("name is missing") 43 | if not self.email: 44 | raise ValidationException("email is missing") 45 | elif self.name.lower() not in self.email: 46 | raise ValidationException("Invalid email!") 47 | 48 | 49 | class BlogPost(Document): 50 | authors = Nested(Author, required=True) 51 | created = Date() 52 | inner = Object() 53 | 54 | 55 | class BlogPostWithStatus(Document): 56 | published: bool = mapped_field(init=False) 57 | 58 | 59 | class AutoNowDate(Date): 60 | def clean(self, data: Any) -> Any: 61 | if data is None: 62 | data = datetime.now() 63 | return super().clean(data) 64 | 65 | 66 | class Log(Document): 67 | timestamp = AutoNowDate(required=True) 68 | data = Text() 69 | 70 | 71 | def test_required_int_can_be_0() -> None: 72 | class DT(Document): 73 | i = Integer(required=True) 74 | 75 | dt = DT(i=0) 76 | dt.full_clean() 77 | 78 | 79 | def test_required_field_cannot_be_empty_list() -> None: 80 | class DT(Document): 81 | i = Integer(required=True) 82 | 83 | dt = DT(i=[]) 84 | with raises(ValidationException): 85 | dt.full_clean() 86 | 87 | 88 | def test_validation_works_for_lists_of_values() -> None: 89 | class DT(Document): 90 | i = Date(required=True) 91 | 92 | dt = DT(i=[datetime.now(), "not date"]) 93 | with raises(ValidationException): 94 | dt.full_clean() 95 | 96 | dt = DT(i=[datetime.now(), datetime.now()]) 97 | dt.full_clean() 98 | 99 | 100 | def test_field_with_custom_clean() -> None: 101 | l = Log() 102 | l.full_clean() 103 | 104 | assert isinstance(l.timestamp, datetime) 105 | 106 | 107 | def test_empty_object() -> None: 108 | d = BlogPost(authors=[{"name": "Honza", "email": "honza@elastic.co"}]) 109 | d.inner = {} # type: ignore[assignment] 110 | 111 | d.full_clean() 112 | 113 | 114 | def test_missing_required_field_raises_validation_exception() -> None: 115 | d = BlogPost() 116 | with raises(ValidationException): 117 | d.full_clean() 118 | 119 | d = BlogPost() 120 | d.authors.append({"name": "Honza"}) 121 | with raises(ValidationException): 122 | d.full_clean() 123 | 124 | d = BlogPost() 125 | d.authors.append({"name": "Honza", "email": "honza@elastic.co"}) 126 | d.full_clean() 127 | 128 | 129 | def test_boolean_doesnt_treat_false_as_empty() -> None: 130 | d = BlogPostWithStatus() 131 | with raises(ValidationException): 132 | d.full_clean() 133 | d.published = False 134 | d.full_clean() 135 | d.published = True 136 | d.full_clean() 137 | 138 | 139 | def test_custom_validation_on_nested_gets_run() -> None: 140 | d = BlogPost(authors=[Author(name="Honza", email="king@example.com")], created=None) 141 | 142 | assert isinstance(d.authors[0], Author) # type: ignore[index] 143 | 144 | with raises(ValidationException): 145 | d.full_clean() 146 | 147 | 148 | def test_accessing_known_fields_returns_empty_value() -> None: 149 | d = BlogPost() 150 | 151 | assert [] == d.authors 152 | 153 | d.authors.append({}) 154 | assert None is d.authors[0].name # type: ignore[index] 155 | assert None is d.authors[0].email 156 | 157 | 158 | def test_empty_values_are_not_serialized() -> None: 159 | d = BlogPost(authors=[{"name": "Honza", "email": "honza@elastic.co"}], created=None) 160 | 161 | d.full_clean() 162 | assert d.to_dict() == {"authors": [{"name": "Honza", "email": "honza@elastic.co"}]} 163 | -------------------------------------------------------------------------------- /tests/test_wrappers.py: -------------------------------------------------------------------------------- 1 | # Licensed to Elasticsearch B.V. under one or more contributor 2 | # license agreements. See the NOTICE file distributed with 3 | # this work for additional information regarding copyright 4 | # ownership. Elasticsearch B.V. licenses this file to you under 5 | # the Apache License, Version 2.0 (the "License"); you may 6 | # not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | from datetime import datetime, timedelta 19 | from typing import TYPE_CHECKING, Any, Mapping, Optional, Sequence 20 | 21 | if TYPE_CHECKING: 22 | from _operator import _SupportsComparison 23 | 24 | import pytest 25 | 26 | from elasticsearch_dsl import Range 27 | 28 | 29 | @pytest.mark.parametrize( 30 | "kwargs, item", 31 | [ 32 | ({}, 1), 33 | ({}, -1), 34 | ({"gte": -1}, -1), 35 | ({"lte": 4}, 4), 36 | ({"lte": 4, "gte": 2}, 4), 37 | ({"lte": 4, "gte": 2}, 2), 38 | ({"gt": datetime.now() - timedelta(seconds=10)}, datetime.now()), 39 | ], 40 | ) 41 | def test_range_contains( 42 | kwargs: Mapping[str, "_SupportsComparison"], item: "_SupportsComparison" 43 | ) -> None: 44 | assert item in Range(**kwargs) 45 | 46 | 47 | @pytest.mark.parametrize( 48 | "kwargs, item", 49 | [ 50 | ({"gt": -1}, -1), 51 | ({"lt": 4}, 4), 52 | ({"lt": 4}, 42), 53 | ({"lte": 4, "gte": 2}, 1), 54 | ({"lte": datetime.now() - timedelta(seconds=10)}, datetime.now()), 55 | ], 56 | ) 57 | def test_range_not_contains( 58 | kwargs: Mapping[str, "_SupportsComparison"], item: "_SupportsComparison" 59 | ) -> None: 60 | assert item not in Range(**kwargs) 61 | 62 | 63 | @pytest.mark.parametrize( 64 | "args,kwargs", 65 | [ 66 | (({},), {"lt": 42}), 67 | ((), {"not_lt": 42}), 68 | ((object(),), {}), 69 | ((), {"lt": 1, "lte": 1}), 70 | ((), {"gt": 1, "gte": 1}), 71 | ], 72 | ) 73 | def test_range_raises_value_error_on_wrong_params( 74 | args: Sequence[Any], kwargs: Mapping[str, "_SupportsComparison"] 75 | ) -> None: 76 | with pytest.raises(ValueError): 77 | Range(*args, **kwargs) 78 | 79 | 80 | @pytest.mark.parametrize( 81 | "range,lower,inclusive", 82 | [ 83 | (Range(gt=1), 1, False), 84 | (Range(gte=1), 1, True), 85 | (Range(), None, False), 86 | (Range(lt=42), None, False), 87 | ], 88 | ) 89 | def test_range_lower( 90 | range: Range["_SupportsComparison"], 91 | lower: Optional["_SupportsComparison"], 92 | inclusive: bool, 93 | ) -> None: 94 | assert (lower, inclusive) == range.lower 95 | 96 | 97 | @pytest.mark.parametrize( 98 | "range,upper,inclusive", 99 | [ 100 | (Range(lt=1), 1, False), 101 | (Range(lte=1), 1, True), 102 | (Range(), None, False), 103 | (Range(gt=42), None, False), 104 | ], 105 | ) 106 | def test_range_upper( 107 | range: Range["_SupportsComparison"], 108 | upper: Optional["_SupportsComparison"], 109 | inclusive: bool, 110 | ) -> None: 111 | assert (upper, inclusive) == range.upper 112 | -------------------------------------------------------------------------------- /utils/build-dists.py: -------------------------------------------------------------------------------- 1 | # Licensed to Elasticsearch B.V. under one or more contributor 2 | # license agreements. See the NOTICE file distributed with 3 | # this work for additional information regarding copyright 4 | # ownership. Elasticsearch B.V. licenses this file to you under 5 | # the Apache License, Version 2.0 (the "License"); you may 6 | # not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | """A command line tool for building and verifying releases 19 | Can be used for building both 'elasticsearch' and 'elasticsearchX' dists. 20 | Only requires 'name' in 'setup.py' and the directory to be changed. 21 | """ 22 | 23 | import contextlib 24 | import os 25 | import re 26 | import shlex 27 | import shutil 28 | import tempfile 29 | 30 | base_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) 31 | tmp_dir = None 32 | 33 | 34 | @contextlib.contextmanager 35 | def set_tmp_dir(): 36 | global tmp_dir 37 | tmp_dir = tempfile.mkdtemp() 38 | yield tmp_dir 39 | shutil.rmtree(tmp_dir) 40 | tmp_dir = None 41 | 42 | 43 | def run(*argv, expect_exit_code=0): 44 | if tmp_dir is None: 45 | os.chdir(base_dir) 46 | else: 47 | os.chdir(tmp_dir) 48 | 49 | cmd = shlex.join(argv) 50 | print("$ " + cmd) 51 | exit_code = os.system(cmd) 52 | if exit_code != expect_exit_code: 53 | print( 54 | "Command exited incorrectly: should have been %d was %d" 55 | % (expect_exit_code, exit_code) 56 | ) 57 | exit(exit_code or 1) 58 | 59 | 60 | def test_dist(dist): 61 | with set_tmp_dir() as tmp_dir: 62 | dist_name = ( 63 | re.match(r"^(elasticsearch\d*[_-]dsl)-", os.path.basename(dist)) 64 | .group(1) 65 | .replace("-", "_") 66 | ) 67 | 68 | # Build the venv and install the dist 69 | run("python", "-m", "venv", os.path.join(tmp_dir, "venv")) 70 | venv_python = os.path.join(tmp_dir, "venv/bin/python") 71 | run(venv_python, "-m", "pip", "install", "-U", "pip") 72 | run(venv_python, "-m", "pip", "install", dist) 73 | 74 | # Test the sync namespaces 75 | run(venv_python, "-c", f"from {dist_name} import Q") 76 | 77 | # Ensure that the namespaces are correct for the dist 78 | for suffix in ("", "1", "2", "5", "6", "7", "8", "9", "10"): 79 | distx_name = f"elasticsearch{suffix}_dsl" 80 | run( 81 | venv_python, 82 | "-c", 83 | f"import {distx_name}", 84 | expect_exit_code=256 if distx_name != dist_name else 0, 85 | ) 86 | # Tests the dependencies of the dist 87 | run( 88 | venv_python, 89 | "-c", 90 | f"import elasticsearch{suffix}", 91 | expect_exit_code=256 if distx_name != dist_name else 0, 92 | ) 93 | 94 | # Uninstall the dist, see that we can't import things anymore 95 | run(venv_python, "-m", "pip", "uninstall", "--yes", dist_name) 96 | run( 97 | venv_python, 98 | "-c", 99 | f"from {dist_name} import Q", 100 | expect_exit_code=256, 101 | ) 102 | 103 | 104 | def main(): 105 | run("rm", "-rf", "build/", "dist/", "*.egg-info", ".eggs") 106 | run("python", "setup.py", "sdist", "bdist_wheel") 107 | 108 | for dist in os.listdir(os.path.join(base_dir, "dist")): 109 | test_dist(os.path.join(base_dir, "dist", dist)) 110 | 111 | # After this run 'python -m twine upload dist/*' 112 | print( 113 | "\n\n" 114 | "===============================\n\n" 115 | " * Releases are ready! *\n\n" 116 | "$ python -m twine upload dist/*\n\n" 117 | "===============================" 118 | ) 119 | 120 | 121 | if __name__ == "__main__": 122 | main() 123 | -------------------------------------------------------------------------------- /utils/license-headers.py: -------------------------------------------------------------------------------- 1 | # Licensed to Elasticsearch B.V. under one or more contributor 2 | # license agreements. See the NOTICE file distributed with 3 | # this work for additional information regarding copyright 4 | # ownership. Elasticsearch B.V. licenses this file to you under 5 | # the Apache License, Version 2.0 (the "License"); you may 6 | # not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | """Script which verifies that all source files have a license header. 19 | Has two modes: 'fix' and 'check'. 'fix' fixes problems, 'check' will 20 | error out if 'fix' would have changed the file. 21 | """ 22 | 23 | import os 24 | import sys 25 | from itertools import chain 26 | from typing import Iterator, List 27 | 28 | lines_to_keep = ["# -*- coding: utf-8 -*-\n", "#!/usr/bin/env python\n"] 29 | license_header_lines = [ 30 | "# Licensed to Elasticsearch B.V. under one or more contributor\n", 31 | "# license agreements. See the NOTICE file distributed with\n", 32 | "# this work for additional information regarding copyright\n", 33 | "# ownership. Elasticsearch B.V. licenses this file to you under\n", 34 | '# the Apache License, Version 2.0 (the "License"); you may\n', 35 | "# not use this file except in compliance with the License.\n", 36 | "# You may obtain a copy of the License at\n", 37 | "#\n", 38 | "# http://www.apache.org/licenses/LICENSE-2.0\n", 39 | "#\n", 40 | "# Unless required by applicable law or agreed to in writing,\n", 41 | "# software distributed under the License is distributed on an\n", 42 | '# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n', 43 | "# KIND, either express or implied. See the License for the\n", 44 | "# specific language governing permissions and limitations\n", 45 | "# under the License.\n", 46 | "\n", 47 | ] 48 | 49 | 50 | def find_files_to_fix(sources: List[str]) -> Iterator[str]: 51 | """Iterates over all files and dirs in 'sources' and returns 52 | only the filepaths that need fixing. 53 | """ 54 | for source in sources: 55 | if os.path.isfile(source) and does_file_need_fix(source): 56 | yield source 57 | elif os.path.isdir(source): 58 | for root, _, filenames in os.walk(source): 59 | for filename in filenames: 60 | filepath = os.path.join(root, filename) 61 | if does_file_need_fix(filepath): 62 | yield filepath 63 | 64 | 65 | def does_file_need_fix(filepath: str) -> bool: 66 | if not filepath.endswith(".py"): 67 | return False 68 | with open(filepath) as f: 69 | first_license_line = None 70 | for line in f: 71 | if line == license_header_lines[0]: 72 | first_license_line = line 73 | break 74 | elif line not in lines_to_keep: 75 | return True 76 | for header_line, line in zip( 77 | license_header_lines, chain((first_license_line,), f) 78 | ): 79 | if line != header_line: 80 | return True 81 | return False 82 | 83 | 84 | def add_header_to_file(filepath: str) -> None: 85 | with open(filepath) as f: 86 | lines = list(f) 87 | i = 0 88 | for i, line in enumerate(lines): 89 | if line not in lines_to_keep: 90 | break 91 | lines = lines[:i] + license_header_lines + lines[i:] 92 | with open(filepath, mode="w") as f: 93 | f.truncate() 94 | f.write("".join(lines)) 95 | print(f"Fixed {os.path.relpath(filepath, os.getcwd())}") 96 | 97 | 98 | def main(): 99 | mode = sys.argv[1] 100 | assert mode in ("fix", "check") 101 | sources = [os.path.abspath(x) for x in sys.argv[2:]] 102 | files_to_fix = find_files_to_fix(sources) 103 | 104 | if mode == "fix": 105 | for filepath in files_to_fix: 106 | add_header_to_file(filepath) 107 | else: 108 | no_license_headers = list(files_to_fix) 109 | if no_license_headers: 110 | print("No license header found in:") 111 | cwd = os.getcwd() 112 | [ 113 | print(f" - {os.path.relpath(filepath, cwd)}") 114 | for filepath in no_license_headers 115 | ] 116 | sys.exit(1) 117 | else: 118 | print("All files had license header") 119 | 120 | 121 | if __name__ == "__main__": 122 | main() 123 | -------------------------------------------------------------------------------- /utils/run-unasync.py: -------------------------------------------------------------------------------- 1 | # Licensed to Elasticsearch B.V. under one or more contributor 2 | # license agreements. See the NOTICE file distributed with 3 | # this work for additional information regarding copyright 4 | # ownership. Elasticsearch B.V. licenses this file to you under 5 | # the Apache License, Version 2.0 (the "License"); you may 6 | # not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | import os 19 | import subprocess 20 | import sys 21 | from glob import glob 22 | from pathlib import Path 23 | 24 | import unasync 25 | 26 | 27 | def main(check=False): 28 | # the list of directories that need to be processed with unasync 29 | # each entry has two paths: 30 | # - the source path with the async sources 31 | # - the destination path where the sync sources should be written 32 | source_dirs = [ 33 | ( 34 | "elasticsearch_dsl/_async/", 35 | "elasticsearch_dsl/_sync/", 36 | ), 37 | ("tests/_async/", "tests/_sync/"), 38 | ( 39 | "tests/test_integration/_async/", 40 | "tests/test_integration/_sync/", 41 | ), 42 | ( 43 | "tests/test_integration/test_examples/_async/", 44 | "tests/test_integration/test_examples/_sync/", 45 | ), 46 | ("examples/async/", "examples/"), 47 | ] 48 | 49 | # Unasync all the generated async code 50 | additional_replacements = { 51 | "_async": "_sync", 52 | "AsyncElasticsearch": "Elasticsearch", 53 | "AsyncSearch": "Search", 54 | "AsyncMultiSearch": "MultiSearch", 55 | "AsyncEmptySearch": "EmptySearch", 56 | "AsyncDocument": "Document", 57 | "AsyncIndexMeta": "IndexMeta", 58 | "AsyncIndexTemplate": "IndexTemplate", 59 | "AsyncIndex": "Index", 60 | "AsyncComposableIndexTemplate": "ComposableIndexTemplate", 61 | "AsyncUpdateByQuery": "UpdateByQuery", 62 | "AsyncMapping": "Mapping", 63 | "AsyncFacetedSearch": "FacetedSearch", 64 | "AsyncUsingType": "UsingType", 65 | "async_connections": "connections", 66 | "async_scan": "scan", 67 | "async_simulate": "simulate", 68 | "async_bulk": "bulk", 69 | "async_mock_client": "mock_client", 70 | "async_client": "client", 71 | "async_data_client": "data_client", 72 | "async_write_client": "write_client", 73 | "async_pull_request": "pull_request", 74 | "async_examples": "examples", 75 | "async_sleep": "sleep", 76 | "assert_awaited_once_with": "assert_called_once_with", 77 | "pytest_asyncio": "pytest", 78 | "asynccontextmanager": "contextmanager", 79 | } 80 | rules = [ 81 | unasync.Rule( 82 | fromdir=dir[0], 83 | todir=f"{dir[0]}_sync_check/" if check else dir[1], 84 | additional_replacements=additional_replacements, 85 | ) 86 | for dir in source_dirs 87 | ] 88 | 89 | filepaths = [] 90 | for root, _, filenames in os.walk(Path(__file__).absolute().parent.parent): 91 | if "/site-packages" in root or "/." in root or "__pycache__" in root: 92 | continue 93 | for filename in filenames: 94 | if filename.rpartition(".")[-1] in ( 95 | "py", 96 | "pyi", 97 | ) and not filename.startswith("utils.py"): 98 | filepaths.append(os.path.join(root, filename)) 99 | 100 | unasync.unasync_files(filepaths, rules) 101 | for dir in source_dirs: 102 | output_dir = f"{dir[0]}_sync_check/" if check else dir[1] 103 | subprocess.check_call(["black", "--target-version=py38", output_dir]) 104 | subprocess.check_call(["isort", output_dir]) 105 | for file in glob("*.py", root_dir=dir[0]): 106 | # remove asyncio from sync files 107 | subprocess.check_call( 108 | ["sed", "-i.bak", "/^import asyncio$/d", f"{output_dir}{file}"] 109 | ) 110 | subprocess.check_call( 111 | [ 112 | "sed", 113 | "-i.bak", 114 | "s/asyncio\\.run(main())/main()/", 115 | f"{output_dir}{file}", 116 | ] 117 | ) 118 | subprocess.check_call( 119 | [ 120 | "sed", 121 | "-i.bak", 122 | "s/elasticsearch-dsl\\[async\\]/elasticsearch-dsl/", 123 | f"{output_dir}{file}", 124 | ] 125 | ) 126 | subprocess.check_call( 127 | [ 128 | "sed", 129 | "-i.bak", 130 | "s/pytest.mark.asyncio/pytest.mark.sync/", 131 | f"{output_dir}{file}", 132 | ] 133 | ) 134 | subprocess.check_call(["rm", f"{output_dir}{file}.bak"]) 135 | 136 | if check: 137 | # make sure there are no differences between _sync and _sync_check 138 | subprocess.check_call( 139 | [ 140 | "diff", 141 | f"{dir[1]}{file}", 142 | f"{output_dir}{file}", 143 | ] 144 | ) 145 | 146 | if check: 147 | subprocess.check_call(["rm", "-rf", output_dir]) 148 | 149 | 150 | if __name__ == "__main__": 151 | main(check="--check" in sys.argv) 152 | -------------------------------------------------------------------------------- /utils/templates/types.py.tpl: -------------------------------------------------------------------------------- 1 | # Licensed to Elasticsearch B.V. under one or more contributor 2 | # license agreements. See the NOTICE file distributed with 3 | # this work for additional information regarding copyright 4 | # ownership. Elasticsearch B.V. licenses this file to you under 5 | # the Apache License, Version 2.0 (the "License"); you may 6 | # not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | from typing import Any, Dict, Literal, Mapping, Sequence, Union 19 | 20 | from elastic_transport.client_utils import DEFAULT, DefaultType 21 | 22 | from elasticsearch_dsl.document_base import InstrumentedField 23 | from elasticsearch_dsl import Query 24 | from elasticsearch_dsl.utils import AttrDict 25 | 26 | PipeSeparatedFlags = str 27 | 28 | 29 | {% for k in classes %} 30 | class {{ k.name }}({{ k.parent if k.parent else "AttrDict[Any]" }}): 31 | {% if k.docstring or k.args %} 32 | """ 33 | {% for line in k.docstring %} 34 | {{ line }} 35 | {% endfor %} 36 | {% if k.args %} 37 | {% if k.docstring %} 38 | 39 | {% endif %} 40 | {% endif %} 41 | {% for arg in k.args %} 42 | {% for line in arg.doc %} 43 | {{ line }} 44 | {% endfor %} 45 | {% endfor %} 46 | """ 47 | {% for arg in k.args %} 48 | {% if arg.name not in ["keys", "items"] %} 49 | {{ arg.name }}: {{ arg.type }} 50 | {% else %} 51 | {{ arg.name }}: {{ arg.type }} # type: ignore[assignment] 52 | {% endif %} 53 | {% endfor %} 54 | {% if not k.for_response %} 55 | 56 | def __init__( 57 | self, 58 | {% for arg in k.args %} 59 | {% if arg.positional %} 60 | {{ arg.name }}: {{ arg.type }} = DEFAULT, 61 | {% endif %} 62 | {% endfor %} 63 | {% if k.args and not k.args[-1].positional %} 64 | *, 65 | {% endif %} 66 | {% for arg in k.args %} 67 | {% if not arg.positional %} 68 | {{ arg.name }}: {{ arg.type }} = DEFAULT, 69 | {% endif %} 70 | {% endfor %} 71 | **kwargs: Any 72 | ): 73 | {% if k.is_single_field %} 74 | if _field is not DEFAULT: 75 | kwargs[str(_field)] = _value 76 | {% elif k.is_multi_field %} 77 | if _fields is not DEFAULT: 78 | for field, value in _fields.items(): 79 | kwargs[str(field)] = value 80 | {% endif %} 81 | {% for arg in k.args %} 82 | {% if not arg.positional %} 83 | if {{ arg.name }} is not DEFAULT: 84 | {% if "InstrumentedField" in arg.type %} 85 | kwargs["{{ arg.name }}"] = str({{ arg.name }}) 86 | {% else %} 87 | kwargs["{{ arg.name }}"] = {{ arg.name }} 88 | {% endif %} 89 | {% endif %} 90 | {% endfor %} 91 | {% if k.parent %} 92 | super().__init__(**kwargs) 93 | {% else %} 94 | super().__init__(kwargs) 95 | {% endif %} 96 | {% endif %} 97 | {% if k.buckets_as_dict %} 98 | 99 | @property 100 | def buckets_as_dict(self) -> Mapping[str, {{ k.buckets_as_dict }}]: 101 | return self.buckets # type: ignore 102 | {% endif %} 103 | {% else %} 104 | pass 105 | {% endif %} 106 | 107 | {% endfor %} 108 | --------------------------------------------------------------------------------