├── tests ├── __init__.py ├── unit │ ├── __init__.py │ ├── test_match_all.py │ ├── test_type.py │ ├── test_term.py │ ├── test_exists.py │ ├── test_terms.py │ ├── test_query_string.py │ ├── test_missing.py │ ├── test_filter.py │ ├── test_copy.py │ ├── test_score.py │ ├── test_range.py │ ├── test_geo_distance.py │ ├── test_sort.py │ ├── test_bool.py │ ├── test_connection.py │ ├── test_aggregation.py │ └── test_queryset.py ├── functional │ ├── __init__.py │ ├── test_exists.py │ ├── test_type.py │ ├── test_match_all.py │ ├── test_missing.py │ ├── test_terms.py │ ├── test_connection.py │ ├── test_query_string.py │ ├── test_bool.py │ ├── test_filters.py │ ├── test_geo_distance.py │ ├── test_range.py │ ├── test_score.py │ ├── test_sorting.py │ ├── test_queryset.py │ └── test_aggregations.py └── helpers.py ├── .coveragerc ├── requirements.txt ├── MANIFEST.in ├── .gitignore ├── development.txt ├── pyeqs ├── dsl │ ├── match_all.py │ ├── exists.py │ ├── type.py │ ├── term.py │ ├── script_score.py │ ├── terms.py │ ├── missing.py │ ├── __init__.py │ ├── sort.py │ ├── range.py │ ├── query_string.py │ ├── geo.py │ └── aggregations.py ├── __init__.py ├── filter.py ├── bool.py ├── query_builder.py └── queryset.py ├── tox.ini ├── CONTRIBUTING.md ├── .travis.yml ├── LICENSE ├── Makefile ├── CHANGELOG.md ├── setup.py ├── README.md ├── API_REFERENCE.md └── pre-commit /tests/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /.coveragerc: -------------------------------------------------------------------------------- 1 | [run] 2 | include = pyeqs/* 3 | omit = pyeqs/packages/* -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | six>=1.0.0,<2.0.0 2 | elasticsearch>=1.0.0,<2.0.0 3 | -------------------------------------------------------------------------------- /tests/unit/__init__.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | from __future__ import unicode_literals 3 | 4 | import sure -------------------------------------------------------------------------------- /tests/functional/__init__.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | from __future__ import unicode_literals 3 | 4 | import sure 5 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | recursive-exclude tests * 2 | include README.md 3 | include COPYING 4 | include Makefile 5 | include requirements.txt 6 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *~ 2 | *.pyc 3 | .coverage 4 | *.egg-info/ 5 | dist/ 6 | .tox/ 7 | .DS_Store 8 | .build.log 9 | htmlcov/ 10 | build/ 11 | -------------------------------------------------------------------------------- /development.txt: -------------------------------------------------------------------------------- 1 | -r requirements.txt 2 | httpretty==0.8.0 3 | coverage==3.6 4 | nose==1.3.1 5 | sure==1.2.2 6 | mock==1.0.1 7 | tox==1.4.3 8 | coveralls 9 | pep8 -------------------------------------------------------------------------------- /pyeqs/dsl/match_all.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | from __future__ import unicode_literals, absolute_import 3 | 4 | 5 | class MatchAll(dict): 6 | 7 | def __init__(self): 8 | super(MatchAll, self).__init__() 9 | self._build_dict() 10 | 11 | def _build_dict(self): 12 | self["match_all"] = {} 13 | -------------------------------------------------------------------------------- /tox.ini: -------------------------------------------------------------------------------- 1 | [tox] 2 | envlist = py26,py27,py34 3 | 4 | [testenv] 5 | downloadcache = {toxworkdir}/_download/ 6 | commands = make all 7 | 8 | [testenv:py26] 9 | basepython = python2.6 10 | 11 | [testenv:py27] 12 | basepython = python2.7 13 | 14 | [testenv:py33] 15 | basepython = python3.3 16 | 17 | [testenv:py34] 18 | basepython = python3.4 -------------------------------------------------------------------------------- /pyeqs/__init__.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | from __future__ import unicode_literals 3 | 4 | from .filter import Filter # noqa 5 | from .bool import Bool # noqa 6 | from .query_builder import QueryBuilder # noqa 7 | from .queryset import QuerySet # noqa 8 | 9 | 10 | __all__ = ( 11 | 'Filter', 12 | 'Bool', 13 | 'Queryset', 14 | 'QueryBuilder', 15 | ) 16 | -------------------------------------------------------------------------------- /pyeqs/dsl/exists.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | from __future__ import unicode_literals, absolute_import 3 | 4 | 5 | class Exists(dict): 6 | 7 | def __init__(self, field): 8 | super(Exists, self).__init__() 9 | self.field = field 10 | self["exists"] = self._build_dict() 11 | 12 | def _build_dict(self): 13 | return { 14 | "field": self.field 15 | } 16 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | ## Contributing 2 | 3 | * Fork and download the `git` repo 4 | * Create a branch for your changes 5 | * Link the provided `pre-commit` hook to `.git/hooks/pre-commit` 6 | * Run the tests, *before you make changes*, to make sure everything works 7 | * Change the code for your PR 8 | * Write (or modify) tests to make sure your changes work. Aim for 100% unit and functional coverage. 9 | * Submit the Pull Request -------------------------------------------------------------------------------- /pyeqs/dsl/type.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | from __future__ import unicode_literals, absolute_import 3 | 4 | 5 | class Type(dict): 6 | 7 | def __init__(self, type_name): 8 | super(Type, self).__init__() 9 | self.type_name = type_name 10 | self["type"] = self._build_dict() 11 | 12 | def _build_dict(self): 13 | return { 14 | "value": self.type_name 15 | } 16 | -------------------------------------------------------------------------------- /pyeqs/filter.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | from __future__ import unicode_literals, absolute_import 3 | 4 | 5 | class Filter(dict): 6 | 7 | def __init__(self, operator="and"): 8 | super(Filter, self).__init__() 9 | self.operator = operator 10 | self[self.operator] = [] 11 | 12 | def filter(self, new_filter): 13 | self[self.operator].append(new_filter) 14 | return self 15 | -------------------------------------------------------------------------------- /pyeqs/dsl/term.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | from __future__ import unicode_literals, absolute_import 3 | 4 | 5 | class Term(dict): 6 | 7 | def __init__(self, field_name, term): 8 | super(Term, self).__init__() 9 | self.field_name = field_name 10 | self.term = term 11 | self["term"] = self._build_dict() 12 | 13 | def _build_dict(self): 14 | return { 15 | self.field_name: self.term 16 | } 17 | -------------------------------------------------------------------------------- /tests/unit/test_match_all.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | from __future__ import unicode_literals 3 | 4 | from pyeqs.dsl import MatchAll 5 | from tests.helpers import homogeneous 6 | 7 | 8 | def test_add_match_all(): 9 | """ 10 | Create Match All Block 11 | """ 12 | # When add a match all filter 13 | t = MatchAll() 14 | 15 | # Then I see the appropriate JSON 16 | results = { 17 | "match_all": {} 18 | } 19 | 20 | homogeneous(t, results) 21 | -------------------------------------------------------------------------------- /tests/unit/test_type.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | from __future__ import unicode_literals 3 | 4 | from pyeqs.dsl import Type 5 | from tests.helpers import homogeneous 6 | 7 | 8 | def test_add_type(): 9 | """ 10 | Create Type Block 11 | """ 12 | # When add a Type Block 13 | t = Type("foo") 14 | 15 | # Then I see the appropriate JSON 16 | results = { 17 | "type": { 18 | "value": "foo" 19 | } 20 | } 21 | 22 | homogeneous(t, results) 23 | -------------------------------------------------------------------------------- /tests/unit/test_term.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | from __future__ import unicode_literals 3 | 4 | from pyeqs.dsl import Term 5 | from tests.helpers import homogeneous 6 | 7 | 8 | def test_add_term(): 9 | """ 10 | Create Term Block 11 | """ 12 | # When add a Term field 13 | t = Term("foo", "bar") 14 | 15 | # Then I see the appropriate JSON 16 | results = { 17 | "term": { 18 | "foo": "bar" 19 | } 20 | } 21 | 22 | homogeneous(t, results) 23 | -------------------------------------------------------------------------------- /tests/unit/test_exists.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | from __future__ import unicode_literals 3 | 4 | from pyeqs.dsl import Exists 5 | from tests.helpers import homogeneous 6 | 7 | 8 | def test_exists(): 9 | """ 10 | Create Exists Block 11 | """ 12 | # When add an Exists Block 13 | t = Exists("foo") 14 | 15 | # Then I see the appropriate JSON 16 | results = { 17 | "exists": { 18 | "field": "foo" 19 | } 20 | } 21 | 22 | homogeneous(t, results) 23 | -------------------------------------------------------------------------------- /pyeqs/dsl/script_score.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | from __future__ import unicode_literals, absolute_import 3 | 4 | 5 | class ScriptScore(dict): 6 | 7 | def __init__(self, script, params=None, lang=None): 8 | super(ScriptScore, self).__init__() 9 | self.script = script 10 | self.params = params 11 | self.lang = lang 12 | self._build_dict() 13 | 14 | def _build_dict(self): 15 | self["script"] = self.script 16 | if self.params: 17 | self["params"] = self.params 18 | if self.lang: 19 | self["lang"] = self.lang 20 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: python 2 | python: 3 | - "2.6" 4 | - "2.7" 5 | - "3.3" 6 | - "3.4" 7 | env: 8 | matrix: 9 | - ES_VERSION=1.0.0 10 | - ES_VERSION=1.0.1 11 | - ES_VERSION=1.0.2 12 | - ES_VERSION=1.0.3 13 | - ES_VERSION=1.1.0 14 | - ES_VERSION=1.1.1 15 | before_install: 16 | - 'wget https://download.elasticsearch.org/elasticsearch/elasticsearch/elasticsearch-${ES_VERSION}.deb -O elasticsearch.deb' 17 | - 'sudo dpkg -i elasticsearch.deb' 18 | - 'sudo service elasticsearch start' 19 | install: "pip install -r development.txt --use-mirrors" 20 | script: 21 | - make 22 | after_success: 23 | - coveralls -------------------------------------------------------------------------------- /pyeqs/dsl/terms.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | from __future__ import unicode_literals, absolute_import 3 | 4 | 5 | class Terms(dict): 6 | 7 | def __init__(self, field_name, terms, execution=None): 8 | super(Terms, self).__init__() 9 | self.field_name = field_name 10 | self.terms = terms 11 | self.execution = execution 12 | self["terms"] = self._build_dict() 13 | 14 | def _build_dict(self): 15 | terms = { 16 | self.field_name: self.terms 17 | } 18 | if self.execution is not None: 19 | terms["execution"] = self.execution 20 | return terms 21 | -------------------------------------------------------------------------------- /pyeqs/dsl/missing.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | from __future__ import unicode_literals, absolute_import 3 | 4 | 5 | class Missing(dict): 6 | 7 | def __init__(self, field, existence=None, null_value=None): 8 | super(Missing, self).__init__() 9 | self.field = field 10 | self.existence = existence 11 | self.null_value = null_value 12 | self["missing"] = self._build_dict() 13 | 14 | def _build_dict(self): 15 | missing = {"field": self.field} 16 | if self.existence is not None: 17 | missing["existence"] = self.existence 18 | if self.null_value is not None: 19 | missing["null_value"] = self.null_value 20 | return missing 21 | -------------------------------------------------------------------------------- /pyeqs/dsl/__init__.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | from __future__ import unicode_literals 3 | 4 | from .aggregations import Aggregations # noqa 5 | from .exists import Exists # noqa 6 | from .geo import GeoDistance # noqa 7 | from .match_all import MatchAll # noqa 8 | from .missing import Missing # noqa 9 | from .query_string import QueryString # noqa 10 | from .range import Range # noqa 11 | from .script_score import ScriptScore # noqa 12 | from .sort import Sort # noqa 13 | from .term import Term # noqa 14 | from .terms import Terms # noqa 15 | from .type import Type # noqa 16 | 17 | __all__ = ( 18 | 'Aggregations', 19 | 'Exists', 20 | 'GeoDistance', 21 | 'MatchAll', 22 | 'Missing', 23 | 'QueryString', 24 | 'Range', 25 | 'ScriptScore', 26 | 'Sort', 27 | 'Term', 28 | 'Terms', 29 | 'Type', 30 | ) 31 | -------------------------------------------------------------------------------- /tests/functional/test_exists.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | from __future__ import unicode_literals 3 | 4 | from sure import scenario 5 | 6 | from pyeqs import QuerySet 7 | from pyeqs.dsl import Exists 8 | from tests.helpers import prepare_data, cleanup_data, add_document 9 | 10 | 11 | @scenario(prepare_data, cleanup_data) 12 | def test_search_with_exists(context): 13 | """ 14 | Search with exists filter 15 | """ 16 | # When create a query block 17 | t = QuerySet("localhost", index="foo") 18 | 19 | # And there are records 20 | add_document("foo", {"bar": 1}) 21 | add_document("foo", {"baz": 1}) 22 | 23 | # And I add an exists filter 24 | exists = Exists("baz") 25 | t.filter(exists) 26 | results = t[0:10] 27 | 28 | # Then my results only have that field 29 | len(results).should.equal(1) 30 | results[0]["_source"]["baz"].should.equal(1) 31 | -------------------------------------------------------------------------------- /pyeqs/dsl/sort.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | from __future__ import unicode_literals, absolute_import 3 | 4 | 5 | class Sort(dict): 6 | 7 | def __init__(self, field, order="asc", mode=None, missing=None, location=None): 8 | super(Sort, self).__init__() 9 | self.field = field 10 | self.order = order 11 | self.mode = mode 12 | self.missing = missing 13 | self.location = location 14 | self._build_dict() 15 | 16 | def _build_dict(self): 17 | sorting = {} 18 | sorting["order"] = self.order 19 | if self.mode: 20 | sorting["mode"] = self.mode 21 | if self.missing: 22 | sorting["missing"] = self.missing 23 | if self.location is not None: 24 | sorting[self.field] = self.location 25 | self["_geo_distance"] = sorting 26 | else: 27 | self[self.field] = sorting 28 | -------------------------------------------------------------------------------- /pyeqs/dsl/range.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | from __future__ import unicode_literals, absolute_import 3 | 4 | 5 | class Range(dict): 6 | 7 | def __init__(self, field_name, gte=None, gt=None, lte=None, lt=None): 8 | super(Range, self).__init__() 9 | self.field_name = field_name 10 | self.gte = gte 11 | self.gt = gt 12 | self.lte = lte 13 | self.lt = lt 14 | self["range"] = self._build_dict() 15 | 16 | def _build_dict(self): 17 | _range = { 18 | self.field_name: {} 19 | } 20 | if self.gte: 21 | _range[self.field_name]["gte"] = self.gte 22 | if self.gt: 23 | _range[self.field_name]["gt"] = self.gt 24 | if self.lte: 25 | _range[self.field_name]["lte"] = self.lte 26 | if self.lt: 27 | _range[self.field_name]["lt"] = self.lt 28 | return _range 29 | -------------------------------------------------------------------------------- /tests/functional/test_type.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | from __future__ import unicode_literals 3 | 4 | from sure import scenario 5 | 6 | from pyeqs import QuerySet 7 | from pyeqs.dsl import Type 8 | from tests.helpers import prepare_data, cleanup_data, add_document 9 | 10 | 11 | @scenario(prepare_data, cleanup_data) 12 | def test_search_with_type(context): 13 | """ 14 | Search with type filter 15 | """ 16 | # When create a query block 17 | t = QuerySet("localhost", index="foo") 18 | 19 | # And there are records 20 | add_document("foo", {"bar": 1}) 21 | add_document("foo", {"bar": 2}) 22 | add_document("foo", {"bar": 3}, doc_type="bar") 23 | 24 | # And I add a type filter 25 | _type = Type("bar") 26 | t.filter(_type) 27 | results = t[0:10] 28 | 29 | # Then my results only have that type 30 | len(results).should.equal(1) 31 | results[0]["_source"]["bar"].should.equal(3) 32 | -------------------------------------------------------------------------------- /tests/unit/test_terms.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | from __future__ import unicode_literals 3 | 4 | from pyeqs.dsl import Terms 5 | from tests.helpers import homogeneous 6 | 7 | 8 | def test_add_terms(): 9 | """ 10 | Create Terms Block 11 | """ 12 | # When add a Terms field 13 | t = Terms("foo", ["bar", "baz"]) 14 | 15 | # Then I see the appropriate JSON 16 | results = { 17 | "terms": { 18 | "foo": ["bar", "baz"] 19 | } 20 | } 21 | 22 | homogeneous(t, results) 23 | 24 | 25 | def test_add_terms_with_execution(): 26 | """ 27 | Create Terms With Execution 28 | """ 29 | # When add a Terms field 30 | t = Terms("foo", ["bar", "baz"], execution="and") 31 | 32 | # Then I see the appropriate JSON 33 | results = { 34 | "terms": { 35 | "foo": ["bar", "baz"], 36 | "execution": "and" 37 | 38 | } 39 | } 40 | 41 | homogeneous(t, results) 42 | -------------------------------------------------------------------------------- /tests/functional/test_match_all.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | from __future__ import unicode_literals 3 | 4 | from sure import scenario 5 | 6 | from pyeqs import QuerySet 7 | from pyeqs.dsl import MatchAll 8 | from tests.helpers import prepare_data, cleanup_data, add_document 9 | 10 | 11 | @scenario(prepare_data, cleanup_data) 12 | def test_match_all_search(context): 13 | """ 14 | Search with match all filter 15 | """ 16 | # When create a queryset 17 | t = QuerySet("localhost", index="foo") 18 | 19 | # And there are records 20 | add_document("foo", {"bar": "baz", "foo": "foo"}) 21 | add_document("foo", {"bar": "bazbaz", "foo": "foo"}) 22 | add_document("foo", {"bar": "baz", "foo": "foofoo"}) 23 | add_document("foo", {"bar": "baz", "foo": "foofoofoo"}) 24 | 25 | # And I filter match_all 26 | t.filter(MatchAll()) 27 | results = t[0:10] 28 | 29 | # Then I get a the expected results 30 | len(results).should.equal(4) 31 | -------------------------------------------------------------------------------- /pyeqs/bool.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | from __future__ import unicode_literals, absolute_import 3 | 4 | 5 | class Bool(dict): 6 | 7 | def __init__(self): 8 | super(Bool, self).__init__() 9 | self._must = [] 10 | self._must_not = [] 11 | self._should = [] 12 | self["bool"] = {} 13 | 14 | def bool(self, must=None, must_not=None, should=None): 15 | if must: 16 | self._must.append(must) 17 | self["bool"]["must"] = self._must 18 | if must_not: 19 | self._must_not.append(must_not) 20 | self["bool"]["must_not"] = self._must_not 21 | if should: 22 | self._should.append(should) 23 | self["bool"]["should"] = self._should 24 | return self 25 | 26 | def must(self, block): 27 | return self.bool(must=block) 28 | 29 | def must_not(self, block): 30 | return self.bool(must_not=block) 31 | 32 | def should(self, block): 33 | return self.bool(should=block) 34 | -------------------------------------------------------------------------------- /pyeqs/dsl/query_string.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | from __future__ import unicode_literals, absolute_import 3 | 4 | 5 | class QueryString(dict): 6 | 7 | def __init__(self, query, fields=None, default_field=None, tie_breaker=None, use_dis_max=None): 8 | super(QueryString, self).__init__() 9 | self.query = query 10 | self.fields = fields 11 | self.default_field = default_field 12 | self.tie_breaker = tie_breaker 13 | self.use_dis_max = use_dis_max 14 | self["query_string"] = self._build_dict() 15 | 16 | def _build_dict(self): 17 | query_string = {"query": self.query} 18 | if self.fields is not None: 19 | query_string["fields"] = self.fields 20 | elif self.default_field is not None: 21 | query_string["default_field"] = self.default_field 22 | if self.use_dis_max is not None: 23 | query_string["use_dis_max"] = self.use_dis_max 24 | if self.tie_breaker is not None: 25 | query_string["tie_breaker"] = self.tie_breaker 26 | return query_string 27 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2014 Yipit.com 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in 13 | all copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 21 | THE SOFTWARE. -------------------------------------------------------------------------------- /tests/unit/test_query_string.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | from __future__ import unicode_literals 3 | 4 | from pyeqs.dsl import QueryString 5 | from tests.helpers import homogeneous 6 | 7 | 8 | def test_query_string(): 9 | """ 10 | Create Query String Block 11 | """ 12 | # When I create a query string 13 | t = QueryString("foo", fields=["bar"], use_dis_max=True, tie_breaker=0.05) 14 | 15 | # Then I see the appropriate JSON 16 | results = { 17 | "query_string": { 18 | "query": "foo", 19 | "fields": ["bar"], 20 | "tie_breaker": 0.05, 21 | "use_dis_max": True 22 | } 23 | } 24 | 25 | homogeneous(t, results) 26 | 27 | 28 | def test_query_string_with_default_field(): 29 | """ 30 | Create Query String Block with default_field instead of fields 31 | """ 32 | # When we create a query string using a default_field 33 | t = QueryString("foo", default_field="bar", use_dis_max=True, tie_breaker=0.05) 34 | 35 | # Then we see the appropriate JSON 36 | results = { 37 | "query_string": { 38 | "query": "foo", 39 | "default_field": "bar", 40 | "tie_breaker": 0.05, 41 | "use_dis_max": True 42 | } 43 | } 44 | 45 | homogeneous(t, results) 46 | -------------------------------------------------------------------------------- /tests/unit/test_missing.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | from __future__ import unicode_literals 3 | 4 | from pyeqs.dsl import Missing 5 | from tests.helpers import homogeneous 6 | 7 | 8 | def test_missing(): 9 | """ 10 | Create Missing Block 11 | """ 12 | # When add a Missing Block 13 | t = Missing("foo") 14 | 15 | # Then I see the appropriate JSON 16 | results = { 17 | "missing": { 18 | "field": "foo" 19 | } 20 | } 21 | 22 | homogeneous(t, results) 23 | 24 | 25 | def test_missing_existence(): 26 | """ 27 | Create Missing Block with Existence 28 | """ 29 | # When add a Missing Block 30 | t = Missing("foo", existence=True) 31 | 32 | # Then I see the appropriate JSON 33 | results = { 34 | "missing": { 35 | "field": "foo", 36 | "existence": True 37 | } 38 | } 39 | 40 | homogeneous(t, results) 41 | 42 | 43 | def test_missing_null_value(): 44 | """ 45 | Create Missing Block with Null Value 46 | """ 47 | # When add a Missing Block 48 | t = Missing("foo", null_value=True) 49 | 50 | # Then I see the appropriate JSON 51 | results = { 52 | "missing": { 53 | "field": "foo", 54 | "null_value": True 55 | } 56 | } 57 | 58 | homogeneous(t, results) 59 | -------------------------------------------------------------------------------- /tests/functional/test_missing.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | from __future__ import unicode_literals 3 | 4 | from sure import scenario 5 | 6 | from pyeqs import QuerySet 7 | from pyeqs.dsl import Missing 8 | from tests.helpers import prepare_data, cleanup_data, add_document 9 | 10 | 11 | @scenario(prepare_data, cleanup_data) 12 | def test_search_with_missing(context): 13 | """ 14 | Search with missing 15 | """ 16 | # When create a query block 17 | t = QuerySet("localhost", index="foo") 18 | 19 | # And there are records 20 | add_document("foo", {"bar": 1}) 21 | add_document("foo", {"baz": 2}) 22 | add_document("foo", {"bar": 3}) 23 | 24 | # And I add filters 25 | t.filter(Missing("bar")) 26 | results = t[0:10] 27 | 28 | # Then my results are filtered correctly 29 | len(results).should.equal(1) 30 | results[0]["_source"]["baz"].should.equal(2) 31 | 32 | 33 | @scenario(prepare_data, cleanup_data) 34 | def test_search_with_missing_existence_null_value(context): 35 | """ 36 | Search with missing via non-existence or a null value 37 | """ 38 | # When create a query block 39 | t = QuerySet("localhost", index="foo") 40 | 41 | # And there are records 42 | add_document("foo", {"bar": 1}) 43 | add_document("foo", {"baz": 2}) 44 | add_document("foo", {"baz": 3, "bar": None}) 45 | 46 | # And I add filters 47 | t.filter(Missing("bar", existence=True, null_value=True)) 48 | results = t[0:10] 49 | 50 | # Then my results are filtered correctly 51 | len(results).should.equal(2) 52 | -------------------------------------------------------------------------------- /pyeqs/dsl/geo.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | from __future__ import unicode_literals, absolute_import 3 | 4 | from six import string_types 5 | 6 | 7 | class GeoDistance(dict): 8 | 9 | def __init__(self, coordinates, distance, field_name="location"): 10 | """ 11 | Geo Distance Filter 12 | 13 | Acceptable Input Styles: 14 | GeoDistance({'lat': 40, 'lon': -70}, '10km') 15 | GeoDistance({'lat': 40, 'lng': -70}, '10km') 16 | GeoDistance([40, -70], '10km') 17 | GeoDistance("-70,40", '10km') 18 | """ 19 | super(GeoDistance, self).__init__() 20 | self.coordinates = list(self._parse_coordinates(coordinates)) 21 | self.distance = distance 22 | self.field_name = field_name 23 | self["geo_distance"] = self._build_dict() 24 | 25 | def _build_dict(self): 26 | geo_distance = { 27 | "distance": self.distance, 28 | self.field_name: { 29 | 'lat': self.coordinates[0], 30 | 'lon': self.coordinates[1] 31 | } 32 | } 33 | return geo_distance 34 | 35 | def _parse_coordinates(self, coordinates): 36 | if isinstance(coordinates, list): 37 | lat = coordinates[1] 38 | lon = coordinates[0] 39 | if isinstance(coordinates, dict): 40 | lat = coordinates.pop('lat') 41 | lon = list(coordinates.values())[0] 42 | if isinstance(coordinates, string_types): 43 | lat, lon = coordinates.split(",") 44 | return map(float, [lat, lon]) 45 | -------------------------------------------------------------------------------- /tests/unit/test_filter.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | from __future__ import unicode_literals 3 | 4 | from pyeqs import Filter 5 | from pyeqs.dsl import Term 6 | from tests.helpers import homogeneous 7 | 8 | 9 | def test_create_filter(): 10 | """ 11 | Create Default Filter 12 | """ 13 | # When create a filter block 14 | t = Filter() 15 | 16 | # Then I see the appropriate JSON 17 | results = { 18 | "and": [] 19 | } 20 | 21 | homogeneous(t, results) 22 | 23 | 24 | def test_add_filter(): 25 | """ 26 | Create Filter with Block 27 | """ 28 | # When I create a filter 29 | t = Filter() 30 | 31 | # And add a block 32 | t.filter(Term("foo", "bar")) 33 | 34 | # Then I see the appropriate JSON 35 | results = { 36 | "and": [ 37 | { 38 | "term": { 39 | "foo": "bar" 40 | } 41 | } 42 | ] 43 | } 44 | 45 | homogeneous(t, results) 46 | 47 | 48 | def test_filter_with_or(): 49 | """ 50 | Create OR Filter 51 | """ 52 | # When create a filter block 53 | t = Filter("or") 54 | 55 | # Then I see the appropriate JSON 56 | results = { 57 | "or": [] 58 | } 59 | 60 | homogeneous(t, results) 61 | 62 | 63 | def test_filter_with_and(): 64 | """ 65 | Create AND Filter 66 | """ 67 | # When create a filter block 68 | t = Filter("and") 69 | 70 | # Then I see the appropriate JSON 71 | results = { 72 | "and": [] 73 | } 74 | 75 | homogeneous(t, results) 76 | -------------------------------------------------------------------------------- /tests/unit/test_copy.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | from __future__ import unicode_literals 3 | 4 | import httpretty 5 | import json 6 | import sure 7 | from mock import Mock 8 | 9 | from pyeqs import QuerySet, Filter 10 | from pyeqs.dsl import Term, Sort, ScriptScore 11 | from tests.helpers import heterogeneous, homogeneous 12 | 13 | 14 | def test_copy_queryset(): 15 | """ 16 | Copy Queryset object when used as Model 17 | """ 18 | # When create a queryset 19 | t = QuerySet("http://foobar:9200") 20 | 21 | new_object = t.objects 22 | 23 | # Then the new object is not the same object as the queryset 24 | assert(new_object is not t) 25 | 26 | # And is not the same query object 27 | assert(new_object._query is not t._query) 28 | 29 | # But it is has the same properties 30 | homogeneous(new_object._query, t._query) 31 | 32 | 33 | def test_copy_queryset_with_filters(): 34 | """ 35 | Copy Queryset object and ensure distinct filters 36 | """ 37 | # When create a queryset 38 | t = QuerySet("http://foobar:9200") 39 | 40 | # With filters 41 | t.filter(Term("foo", "bar")) 42 | 43 | # And I clone the queryset 44 | new_object = t.objects 45 | 46 | # And add new filters 47 | new_object.filter(Term("bar", "baz")) 48 | 49 | # Then the new object is not the same object as the queryset 50 | assert(new_object is not t) 51 | 52 | # And is not the same query object 53 | assert(new_object._query is not t._query) 54 | 55 | # But it is has the same properties 56 | heterogeneous(new_object._query, t._query) 57 | -------------------------------------------------------------------------------- /tests/functional/test_terms.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | from __future__ import unicode_literals 3 | 4 | from sure import scenario 5 | 6 | from pyeqs import QuerySet 7 | from pyeqs.dsl import Terms 8 | from tests.helpers import prepare_data, cleanup_data, add_document 9 | 10 | 11 | @scenario(prepare_data, cleanup_data) 12 | def test_terms_search(context): 13 | """ 14 | Search with terms filter 15 | """ 16 | # When create a queryset 17 | t = QuerySet("localhost", index="foo") 18 | 19 | # And there are records 20 | add_document("foo", {"bar": "baz", "foo": "foo"}) 21 | add_document("foo", {"bar": "bazbaz", "foo": "foo"}) 22 | add_document("foo", {"bar": "baz", "foo": "foofoo"}) 23 | add_document("foo", {"bar": "baz", "foo": "foofoofoo"}) 24 | 25 | # And I filter for terms 26 | t.filter(Terms("foo", ["foo", "foofoo"])) 27 | results = t[0:10] 28 | 29 | # Then I get a the expected results 30 | len(results).should.equal(3) 31 | 32 | 33 | @scenario(prepare_data, cleanup_data) 34 | def test_terms_search_with_execution(context): 35 | """ 36 | Search with terms filter with execution 37 | """ 38 | # When create a queryset 39 | t = QuerySet("localhost", index="foo") 40 | 41 | # And there are records 42 | add_document("foo", {"foo": ["foo", "bar"]}) 43 | add_document("foo", {"foo": ["foo", "baz"]}) 44 | 45 | # And I filter for terms 46 | t.filter(Terms("foo", ["foo", "bar"], execution="and")) 47 | results = t[0:10] 48 | 49 | # Then I get a the expected results 50 | len(results).should.equal(1) 51 | results[0]["_source"]["foo"].should.equal(["foo", "bar"]) 52 | -------------------------------------------------------------------------------- /tests/unit/test_score.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | from __future__ import unicode_literals 3 | 4 | from pyeqs.dsl import ScriptScore 5 | from tests.helpers import homogeneous 6 | 7 | 8 | def test_add_score(): 9 | """ 10 | Create Score Block 11 | """ 12 | # When add a Score field 13 | t = ScriptScore("foo") 14 | 15 | # Then I see the appropriate JSON 16 | results = { 17 | "script": "foo" 18 | } 19 | 20 | homogeneous(t, results) 21 | 22 | 23 | def test_add_score_with_params(): 24 | """ 25 | Create Score Block with Params 26 | """ 27 | # When add a Score field 28 | t = ScriptScore("foo", params={"bar": "baz"}) 29 | 30 | # Then I see the appropriate JSON 31 | results = { 32 | "script": "foo", 33 | "params": { 34 | "bar": "baz" 35 | } 36 | } 37 | 38 | homogeneous(t, results) 39 | 40 | 41 | def test_add_score_with_lang(): 42 | """ 43 | Create Score Block with Language 44 | """ 45 | # When add a Score field 46 | t = ScriptScore("foo", lang="mvel") 47 | 48 | # Then I see the appropriate JSON 49 | results = { 50 | "script": "foo", 51 | "lang": "mvel" 52 | } 53 | 54 | homogeneous(t, results) 55 | 56 | 57 | def test_add_score_with_params_and_lang(): 58 | """ 59 | Create Score Block with Params and Language 60 | """ 61 | # When add a Score field 62 | t = ScriptScore("foo", params={"bar": "baz"}, lang="mvel") 63 | 64 | # Then I see the appropriate JSON 65 | results = { 66 | "script": "foo", 67 | "params": { 68 | "bar": "baz" 69 | }, 70 | "lang": "mvel" 71 | } 72 | 73 | homogeneous(t, results) 74 | -------------------------------------------------------------------------------- /tests/unit/test_range.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | from __future__ import unicode_literals 3 | 4 | from pyeqs.dsl import Range 5 | from tests.helpers import homogeneous 6 | 7 | 8 | def test_add_gte_range(): 9 | """ 10 | Create Greater Than Or Equal Range Block 11 | """ 12 | # When add a Range Block 13 | t = Range("foo", gte="bar") 14 | 15 | # Then I see the appropriate JSON 16 | results = { 17 | "range": { 18 | "foo": { 19 | "gte": "bar", 20 | } 21 | } 22 | } 23 | 24 | homogeneous(t, results) 25 | 26 | 27 | def test_add_gt_range(): 28 | """ 29 | Create Greater Than Range Block 30 | """ 31 | # When add a Range Block 32 | t = Range("foo", gt="bar") 33 | 34 | # Then I see the appropriate JSON 35 | results = { 36 | "range": { 37 | "foo": { 38 | "gt": "bar" 39 | } 40 | } 41 | } 42 | 43 | homogeneous(t, results) 44 | 45 | 46 | def test_add_lte_range(): 47 | """ 48 | Create Less Than Or Equal Range Block 49 | """ 50 | # When add a Range Block 51 | t = Range("foo", lte="bar") 52 | 53 | # Then I see the appropriate JSON 54 | results = { 55 | "range": { 56 | "foo": { 57 | "lte": "bar" 58 | } 59 | } 60 | } 61 | 62 | homogeneous(t, results) 63 | 64 | 65 | def test_add_lt_range(): 66 | """ 67 | Create Less Than Range Block 68 | """ 69 | # When add a Range Block 70 | t = Range("foo", lt="bar") 71 | 72 | # Then I see the appropriate JSON 73 | results = { 74 | "range": { 75 | "foo": { 76 | "lt": "bar" 77 | } 78 | } 79 | } 80 | 81 | homogeneous(t, results) 82 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | # Variables you might need to change in the first place 2 | # 3 | # This is probably the only section you'll need to change in this Makefile. 4 | # Also, make sure you don't remove the `' tag. Cause those marks 5 | # are going to be used to update this file automatically. 6 | # 7 | # 8 | PACKAGE=pyeqs 9 | CUSTOM_PIP_INDEX=pypi 10 | # 11 | 12 | all: unit functional 13 | 14 | unit: 15 | @make run_test suite=unit 16 | 17 | functional: 18 | @make run_test suite=functional 19 | 20 | prepare: clean install_deps 21 | 22 | run_test: 23 | @if [ -d tests/$(suite) ]; then \ 24 | echo "Running \033[0;32m$(suite)\033[0m test suite"; \ 25 | make prepare && \ 26 | nosetests --stop --with-coverage --cover-package=$(PACKAGE) \ 27 | --cover-branches --cover-erase --verbosity=2 -s tests/$(suite) ; \ 28 | fi 29 | 30 | install_deps: 31 | @if [ -z $$VIRTUAL_ENV ]; then \ 32 | echo "You're not running this from a virtualenv, wtf?"; \ 33 | exit 1; \ 34 | fi 35 | 36 | @if [ -z $$SKIP_DEPS ]; then \ 37 | echo "Installing missing dependencies..."; \ 38 | [ -e development.txt ] && pip install -r development.txt --quiet; \ 39 | fi 40 | 41 | @python setup.py develop &> .build.log 42 | 43 | clean: 44 | @echo "Removing garbage..." 45 | @find . -name '*.pyc' -delete 46 | @find . -name '*.so' -delete 47 | @find . -name __pycache__ -delete 48 | @rm -rf .coverage *.egg-info *.log build dist MANIFEST htmlcov .DS_Store .build.log 49 | 50 | publish: 51 | @if [ -e "$$HOME/.pypirc" ]; then \ 52 | echo "Uploading to '$(CUSTOM_PIP_INDEX)'"; \ 53 | python setup.py register -r "$(CUSTOM_PIP_INDEX)"; \ 54 | python setup.py sdist upload -r "$(CUSTOM_PIP_INDEX)"; \ 55 | else \ 56 | echo "You should create a file called \`.pypirc' under your home dir.\n"; \ 57 | echo "That's the right place to configure \`pypi' repos.\n"; \ 58 | exit 1; \ 59 | fi 60 | -------------------------------------------------------------------------------- /tests/functional/test_connection.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | from __future__ import unicode_literals 3 | 4 | from sure import scenario 5 | 6 | from pyeqs import QuerySet 7 | from tests.helpers import prepare_data, cleanup_data, add_document 8 | 9 | 10 | @scenario(prepare_data, cleanup_data) 11 | def test_simple_search_with_host_string(context): 12 | """ 13 | Connect with host string 14 | """ 15 | # When create a queryset 16 | t = QuerySet("localhost", index="foo") 17 | 18 | # And there are records 19 | add_document("foo", {"bar": "baz"}) 20 | 21 | # And I do a search 22 | results = t[0:1] 23 | 24 | # Then I get a the expected results 25 | len(results).should.equal(1) 26 | results[0]['_source'].should.equal({"bar": "baz"}) 27 | 28 | 29 | @scenario(prepare_data, cleanup_data) 30 | def test_simple_search_with_host_dict(context): 31 | """ 32 | Connect with host dict 33 | """ 34 | # When create a queryset 35 | connection_info = {"host": "localhost", "port": 9200} 36 | t = QuerySet(connection_info, index="foo") 37 | 38 | # And there are records 39 | add_document("foo", {"bar": "baz"}) 40 | 41 | # And I do a search 42 | results = t[0:1] 43 | 44 | # Then I get a the expected results 45 | len(results).should.equal(1) 46 | results[0]['_source'].should.equal({"bar": "baz"}) 47 | 48 | 49 | @scenario(prepare_data, cleanup_data) 50 | def test_simple_search_with_host_list(context): 51 | """ 52 | Connect with host list 53 | """ 54 | # When create a queryset 55 | connection_info = [{"host": "localhost", "port": 9200}] 56 | t = QuerySet(connection_info, index="foo") 57 | 58 | # And there are records 59 | add_document("foo", {"bar": "baz"}) 60 | 61 | # And I do a search 62 | results = t[0:1] 63 | 64 | # Then I get a the expected results 65 | len(results).should.equal(1) 66 | results[0]['_source'].should.equal({"bar": "baz"}) 67 | -------------------------------------------------------------------------------- /tests/functional/test_query_string.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | from __future__ import unicode_literals 3 | 4 | from sure import scenario 5 | 6 | from pyeqs import QuerySet 7 | from pyeqs.dsl import QueryString 8 | from tests.helpers import prepare_data, cleanup_data, add_document 9 | 10 | 11 | @scenario(prepare_data, cleanup_data) 12 | def test_query_string(context): 13 | """ 14 | Search with query string 15 | """ 16 | # When I create a queryset with a query string 17 | qs = QueryString("cheese") 18 | t = QuerySet("localhost", index="foo", query=qs) 19 | 20 | # And there are records 21 | add_document("foo", {"bar": "baz"}) 22 | add_document("foo", {"bar": "cheese"}) 23 | 24 | # Then I get a the expected results 25 | results = t[0:10] 26 | len(results).should.equal(1) 27 | 28 | 29 | @scenario(prepare_data, cleanup_data) 30 | def test_query_string_with_additional_parameters(context): 31 | """ 32 | Search with query string and additional parameters 33 | """ 34 | # When I create a queryset with parameters 35 | qs = QueryString("cheese", fields=["bar"], use_dis_max=False, tie_breaker=0.05) 36 | t = QuerySet("localhost", index="foo", query=qs) 37 | 38 | # And there are records 39 | add_document("foo", {"bar": "baz"}) 40 | add_document("foo", {"bar": "cheese"}) 41 | 42 | # Then I get a the expected results 43 | results = t[0:10] 44 | len(results).should.equal(1) 45 | 46 | 47 | @scenario(prepare_data, cleanup_data) 48 | def test_query_string_with_different_parameters(context): 49 | """ 50 | Search with query string and different parameters 51 | """ 52 | # When I create a queryset with parameters 53 | qs = QueryString("cheese", default_field="bar", use_dis_max=False, tie_breaker=0.05) 54 | t = QuerySet("localhost", index="foo", query=qs) 55 | 56 | # And there are records 57 | add_document("foo", {"bar": "baz"}) 58 | add_document("foo", {"bar": "cheese"}) 59 | 60 | # Then I get a the expected results 61 | results = t[0:10] 62 | len(results).should.equal(1) 63 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | ## `0.13.1` 2 | * Add `default_field` support for `QueryString` 3 | 4 | ## `0.13.0` 5 | 6 | * Add additional `aggregations` functionality, thanks to @zkourouma 7 | 8 | ### `0.12.0` 9 | 10 | * Add `max_score()` method to queryset 11 | 12 | ### `0.11.0` 13 | 14 | * Add `size` parameter to `Terms()` aggregations 15 | 16 | ### `0.10.1` 17 | 18 | * Fixing python 2.6 compat in aggregations 19 | 20 | ### `0.10.0` 21 | 22 | * Add `aggregations` functionality. Special thanks to @zkourouma for the hard work. 23 | 24 | ### `0.9.0` 25 | 26 | * Properly increment version number for SemVer. 27 | 28 | ### `0.8.2` 29 | 30 | * Add `post_query_actions` to provide hooks for logging and debugging inside the iterator. 31 | 32 | ### `0.8.1` 33 | 34 | * Add `score_mode` kwarg to scoring blocks. 35 | 36 | ### `0.8.0` 37 | 38 | * Add Exists Block 39 | * Update `function_score` to support multiple scoring blocks. 40 | 41 | ### `0.7.4` 42 | 43 | * Add ability to sort on `location` fields. See the API Reference for details. 44 | 45 | ### `0.7.3` 46 | 47 | * Add `min_score` and `track_scores` options to `score`. 48 | 49 | ### `0.7.2` 50 | 51 | * Add `QuerySet` to DSL 52 | 53 | ### `0.7.1` 54 | 55 | * Add `execution` option to Terms DSL 56 | 57 | ### `0.7.0` 58 | 59 | * Add python 3 support 60 | 61 | ### `0.6.3` 62 | 63 | * Revent cloning change 64 | 65 | ### `0.6.2` 66 | 67 | * Force creation of clones as `QuerySet` objects 68 | 69 | ### `0.6.1` 70 | 71 | * Fix broken syntax around `clone` method 72 | 73 | ### `0.6.0` 74 | 75 | * Add `missing` to DSL 76 | 77 | ### `0.5.1` 78 | 79 | * Allow custom field names in GeoDistance queries 80 | 81 | ### `0.5.0` 82 | 83 | * Add `match_all` to DSL. 84 | * Switch default query to use new DSL match all 85 | 86 | ### `0.4.0` 87 | 88 | * Add option to specify 'missing' when creating a sort block 89 | 90 | ### `0.3.3` 91 | 92 | * More flexible passing of connection parameters to backend library 93 | 94 | ### `0.3.2` 95 | 96 | * Don't lock `six` version so explicitly. 97 | 98 | ### `0.3.1` 99 | 100 | * Don't override `range` keyword when constructing `Range` dict. 101 | -------------------------------------------------------------------------------- /tests/functional/test_bool.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | from __future__ import unicode_literals 3 | 4 | from sure import scenario 5 | 6 | from pyeqs import QuerySet, Bool 7 | from pyeqs.dsl import Term 8 | from tests.helpers import prepare_data, cleanup_data, add_document 9 | 10 | 11 | @scenario(prepare_data, cleanup_data) 12 | def test_simple_search_with_boolean_must(context): 13 | """ 14 | Search with boolean must 15 | """ 16 | # When create a queryset 17 | t = QuerySet("localhost", index="foo") 18 | 19 | # And there are records 20 | add_document("foo", {"bar": "baz"}) 21 | add_document("foo", {"bar": "bazbaz"}) 22 | 23 | # And I do a filtered search 24 | b = Bool() 25 | b.must(Term("bar", "baz")) 26 | t.filter(b) 27 | results = t[0:10] 28 | 29 | # Then I get a the expected results 30 | len(results).should.equal(1) 31 | results[0]['_source'].should.equal({"bar": "baz"}) 32 | 33 | 34 | @scenario(prepare_data, cleanup_data) 35 | def test_simple_search_with_boolean_must_not(context): 36 | """ 37 | Search with boolean must not 38 | """ 39 | # When create a queryset 40 | t = QuerySet("localhost", index="foo") 41 | 42 | # And there are records 43 | add_document("foo", {"bar": "baz"}) 44 | add_document("foo", {"bar": "bazbaz"}) 45 | 46 | # And I do a filtered search 47 | b = Bool() 48 | b.must_not(Term("bar", "baz")) 49 | t.filter(b) 50 | results = t[0:10] 51 | 52 | # Then I get a the expected results 53 | len(results).should.equal(1) 54 | results[0]['_source'].should.equal({"bar": "bazbaz"}) 55 | 56 | 57 | @scenario(prepare_data, cleanup_data) 58 | def test_simple_search_with_boolean_should(context): 59 | """ 60 | Search with boolean should 61 | """ 62 | # When create a queryset 63 | t = QuerySet("localhost", index="foo") 64 | 65 | # And there are records 66 | add_document("foo", {"bar": "baz"}) 67 | add_document("foo", {"bar": "bazbaz"}) 68 | 69 | # And I do a filtered search 70 | b = Bool() 71 | b.should(Term("bar", "baz")) 72 | t.filter(b) 73 | results = t[0:10] 74 | 75 | # Then I get a the expected results 76 | len(results).should.equal(1) 77 | results[0]['_source'].should.equal({"bar": "baz"}) 78 | -------------------------------------------------------------------------------- /tests/unit/test_geo_distance.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | from __future__ import unicode_literals 3 | 4 | from pyeqs.dsl import GeoDistance 5 | from tests.helpers import homogeneous 6 | 7 | 8 | def test_add_geo_distance_with_dict(): 9 | """ 10 | Create Geo Distance with Dictionary 11 | """ 12 | # When add a Geo Distance field 13 | t = GeoDistance({"lat": 1.0, "lon": 2.0}, "20mi") 14 | 15 | # Then I see the appropriate JSON 16 | results = { 17 | "geo_distance": { 18 | "distance": "20mi", 19 | "location": { 20 | "lat": 1.0, 21 | "lon": 2.0 22 | } 23 | } 24 | } 25 | 26 | homogeneous(t, results) 27 | 28 | 29 | def test_add_geo_distance_with_string(): 30 | """ 31 | Create Geo Distance with String 32 | """ 33 | # When add a Geo Distance field 34 | t = GeoDistance("1.0,2.0", "20mi") 35 | 36 | # Then I see the appropriate JSON 37 | results = { 38 | "geo_distance": { 39 | "distance": "20mi", 40 | "location": { 41 | "lat": 1.0, 42 | "lon": 2.0 43 | } 44 | } 45 | } 46 | 47 | homogeneous(t, results) 48 | 49 | 50 | def test_add_geo_distance_with_array(): 51 | """ 52 | Create Geo Distance with Array 53 | """ 54 | # When add a Geo Distance field 55 | t = GeoDistance([2.0, 1.0], "20mi") 56 | 57 | # Then I see the appropriate JSON 58 | results = { 59 | "geo_distance": { 60 | "distance": "20mi", 61 | "location": { 62 | "lat": 1.0, 63 | "lon": 2.0 64 | } 65 | } 66 | } 67 | 68 | homogeneous(t, results) 69 | 70 | 71 | def test_add_geo_distance_with_field_name(): 72 | """ 73 | Create Geo Distance with Field Name 74 | """ 75 | # When add a Geo Distance field with a field name 76 | t = GeoDistance({"lat": 1.0, "lon": 2.0}, "20mi", field_name="locations.location") 77 | 78 | # Then I see the appropriate JSON 79 | results = { 80 | "geo_distance": { 81 | "distance": "20mi", 82 | "locations.location": { 83 | "lat": 1.0, 84 | "lon": 2.0 85 | } 86 | } 87 | } 88 | 89 | homogeneous(t, results) 90 | -------------------------------------------------------------------------------- /tests/functional/test_filters.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | from __future__ import unicode_literals 3 | 4 | from sure import scenario 5 | 6 | from pyeqs import QuerySet, Filter 7 | from pyeqs.dsl import Term 8 | from tests.helpers import prepare_data, cleanup_data, add_document 9 | 10 | 11 | @scenario(prepare_data, cleanup_data) 12 | def test_simple_search_with_filter(context): 13 | """ 14 | Search with filter 15 | """ 16 | # When create a queryset 17 | t = QuerySet("localhost", index="foo") 18 | 19 | # And there are records 20 | add_document("foo", {"bar": "baz"}) 21 | add_document("foo", {"bar": "bazbaz"}) 22 | 23 | # And I do a filtered search 24 | t.filter(Term("bar", "baz")) 25 | results = t[0:10] 26 | 27 | # Then I get a the expected results 28 | len(results).should.equal(1) 29 | results[0]['_source'].should.equal({"bar": "baz"}) 30 | 31 | 32 | @scenario(prepare_data, cleanup_data) 33 | def test_search_with_multiple_filters(context): 34 | """ 35 | Search with multiple filters 36 | """ 37 | # When create a query block 38 | t = QuerySet("localhost", index="foo") 39 | 40 | # And there are records 41 | add_document("foo", {"bar": "baz", "foo": "foo"}) 42 | add_document("foo", {"bar": "bazbaz", "foo": "foo"}) 43 | add_document("foo", {"bar": "bazbaz", "foo": "foofoo"}) 44 | 45 | # And I do a filtered search 46 | t.filter(Term("bar", "bazbaz")) 47 | t.filter(Term("foo", "foo")) 48 | results = t[0:10] 49 | 50 | # Then I get the appropriate response 51 | len(results).should.equal(1) 52 | results[0]['_source'].should.equal({"bar": "bazbaz", "foo": "foo"}) 53 | 54 | 55 | @scenario(prepare_data, cleanup_data) 56 | def test_search_with_filter_block(context): 57 | """ 58 | Search with Filter Block 59 | """ 60 | # When create a query block 61 | t = QuerySet("localhost", index="foo") 62 | 63 | # And there are records 64 | add_document("foo", {"bar": "baz", "foo": "foo"}) 65 | add_document("foo", {"bar": "bazbaz", "foo": "foo"}) 66 | add_document("foo", {"bar": "bazbaz", "foo": "foofoo"}) 67 | 68 | # And I do a filtered search 69 | f = Filter("or").filter(Term("bar", "baz")).filter(Term("foo", "foo")) 70 | t.filter(f) 71 | results = t[0:10] 72 | 73 | # Then I get the appropriate response 74 | len(results).should.equal(2) 75 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | # #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | from __future__ import unicode_literals, absolute_import 4 | 5 | import re 6 | import os 7 | from setuptools import setup, find_packages 8 | 9 | 10 | def parse_requirements(): 11 | """ 12 | Rudimentary parser for the `requirements.txt` file 13 | 14 | We just want to separate regular packages from links to pass them to the 15 | `install_requires` and `dependency_links` params of the `setup()` 16 | function properly. 17 | """ 18 | try: 19 | requirements = \ 20 | map(str.strip, local_file('requirements.txt')) 21 | except IOError: 22 | raise RuntimeError("Couldn't find the `requirements.txt' file :(") 23 | 24 | links = [] 25 | pkgs = [] 26 | for req in requirements: 27 | if not req: 28 | continue 29 | if 'http:' in req or 'https:' in req: 30 | links.append(req) 31 | name, version = re.findall("\#egg=([^\-]+)-(.+$)", req)[0] 32 | pkgs.append('{0}=={1}'.format(name, version)) 33 | else: 34 | pkgs.append(req) 35 | 36 | return pkgs, links 37 | 38 | local_file = lambda f: \ 39 | open(os.path.join(os.path.dirname(__file__), f)).readlines() 40 | 41 | 42 | if __name__ == '__main__': 43 | 44 | packages = find_packages(exclude=['*tests*']) 45 | pkgs, links = parse_requirements() 46 | 47 | setup( 48 | name="pyeqs", 49 | license="MIT", 50 | version='0.13.1', 51 | description=u'Django Querysets-esque implementation for Elasticsearch', 52 | author=u'Andrew Gross', 53 | author_email=u'andrew.w.gross@gmail.com', 54 | include_package_data=True, 55 | url='https://github.com/yipit/pyeqs', 56 | packages=packages, 57 | install_requires=pkgs, 58 | classifiers=( 59 | 'Development Status :: 3 - Alpha', 60 | 'Intended Audience :: Developers', 61 | 'License :: OSI Approved :: MIT', 62 | 'Natural Language :: English', 63 | 'Operating System :: Microsoft', 64 | 'Operating System :: POSIX', 65 | 'Programming Language :: Python', 66 | 'Programming Language :: Python :: 2.6', 67 | 'Programming Language :: Python :: 2.7', 68 | 'Programming Language :: Python :: 3.3', 69 | 'Programming Language :: Python :: 3.4', 70 | ) 71 | ) 72 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # PyEQS [![Build Status](https://travis-ci.org/Yipit/pyeqs.svg?branch=master)](https://travis-ci.org/Yipit/pyeqs) [![Coverage Status](https://coveralls.io/repos/Yipit/pyeqs/badge.png)](https://coveralls.io/r/Yipit/pyeqs) 2 | 3 | #### Python Elasticsearch QuerySets 4 | 5 | A python library to simplify building complex Elasticsearch JSON queries. Based on the Django QuerySet API, backed by the [official python elasticsearch library](https://github.com/elasticsearch/elasticsearch-py). Supports Elasticsearch `1.0+`. 6 | 7 | This is an attempt to provide an interface familiar to users of Django Querysets. Due to the differences in the backends it was impossible to mirror the Queryset API and maintain full search functionality. Be aware when using this library that the interfaces may not have the same trade-offs and caveats. 8 | 9 | #### Current Development Status 10 | 11 | Currently pre `v1.0`, so the API is not locked in. This project aims to follow [semantic versioning](http://semver.org/) once it reaches a stable API. Issues may arise as the backend `elasticsearch-py` library locks its versions to **Elasticsearch** releases. 12 | 13 | ## Installation 14 | 15 | ```bash 16 | pip install pyeqs 17 | ``` 18 | 19 | ## Usage 20 | 21 | Check out the [API Reference](https://github.com/Yipit/pyeqs/blob/master/API_REFERENCE.md) for examples. 22 | 23 | ## Alternatives 24 | 25 | #### Python 26 | * [ElasticUtils](http://elasticutils.readthedocs.org/en/latest/): A library by Mozilla uses a syntax leveraging built-in &, | and ~ to construct queries. 27 | * [Elasticsearch-dsl-py](https://github.com/elasticsearch/elasticsearch-dsl-py): A library by Elasticsearch that is similar and compatible with ElasticUtils. 28 | * [Django-Haystack](https://github.com/toastdriven/django-haystack): A library that wraps multiple search backends and presents them in the same interface as Django models. In my experience a very all-in-one solution that makes it hard to manipulate Elasticsearch directly, but wonderful when you need the feature set. 29 | 30 | #### Ruby 31 | * [Plunk](https://github.com/elbii/plunk): A ruby library to allow you to write strings to queries that have more power than simple 'query string' requests 32 | 33 | #### Haskell 34 | * [Bloodhound](https://github.com/bitemyapp/bloodhound/): A basic Elasticsearch Client that also has the ability to leverage the language's built-in operators to construct queries. 35 | 36 | #### Perl 37 | * [ElasticSearch::SearchBuilder](https://metacpan.org/pod/ElasticSearch::SearchBuilder): An Elasticsearch Client to help with constructing complex queries and filters. 38 | -------------------------------------------------------------------------------- /tests/unit/test_sort.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | from __future__ import unicode_literals 3 | 4 | from pyeqs.dsl import Sort 5 | from tests.helpers import homogeneous 6 | 7 | 8 | def test_add_sort(): 9 | """ 10 | Create Sort Block 11 | """ 12 | # When add a Sort block 13 | t = Sort("foo") 14 | 15 | # Then I see the appropriate JSON 16 | results = { 17 | "foo": { 18 | "order": "asc" 19 | } 20 | } 21 | 22 | homogeneous(t, results) 23 | 24 | 25 | def test_add_sort_asc(): 26 | """ 27 | Create Sort Block Ascending 28 | """ 29 | # When add a Sort block 30 | t = Sort("foo", order="asc") 31 | 32 | # Then I see the appropriate JSON 33 | results = { 34 | "foo": { 35 | "order": "asc" 36 | } 37 | } 38 | 39 | homogeneous(t, results) 40 | 41 | 42 | def test_add_sort_desc(): 43 | """ 44 | Create Sort Block Descending 45 | """ 46 | # When add a Sort block 47 | t = Sort("foo", order="dsc") 48 | 49 | # Then I see the appropriate JSON 50 | results = { 51 | "foo": { 52 | "order": "dsc" 53 | } 54 | } 55 | 56 | homogeneous(t, results) 57 | 58 | 59 | def test_add_sort_with_mode(): 60 | """ 61 | Create Sort Block with Mode 62 | """ 63 | # When add a Sort block 64 | t = Sort("foo", mode="bar") 65 | 66 | # Then I see the appropriate JSON 67 | results = { 68 | "foo": { 69 | "order": "asc", 70 | "mode": "bar" 71 | } 72 | } 73 | 74 | homogeneous(t, results) 75 | 76 | 77 | def test_add_sort_with_mode_and_missing(): 78 | """ 79 | Create Sort Block with Mode and Missing 80 | """ 81 | # When add a Sort block 82 | t = Sort("foo", mode="bar", missing="_last") 83 | 84 | # Then I see the appropriate JSON 85 | results = { 86 | "foo": { 87 | "order": "asc", 88 | "mode": "bar", 89 | "missing": "_last" 90 | } 91 | } 92 | 93 | homogeneous(t, results) 94 | 95 | 96 | def test_add_sort_with_value(): 97 | """ 98 | Create Sort Block with Value 99 | """ 100 | # When add a Sort block 101 | t = Sort("location", location=[1.23, 3.45]) 102 | 103 | # Then I see the appropriate JSON 104 | results = { 105 | "_geo_distance": { 106 | "location": [1.23, 3.45], 107 | "order": "asc" 108 | } 109 | } 110 | 111 | homogeneous(t, results) 112 | -------------------------------------------------------------------------------- /tests/functional/test_geo_distance.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | from __future__ import unicode_literals 3 | 4 | from sure import scenario 5 | 6 | from pyeqs import QuerySet 7 | from pyeqs.dsl import GeoDistance 8 | from tests.helpers import prepare_data, cleanup_data, add_document 9 | 10 | 11 | @scenario(prepare_data, cleanup_data) 12 | def test_geo_distance_search_dict(context): 13 | """ 14 | Search with geo distance filter with dictionary 15 | """ 16 | # When create a queryset 17 | t = QuerySet("localhost", index="foo") 18 | 19 | # And there are records 20 | add_document("foo", {"location": {"lat": 1.1, "lon": 2.1}}) 21 | add_document("foo", {"location": {"lat": 40.1, "lon": 80.1}}) 22 | 23 | # And I filter for distance 24 | t.filter(GeoDistance({"lat": 1.0, "lon": 2.0}, "20mi")) 25 | results = t[0:10] 26 | 27 | # Then I get a the expected results 28 | len(results).should.equal(1) 29 | 30 | 31 | @scenario(prepare_data, cleanup_data) 32 | def test_geo_distance_search_string(context): 33 | """ 34 | Search with geo distance filter with string 35 | """ 36 | # When create a queryset 37 | t = QuerySet("localhost", index="foo") 38 | 39 | # And there are records 40 | add_document("foo", {"location": {"lat": 1.1, "lon": 2.1}}) 41 | add_document("foo", {"location": {"lat": 40.1, "lon": 80.1}}) 42 | 43 | # And I filter for distance 44 | t.filter(GeoDistance("1.0,2.0", "20mi")) 45 | results = t[0:10] 46 | 47 | # Then I get a the expected results 48 | len(results).should.equal(1) 49 | 50 | 51 | @scenario(prepare_data, cleanup_data) 52 | def test_geo_distance_search_array(context): 53 | """ 54 | Search with geo distance filter with array 55 | """ 56 | # When create a queryset 57 | t = QuerySet("localhost", index="foo") 58 | 59 | # And there are records 60 | add_document("foo", {"location": {"lat": 1.1, "lon": 2.1}}) 61 | add_document("foo", {"location": {"lat": 40.1, "lon": 80.1}}) 62 | 63 | # And I filter for distance 64 | t.filter(GeoDistance([2.0, 1.0], "20mi")) 65 | results = t[0:10] 66 | 67 | # Then I get a the expected results 68 | len(results).should.equal(1) 69 | 70 | 71 | @scenario(prepare_data, cleanup_data) 72 | def test_geo_distance_search_with_field_name(context): 73 | """ 74 | Search with geo distance filter with field_name 75 | """ 76 | # When create a queryset 77 | t = QuerySet("localhost", index="foo") 78 | 79 | # And there are records 80 | add_document("foo", {"foo_loc": {"lat": 1.1, "lon": 2.1}}) 81 | add_document("foo", {"foo_loc": {"lat": 40.1, "lon": 80.1}}) 82 | 83 | # And I filter for distance 84 | t.filter(GeoDistance({"lat": 1.0, "lon": 2.0}, "20mi", field_name="foo_loc")) 85 | results = t[0:10] 86 | 87 | # Then I get a the expected results 88 | len(results).should.equal(1) 89 | -------------------------------------------------------------------------------- /tests/functional/test_range.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | from __future__ import unicode_literals 3 | 4 | from sure import scenario 5 | 6 | from pyeqs import QuerySet 7 | from pyeqs.dsl import Range 8 | from tests.helpers import prepare_data, cleanup_data, add_document 9 | 10 | 11 | @scenario(prepare_data, cleanup_data) 12 | def test_search_with_lte_range(context): 13 | """ 14 | Search with lte range filter 15 | """ 16 | # When create a query block 17 | t = QuerySet("localhost", index="foo") 18 | 19 | # And there are records 20 | add_document("foo", {"bar": 1}) 21 | add_document("foo", {"bar": 2}) 22 | add_document("foo", {"bar": 3}) 23 | 24 | # And I add a range filter 25 | _type = Range("bar", lte=1) 26 | t.filter(_type) 27 | results = t[0:10] 28 | 29 | # Then my results only have that type 30 | len(results).should.equal(1) 31 | results[0]["_source"]["bar"].should.equal(1) 32 | 33 | 34 | @scenario(prepare_data, cleanup_data) 35 | def test_search_with_lt_range(context): 36 | """ 37 | Search with lt range filter 38 | """ 39 | # When create a query block 40 | t = QuerySet("localhost", index="foo") 41 | 42 | # And there are records 43 | add_document("foo", {"bar": 1}) 44 | add_document("foo", {"bar": 2}) 45 | add_document("foo", {"bar": 3}) 46 | 47 | # And I add a range filter 48 | _type = Range("bar", lt=2) 49 | t.filter(_type) 50 | results = t[0:10] 51 | 52 | # Then my results only have that type 53 | len(results).should.equal(1) 54 | results[0]["_source"]["bar"].should.equal(1) 55 | 56 | 57 | @scenario(prepare_data, cleanup_data) 58 | def test_search_with_gte_range(context): 59 | """ 60 | Search with gte range filter 61 | """ 62 | # When create a query block 63 | t = QuerySet("localhost", index="foo") 64 | 65 | # And there are records 66 | add_document("foo", {"bar": 1}) 67 | add_document("foo", {"bar": 2}) 68 | add_document("foo", {"bar": 3}) 69 | 70 | # And I add a range filter 71 | _type = Range("bar", gte=3) 72 | t.filter(_type) 73 | results = t[0:10] 74 | 75 | # Then my results only have that type 76 | len(results).should.equal(1) 77 | results[0]["_source"]["bar"].should.equal(3) 78 | 79 | 80 | @scenario(prepare_data, cleanup_data) 81 | def test_search_with_gt_range(context): 82 | """ 83 | Search with gt range filter 84 | """ 85 | # When create a query block 86 | t = QuerySet("localhost", index="foo") 87 | 88 | # And there are records 89 | add_document("foo", {"bar": 1}) 90 | add_document("foo", {"bar": 2}) 91 | add_document("foo", {"bar": 3}) 92 | 93 | # And I add a range filter 94 | _type = Range("bar", gt=2) 95 | t.filter(_type) 96 | results = t[0:10] 97 | 98 | # Then my results only have that type 99 | len(results).should.equal(1) 100 | results[0]["_source"]["bar"].should.equal(3) 101 | -------------------------------------------------------------------------------- /tests/helpers.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | from __future__ import unicode_literals 3 | 4 | import json 5 | import os 6 | 7 | from distutils.version import StrictVersion 8 | from nose.plugins.skip import SkipTest 9 | 10 | from elasticsearch import ( 11 | Elasticsearch 12 | ) 13 | 14 | ELASTICSEARCH_URL = "localhost" 15 | conn = Elasticsearch(ELASTICSEARCH_URL) 16 | index_name = "foo" 17 | default_doc_type = "my_doc_type" 18 | 19 | 20 | def homogeneous(a, b): 21 | json.dumps(a, sort_keys=True).should.equal(json.dumps(b, sort_keys=True)) 22 | 23 | 24 | def heterogeneous(a, b): 25 | json.dumps(a, sort_keys=True).shouldnt.equal(json.dumps(b, sort_keys=True)) 26 | 27 | 28 | def add_document(index, document, **kwargs): 29 | kwargs = _set_doc_type(kwargs) 30 | conn.create(index=index, body=document, refresh=True, **kwargs) 31 | 32 | 33 | def clean_elasticsearch(context): 34 | _delete_es_index(index_name) 35 | 36 | 37 | def prepare_elasticsearch(context): 38 | clean_elasticsearch(context) 39 | _create_foo_index() 40 | conn.cluster.health(wait_for_status='yellow', index=index_name) 41 | 42 | 43 | def _create_foo_index(): 44 | mapping = _get_mapping(index=index_name) 45 | conn.indices.create(index=index_name, ignore=400, body=mapping) 46 | conn.indices.refresh(index=index_name) 47 | 48 | 49 | def _delete_es_index(index): 50 | conn.indices.delete(index=index, ignore=[400, 404]) 51 | 52 | 53 | def _get_mapping(index, **kwargs): 54 | kwargs = _set_doc_type(kwargs) 55 | doc_type = kwargs['doc_type'] 56 | mapping = { 57 | "mappings": { 58 | doc_type: { 59 | "properties": { 60 | "location": { 61 | "type": "geo_point" 62 | }, 63 | "foo_loc": { 64 | "type": "geo_point" 65 | }, 66 | "child": { 67 | "type": "nested" 68 | } 69 | } 70 | } 71 | } 72 | } 73 | return mapping 74 | 75 | 76 | def _set_doc_type(kwargs): 77 | if "doc_type" not in kwargs: 78 | # Allow overriding doc type defaults 79 | kwargs["doc_type"] = default_doc_type 80 | return kwargs 81 | 82 | 83 | prepare_data = [ 84 | prepare_elasticsearch 85 | ] 86 | 87 | cleanup_data = [ 88 | clean_elasticsearch 89 | ] 90 | 91 | 92 | def version_tuple(v): 93 | return tuple(map(int, (v.split(".")))) 94 | 95 | 96 | class requires_es_gte(object): 97 | """ 98 | Decorator for requiring Elasticsearch version 99 | greater than or equal to 'version' 100 | """ 101 | def __init__(self, version): 102 | self.version = version 103 | 104 | def __call__(self, test): 105 | es_version_string = os.environ.get("ES_VERSION", None) 106 | if es_version_string is None: # Skip check if we don't know our version 107 | return test 108 | es_version = StrictVersion(es_version_string) 109 | required = StrictVersion(self.version) 110 | if es_version >= required: 111 | return test 112 | raise SkipTest 113 | -------------------------------------------------------------------------------- /tests/unit/test_bool.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | from __future__ import unicode_literals 3 | 4 | from pyeqs import Bool 5 | from pyeqs.dsl import Term 6 | from tests.helpers import homogeneous 7 | 8 | 9 | def test_create_bool(): 10 | """ 11 | Create Bool Block 12 | """ 13 | # When I create a Bool block 14 | t = Bool() 15 | 16 | # Then I see the appropriate JSON 17 | results = { 18 | "bool": {} 19 | } 20 | 21 | homogeneous(t, results) 22 | 23 | 24 | def test_create_bool_with_must(): 25 | """ 26 | Create Bool Block with Must 27 | """ 28 | # When I create a Bool block 29 | t = Bool() 30 | 31 | # And add a 'must' condition with a Term 32 | t.must(Term("foo", "bar")) 33 | 34 | # Then I see the appropriate JSON 35 | results = { 36 | "bool": { 37 | "must": [ 38 | { 39 | "term": { 40 | "foo": "bar" 41 | } 42 | } 43 | ] 44 | } 45 | } 46 | 47 | homogeneous(t, results) 48 | 49 | 50 | def test_create_bool_with_must_not(): 51 | """ 52 | Create Bool Block with Must Not 53 | """ 54 | # When I create a Bool block 55 | t = Bool() 56 | 57 | # And add a 'must_not' condition with a Term 58 | t.must_not(Term("foo", "bar")) 59 | 60 | # Then I see the appropriate JSON 61 | results = { 62 | "bool": { 63 | "must_not": [ 64 | { 65 | "term": { 66 | "foo": "bar" 67 | } 68 | } 69 | ] 70 | } 71 | } 72 | 73 | homogeneous(t, results) 74 | 75 | 76 | def test_create_bool_with_should(): 77 | """ 78 | Create Bool Block with Should 79 | """ 80 | # When I create a Bool block 81 | t = Bool() 82 | 83 | # And add a 'should' condition with a Term 84 | t.should(Term("foo", "bar")) 85 | 86 | # Then I see the appropriate JSON 87 | results = { 88 | "bool": { 89 | "should": [ 90 | { 91 | "term": { 92 | "foo": "bar" 93 | } 94 | } 95 | ] 96 | } 97 | } 98 | 99 | homogeneous(t, results) 100 | 101 | 102 | def test_create_bool_with_multiple_clauses(): 103 | """ 104 | Create Bool Block with Multiple Clauses 105 | """ 106 | # When I create a Bool block 107 | t = Bool() 108 | 109 | # And add multiple conditions 110 | t.must_not(Term("foo", "foo"))\ 111 | .must(Term("bar", "bar"))\ 112 | .should(Term("baz", "baz"))\ 113 | .should(Term("foobar", "foobar")) 114 | 115 | # Then I see the appropriate JSON 116 | results = { 117 | "bool": { 118 | "must_not": [ 119 | { 120 | "term": { 121 | "foo": "foo" 122 | } 123 | } 124 | ], 125 | "must": [ 126 | { 127 | "term": { 128 | "bar": "bar" 129 | } 130 | } 131 | ], 132 | "should": [ 133 | { 134 | "term": { 135 | "baz": "baz" 136 | } 137 | }, 138 | { 139 | "term": { 140 | "foobar": "foobar" 141 | } 142 | }, 143 | ] 144 | } 145 | } 146 | 147 | homogeneous(t, results) 148 | -------------------------------------------------------------------------------- /tests/functional/test_score.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | from __future__ import unicode_literals 3 | 4 | from sure import scenario 5 | 6 | from pyeqs import QuerySet 7 | from pyeqs.dsl import ScriptScore, Exists 8 | from tests.helpers import prepare_data, cleanup_data, add_document 9 | 10 | 11 | @scenario(prepare_data, cleanup_data) 12 | def test_search_with_scoring(context): 13 | """ 14 | Search with custom scoring 15 | """ 16 | # When create a query block 17 | t = QuerySet("localhost", index="foo") 18 | 19 | # And there are records 20 | add_document("foo", {"bar": 1}) 21 | add_document("foo", {"bar": 2}) 22 | add_document("foo", {"bar": 3}) 23 | 24 | # And I add scoring 25 | score = ScriptScore("s = 0 + doc['bar'].value") 26 | t.score(score) 27 | results = t[0:10] 28 | 29 | # Then my results are scored correctly 30 | len(results).should.equal(3) 31 | results[0]["_source"]["bar"].should.equal(3) 32 | results[1]["_source"]["bar"].should.equal(2) 33 | results[2]["_source"]["bar"].should.equal(1) 34 | 35 | 36 | @scenario(prepare_data, cleanup_data) 37 | def test_search_with_scoring_and_lang(context): 38 | """ 39 | Search with custom scoring and language 40 | """ 41 | # When create a query block 42 | t = QuerySet("localhost", index="foo") 43 | 44 | # And there are records 45 | add_document("foo", {"bar": 1}) 46 | add_document("foo", {"bar": 2}) 47 | add_document("foo", {"bar": 3}) 48 | 49 | # And I add scoring with a language 50 | score = ScriptScore("s = 0 + doc['bar'].value", lang="mvel") 51 | t.score(score) 52 | results = t[0:10] 53 | 54 | # Then my results are scored correctly 55 | len(results).should.equal(3) 56 | results[0]["_source"]["bar"].should.equal(3) 57 | results[1]["_source"]["bar"].should.equal(2) 58 | results[2]["_source"]["bar"].should.equal(1) 59 | 60 | 61 | @scenario(prepare_data, cleanup_data) 62 | def test_search_with_scoring_and_params(context): 63 | """ 64 | Search with custom scoring and params 65 | """ 66 | # When create a query block 67 | t = QuerySet("localhost", index="foo") 68 | 69 | # And there are records 70 | add_document("foo", {"bar": 1}) 71 | add_document("foo", {"bar": 2}) 72 | add_document("foo", {"bar": 3}) 73 | 74 | # And I add scoring with params 75 | score = ScriptScore("s = custom_param + doc['bar'].value", params={"custom_param": 1}) 76 | t.score(score) 77 | results = t[0:10] 78 | 79 | # Then my results are scored correctly 80 | len(results).should.equal(3) 81 | results[0]["_source"]["bar"].should.equal(3) 82 | results[1]["_source"]["bar"].should.equal(2) 83 | results[2]["_source"]["bar"].should.equal(1) 84 | 85 | 86 | @scenario(prepare_data, cleanup_data) 87 | def test_search_multiple_scoring(context): 88 | """ 89 | Search with custom scoring and params 90 | """ 91 | # When create a query block 92 | t = QuerySet("localhost", index="foo") 93 | 94 | # And there are records 95 | add_document("foo", {"bar": 1, "baz": 4}) 96 | add_document("foo", {"bar": 1}) 97 | 98 | # And I add scoring with params 99 | score = ScriptScore("s = custom_param + doc['bar'].value", params={"custom_param": 1}) 100 | t.score(score) 101 | 102 | boost = { 103 | "boost_factor": "10", 104 | "filter": Exists("baz") 105 | } 106 | t.score(boost) 107 | results = t[0:10] 108 | 109 | # Then my results are scored correctly 110 | len(results).should.equal(2) 111 | results[0]["_source"]["baz"].should.equal(4) 112 | ("baz" in results[1]["_source"].keys()).should.be.false 113 | -------------------------------------------------------------------------------- /pyeqs/dsl/aggregations.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | from __future__ import unicode_literals, absolute_import 3 | 4 | 5 | class Aggregations(dict): 6 | 7 | def __init__(self, agg_name, field_name, metric, size=0, min_doc_count=1, 8 | order_type=None, order_dir="desc", filter_val=None, filter_name=None, 9 | global_name=None, nested_path=None, range_list=None, range_name=None, 10 | histogram_interval=None): 11 | super(Aggregations, self).__init__() 12 | self.agg_name = agg_name 13 | self.field_name = field_name 14 | self.metric = metric 15 | self.size = size 16 | self.min_doc_count = min_doc_count 17 | self.order_type = self._pick_order_type(order_type, histogram_interval) 18 | self.order_dir = order_dir 19 | self.filter_val = filter_val 20 | self.filter_name = filter_name 21 | self.global_name = global_name 22 | self.nested_path = nested_path 23 | self.range_list = range_list 24 | self.range_name = range_name 25 | self.interval = histogram_interval 26 | self._build_dict() 27 | 28 | def _build_dict(self): 29 | if self.nested_path: 30 | self[self.nested_path] = self._nesting() 31 | else: 32 | self[self.agg_name] = {self.metric: {"field": self.field_name}} 33 | if self.metric == "terms": 34 | self[self.agg_name][self.metric].update({ 35 | "size": self.size, 36 | "order": {self.order_type: self.order_dir}, 37 | "min_doc_count": self.min_doc_count 38 | }) 39 | 40 | if self.range_list: 41 | if not self.range_name: 42 | range_name = "{name}_ranges".format(name=self.field_name) 43 | else: 44 | range_name = self.range_name 45 | self[range_name] = {"range": { 46 | "field": self.field_name, 47 | "ranges": self._ranging() 48 | }} 49 | self.pop(self.agg_name) 50 | if self.interval: 51 | self[self.agg_name]["histogram"] = { 52 | "field": self.field_name, 53 | "interval": self.interval, 54 | "order": {self.order_type: self.order_dir}, 55 | "min_doc_count": self.min_doc_count 56 | } 57 | self[self.agg_name].pop(self.metric) 58 | elif self.filter_val and self.filter_name: 59 | self[self.filter_name] = {'filter': self.filter_val, 'aggregations': {}} 60 | self[self.filter_name]['aggregations'][self.agg_name] = self.pop(self.agg_name) 61 | elif self.global_name: 62 | self[self.global_name] = {"global": {}, "aggregations": {}} 63 | self[self.global_name]['aggregations'][self.agg_name] = self.pop(self.agg_name) 64 | 65 | def _nesting(self): 66 | nesting = { 67 | "nested": {"path": self.nested_path}, 68 | "aggregations": { 69 | self.agg_name: { 70 | self.metric: {"field": "{path}.{name}".format(path=self.nested_path, name=self.field_name)} 71 | }} 72 | } 73 | if self.metric == "terms": 74 | nesting["aggregations"][self.agg_name][self.metric].update({ 75 | "size": self.size, 76 | "order": {self.order_type: self.order_dir}, 77 | "min_doc_count": self.min_doc_count 78 | }) 79 | return nesting 80 | 81 | def _ranging(self): 82 | """ 83 | Should be a list of values to designate the buckets 84 | """ 85 | agg_ranges = [] 86 | for i, val in enumerate(self.range_list): 87 | if i == 0: 88 | agg_ranges.append({"to": val}) 89 | else: 90 | previous = self.range_list[i - 1] 91 | agg_ranges.append({"from": previous, "to": val}) 92 | 93 | if i + 1 == len(self.range_list): 94 | agg_ranges.append({"from": val}) 95 | return agg_ranges 96 | 97 | def _pick_order_type(self, order_type, hist): 98 | if order_type: 99 | return order_type 100 | elif hist: 101 | return "_key" 102 | else: 103 | return "_count" 104 | -------------------------------------------------------------------------------- /tests/unit/test_connection.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | from __future__ import unicode_literals 3 | 4 | import httpretty 5 | import json 6 | import sure 7 | 8 | from pyeqs import QuerySet, Filter 9 | from pyeqs.dsl import Term, Sort, ScriptScore 10 | from tests.helpers import homogeneous 11 | 12 | 13 | @httpretty.activate 14 | def test_create_queryset_with_host_string(): 15 | """ 16 | Create a queryset with a host given as a string 17 | """ 18 | # When create a queryset 19 | t = QuerySet("localhost", index="bar") 20 | 21 | # And I have records 22 | response = { 23 | "took": 1, 24 | "hits": { 25 | "total": 1, 26 | "max_score": 1, 27 | "hits": [ 28 | { 29 | "_index": "bar", 30 | "_type": "baz", 31 | "_id": "1", 32 | "_score": 10, 33 | "_source": { 34 | "foo": "bar" 35 | }, 36 | "sort": [ 37 | 1395687078000 38 | ] 39 | } 40 | ] 41 | } 42 | } 43 | 44 | httpretty.register_uri(httpretty.GET, "http://localhost:9200/bar/_search", 45 | body=json.dumps(response), 46 | content_type="application/json") 47 | 48 | # When I run a query 49 | results = t[0:1] 50 | 51 | # Then I see the response. 52 | len(results).should.equal(1) 53 | 54 | 55 | @httpretty.activate 56 | def test_create_queryset_with_host_dict(): 57 | """ 58 | Create a queryset with a host given as a dict 59 | """ 60 | # When create a queryset 61 | connection_info = {"host": "localhost", "port": 8080} 62 | t = QuerySet(connection_info, index="bar") 63 | 64 | # And I have records 65 | good_response = { 66 | "took": 1, 67 | "hits": { 68 | "total": 1, 69 | "max_score": 1, 70 | "hits": [ 71 | { 72 | "_index": "bar", 73 | "_type": "baz", 74 | "_id": "1", 75 | "_score": 10, 76 | "_source": { 77 | "foo": "bar" 78 | }, 79 | "sort": [ 80 | 1395687078000 81 | ] 82 | } 83 | ] 84 | } 85 | } 86 | 87 | bad_response = { 88 | "took": 1, 89 | "hits": { 90 | "total": 0, 91 | "max_score": None, 92 | "hits": [] 93 | } 94 | } 95 | httpretty.register_uri(httpretty.GET, "http://localhost:9200/bar/_search", 96 | body=json.dumps(bad_response), 97 | content_type="application/json") 98 | 99 | httpretty.register_uri(httpretty.GET, "http://localhost:8080/bar/_search", 100 | body=json.dumps(good_response), 101 | content_type="application/json") 102 | 103 | # When I run a query 104 | results = t[0:1] 105 | 106 | # Then I see the response. 107 | len(results).should.equal(1) 108 | results[0]["_source"]["foo"].should.equal("bar") 109 | 110 | 111 | @httpretty.activate 112 | def test_create_queryset_with_host_list(): 113 | """ 114 | Create a queryset with a host given as a list 115 | """ 116 | # When create a queryset 117 | connection_info = [{"host": "localhost", "port": 8080}] 118 | t = QuerySet(connection_info, index="bar") 119 | 120 | # And I have records 121 | good_response = { 122 | "took": 1, 123 | "hits": { 124 | "total": 1, 125 | "max_score": 1, 126 | "hits": [ 127 | { 128 | "_index": "bar", 129 | "_type": "baz", 130 | "_id": "1", 131 | "_score": 10, 132 | "_source": { 133 | "foo": "bar" 134 | }, 135 | "sort": [ 136 | 1395687078000 137 | ] 138 | } 139 | ] 140 | } 141 | } 142 | 143 | bad_response = { 144 | "took": 1, 145 | "hits": { 146 | "total": 0, 147 | "max_score": None, 148 | "hits": [] 149 | } 150 | } 151 | httpretty.register_uri(httpretty.GET, "http://localhost:9200/bar/_search", 152 | body=json.dumps(bad_response), 153 | content_type="application/json") 154 | 155 | httpretty.register_uri(httpretty.GET, "http://localhost:8080/bar/_search", 156 | body=json.dumps(good_response), 157 | content_type="application/json") 158 | 159 | # When I run a query 160 | results = t[0:1] 161 | 162 | # Then I see the response. 163 | len(results).should.equal(1) 164 | results[0]["_source"]["foo"].should.equal("bar") 165 | -------------------------------------------------------------------------------- /tests/functional/test_sorting.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | from __future__ import unicode_literals 3 | 4 | from sure import scenario 5 | 6 | from pyeqs import QuerySet 7 | from pyeqs.dsl import Sort 8 | from tests.helpers import prepare_data, cleanup_data, add_document 9 | 10 | 11 | @scenario(prepare_data, cleanup_data) 12 | def test_search_with_asc_sorting(context): 13 | """ 14 | Search with ascending sorting 15 | """ 16 | # When create a query block 17 | t = QuerySet("localhost", index="foo") 18 | 19 | # And there are records 20 | add_document("foo", {"bar": 1}) 21 | add_document("foo", {"bar": 2}) 22 | add_document("foo", {"bar": 3}) 23 | 24 | # And I add sorting 25 | s = Sort("bar", order="asc") 26 | t.order_by(s) 27 | results = t[0:10] 28 | 29 | # Then my results have the proper sorting 30 | results[0]["_source"]["bar"].should.equal(1) 31 | results[1]["_source"]["bar"].should.equal(2) 32 | results[2]["_source"]["bar"].should.equal(3) 33 | 34 | 35 | @scenario(prepare_data, cleanup_data) 36 | def test_search_with_desc_sorting(context): 37 | """ 38 | Search with descending sorting 39 | """ 40 | # When create a query block 41 | t = QuerySet("localhost", index="foo") 42 | 43 | # And there are records 44 | add_document("foo", {"bar": 1}) 45 | add_document("foo", {"bar": 2}) 46 | add_document("foo", {"bar": 3}) 47 | 48 | # And I add sorting 49 | s = Sort("bar", order="desc") 50 | t.order_by(s) 51 | results = t[0:10] 52 | 53 | # Then my results have the proper sorting 54 | results[0]["_source"]["bar"].should.equal(3) 55 | results[1]["_source"]["bar"].should.equal(2) 56 | results[2]["_source"]["bar"].should.equal(1) 57 | 58 | 59 | @scenario(prepare_data, cleanup_data) 60 | def test_search_with_mode_sorting(context): 61 | """ 62 | Search with descending sorting and mode 63 | """ 64 | # When create a query block 65 | t = QuerySet("localhost", index="foo") 66 | 67 | # And there are records 68 | add_document("foo", {"bar": [1, 10]}) 69 | add_document("foo", {"bar": [2, 10]}) 70 | add_document("foo", {"bar": [3, 10]}) 71 | 72 | # And I add sorting 73 | s = Sort("bar", order="desc", mode="min") 74 | t.order_by(s) 75 | results = t[0:10] 76 | 77 | # Then my results have the proper sorting 78 | results[0]["_source"]["bar"].should.equal([3, 10]) 79 | results[1]["_source"]["bar"].should.equal([2, 10]) 80 | results[2]["_source"]["bar"].should.equal([1, 10]) 81 | 82 | 83 | @scenario(prepare_data, cleanup_data) 84 | def test_search_with_multiple_sorts(context): 85 | """ 86 | Search with multiple sorts 87 | """ 88 | # When create a query block 89 | t = QuerySet("localhost", index="foo") 90 | 91 | # And there are records 92 | add_document("foo", {"bar": 10, "baz": 1}) 93 | add_document("foo", {"bar": 10, "baz": 2}) 94 | add_document("foo", {"bar": 10, "baz": 3}) 95 | 96 | # And I add sorting 97 | first_sort = Sort("bar", order="asc") 98 | second_sort = Sort("baz", order="asc") 99 | t.order_by(first_sort) 100 | t.order_by(second_sort) 101 | results = t[0:10] 102 | 103 | # Then my results have the proper sorting 104 | results[0]["_source"]["baz"].should.equal(1) 105 | results[1]["_source"]["baz"].should.equal(2) 106 | results[2]["_source"]["baz"].should.equal(3) 107 | 108 | 109 | @scenario(prepare_data, cleanup_data) 110 | def test_search_with_missing_sort(context): 111 | """ 112 | Search with 'missing' sort 113 | """ 114 | # When create a query block 115 | t = QuerySet("localhost", index="foo") 116 | 117 | # And there are records 118 | add_document("foo", {"bar": 10, "baz": 1}) 119 | add_document("foo", {"bar": 10, "baz": 2}) 120 | add_document("foo", {"bar": 10, "baz": None}) 121 | 122 | # And I add sorting 123 | sort = Sort("baz", order="asc", missing='_last') 124 | t.order_by(sort) 125 | results = t[0:10] 126 | 127 | # Then my results have the proper sorting 128 | results[0]["_source"]["baz"].should.equal(1) 129 | results[1]["_source"]["baz"].should.equal(2) 130 | results[2]["_source"]["baz"].should.equal(None) 131 | 132 | 133 | @scenario(prepare_data, cleanup_data) 134 | def test_search_with_location_sort(context): 135 | """ 136 | Search with location sort 137 | """ 138 | # When create a query block 139 | t = QuerySet("localhost", index="foo") 140 | 141 | # And there are locaiton records 142 | add_document("foo", {"baz": 1, "location": {"lat": 40.0, "lon": 70.0}}) 143 | add_document("foo", {"baz": 2, "location": {"lat": 40.0, "lon": 75.0}}) 144 | 145 | # And I add sorting 146 | sort = Sort("location", order="asc", location=[71.0, 40.0]) 147 | t.order_by(sort) 148 | results = t[0:10] 149 | 150 | # Then my results have the proper sorting 151 | results[0]["_source"]["baz"].should.equal(1) 152 | results[1]["_source"]["baz"].should.equal(2) 153 | -------------------------------------------------------------------------------- /pyeqs/query_builder.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | from __future__ import unicode_literals, absolute_import 3 | 4 | from . import Filter 5 | from copy import deepcopy 6 | from six import string_types 7 | from pyeqs.dsl import MatchAll, QueryString, ScriptScore 8 | 9 | 10 | class QueryBuilder(object): 11 | 12 | def __init__(self, query_string=None): 13 | super(QueryBuilder, self).__init__() 14 | self._query = {} 15 | self._query_dsl = {} 16 | self._query_string = query_string 17 | self._filtered = False 18 | self._filter_dsl = None 19 | self._scored = False 20 | self._score_dsl = None 21 | self._min_score = None 22 | self._track_scores = False 23 | self._sorted = False 24 | self._sorting = None 25 | self._fields = [] 26 | self._aggregated = False 27 | self._aggregation_dsl = None 28 | self._build_query() 29 | 30 | def _build_query(self): 31 | """ 32 | Build the base query dictionary 33 | """ 34 | if isinstance(self._query_string, QueryString): 35 | self._query_dsl = self._query_string 36 | elif isinstance(self._query_string, string_types): 37 | self._query_dsl = QueryString(self._query_string) 38 | else: 39 | self._query_dsl = MatchAll() 40 | 41 | def _build_filtered_query(self, f, operator): 42 | """ 43 | Create the root of the filter tree 44 | """ 45 | self._filtered = True 46 | if isinstance(f, Filter): 47 | filter_object = f 48 | else: 49 | filter_object = Filter(operator).filter(f) 50 | self._filter_dsl = filter_object 51 | 52 | def filter(self, f, operator="and"): 53 | """ 54 | Add a filter to the query 55 | 56 | Takes a Filter object, or a filterable DSL object. 57 | """ 58 | if self._filtered: 59 | self._filter_dsl.filter(f) 60 | else: 61 | self._build_filtered_query(f, operator) 62 | return self 63 | 64 | def _build_sorted_query(self): 65 | self._sorted = True 66 | self._sorting = [] 67 | 68 | def sort(self, sorting): 69 | if not self._sorted: 70 | self._build_sorted_query() 71 | self._sorting.append(sorting) 72 | return self 73 | 74 | def score(self, scoring_block, boost_mode="replace", score_mode="multiply", min_score=None, track_scores=False): 75 | if not self._scored: 76 | self._scored = True 77 | self._score_dsl = { 78 | "function_score": { 79 | "functions": [], 80 | "boost_mode": boost_mode, 81 | "score_mode": score_mode 82 | } 83 | } 84 | if isinstance(scoring_block, ScriptScore): 85 | self._score_dsl["function_score"]["functions"].append({"script_score": scoring_block}) 86 | else: 87 | self._score_dsl["function_score"]["functions"].append(scoring_block) 88 | if min_score is not None: 89 | self._min_score = min_score 90 | if track_scores: 91 | self._track_scores = track_scores 92 | return self 93 | 94 | def _build_aggregation_query(self, aggregation): 95 | self._aggregated = True 96 | return aggregation 97 | 98 | def aggregate(self, aggregation): 99 | if self._aggregated: 100 | self._aggregation_dsl.update(aggregation) 101 | else: 102 | self._aggregation_dsl = self._build_aggregation_query(aggregation) 103 | return self 104 | 105 | def fields(self, fields): 106 | self._fields.append(fields) 107 | 108 | def _finalize_query(self): 109 | query = { 110 | "query": self._query_dsl 111 | } 112 | if self._filtered: 113 | filtered_query = { 114 | "query": { 115 | "filtered": { 116 | "filter": self._filter_dsl 117 | } 118 | } 119 | } 120 | filtered_query["query"]["filtered"]["query"] = deepcopy(query["query"]) 121 | query = filtered_query 122 | 123 | if self._scored: 124 | scored_query = { 125 | "query": self._score_dsl 126 | } 127 | if self._min_score is not None: 128 | scored_query["min_score"] = self._min_score 129 | if self._track_scores: 130 | scored_query["track_scores"] = self._track_scores 131 | scored_query["query"]["function_score"]["query"] = deepcopy(query["query"]) 132 | query = scored_query 133 | 134 | if self._sorted: 135 | query["sort"] = self._sorting 136 | 137 | if self._fields: 138 | query["fields"] = self._fields 139 | 140 | if self._aggregated: 141 | query["aggregations"] = self._aggregation_dsl 142 | 143 | return query 144 | -------------------------------------------------------------------------------- /pyeqs/queryset.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | from __future__ import unicode_literals, absolute_import 3 | 4 | from copy import deepcopy 5 | 6 | from elasticsearch import Elasticsearch 7 | from six import string_types 8 | from . import QueryBuilder 9 | 10 | 11 | class QuerySet(object): 12 | 13 | def __init__(self, host, query=None, index=''): 14 | self._host = host 15 | self._index = index 16 | if isinstance(query, QueryBuilder): 17 | self._q = query 18 | else: 19 | self._q = QueryBuilder(query_string=query) 20 | self._wrappers = [] 21 | self._post_query_actions = [] 22 | self._count = None 23 | self._max_score = None 24 | self._conn = None 25 | self._finalized_query = None 26 | self._aggregations = None 27 | # Caching 28 | self._cache = None 29 | self._retrieved = 0 30 | self._per_request = 10 31 | 32 | def _clone(self): 33 | klass = self.__class__ 34 | query = deepcopy(self._q) 35 | clone = klass(self._host, query=query, index=self._index) 36 | clone._conn = self._conn 37 | return clone 38 | 39 | @property 40 | def objects(self): 41 | return self._clone() 42 | 43 | @property 44 | def _query(self): 45 | return self._q._finalize_query() 46 | 47 | def filter(self, f, operator="and"): 48 | self._q.filter(f, operator=operator) 49 | return self 50 | 51 | def score(self, scoring_block, boost_mode="replace", score_mode="multiply", min_score=None, track_scores=False): 52 | self._q.score(scoring_block, boost_mode=boost_mode, score_mode=score_mode, min_score=min_score, track_scores=track_scores) 53 | return self 54 | 55 | def only(self, fields): 56 | self._q.fields(fields) 57 | return self 58 | 59 | def order_by(self, sorting): 60 | self._q.sort(sorting) 61 | return self 62 | 63 | def wrappers(self, wrapper): 64 | self._wrappers.append(wrapper) 65 | return self 66 | 67 | def post_query_actions(self, action): 68 | self._post_query_actions.append(action) 69 | return self 70 | 71 | def count(self): 72 | return self._count 73 | 74 | def max_score(self): 75 | return self._max_score 76 | 77 | def aggregations(self): 78 | return self._aggregations 79 | 80 | def aggregate(self, aggregation): 81 | self._q.aggregate(aggregation) 82 | return self 83 | 84 | def __next__(self): 85 | return self.next() # pragma: no cover 86 | 87 | def next(self): 88 | """ 89 | Provide iteration capabilities 90 | 91 | Use a small object cache for performance 92 | """ 93 | if not self._cache: 94 | self._cache = self._get_results() 95 | self._retrieved += len(self._cache) 96 | 97 | # If we don't have any other data to return, we just 98 | # stop the iteration. 99 | if not self._cache: 100 | raise StopIteration() 101 | 102 | # Consuming the cache and updating the "cursor" 103 | return self._cache.pop(0) 104 | 105 | def _get_results(self): 106 | start = self._retrieved 107 | if self._count is None: 108 | # Always perform the first query since we don't know the count 109 | upper_limit = start + 1 110 | else: 111 | upper_limit = self._count 112 | if start < upper_limit: 113 | end = self._retrieved + self._per_request 114 | results = self[start:end] 115 | return results 116 | return [] 117 | 118 | def __len__(self): 119 | return self.count() 120 | 121 | def __iter__(self): 122 | return self 123 | 124 | def __getitem__(self, val): 125 | """ 126 | Override __getitem__ so we can activate our ES call when we try to slice 127 | """ 128 | start = val.start 129 | end = val.stop 130 | raw_results = self._search(start, end) 131 | for action in self._post_query_actions: 132 | action(self, raw_results, start, end) 133 | results = raw_results["hits"]["hits"] 134 | for wrapper in self._wrappers: 135 | results = wrapper(results) 136 | if raw_results.get('aggregations'): 137 | self._aggregations = raw_results.get('aggregations') 138 | return results 139 | 140 | def _search(self, start, end): 141 | conn = self._get_connection() 142 | pagination_kwargs = self._get_pagination_kwargs(start, end) 143 | raw_results = conn.search(index=self._index, body=self._query, **pagination_kwargs) 144 | self._count = self._get_result_count(raw_results) 145 | self._max_score = self._get_max_score(raw_results) 146 | return raw_results 147 | 148 | def _get_result_count(self, results): 149 | return int(results["hits"]["total"]) 150 | 151 | def _get_max_score(self, results): 152 | return results.get("hits", {}).get("max_score") 153 | 154 | def _get_pagination_kwargs(self, start, end): 155 | size = end - start 156 | kwargs = { 157 | 'from_': start, # from is a reserved word, so we use 'from_' 158 | 'size': size 159 | } 160 | return kwargs 161 | 162 | def _get_connection(self): 163 | if not self._conn: 164 | host_connection_info = self._parse_host_connection_info(self._host) 165 | self._conn = Elasticsearch(host_connection_info) 166 | return self._conn 167 | 168 | def _parse_host_connection_info(self, host): 169 | if isinstance(host, string_types): 170 | return [{"host": host}] 171 | if isinstance(host, dict): 172 | return [host] 173 | # Default to just using what was given to us 174 | return host 175 | -------------------------------------------------------------------------------- /API_REFERENCE.md: -------------------------------------------------------------------------------- 1 | #### Simple querying 2 | 3 | ```python 4 | from pyeqs import QuerySet 5 | qs = QuerySet("127.0.0.1", index="my_index") 6 | print qs._query 7 | """ 8 | { 9 | 'query': { 10 | 'match_all': {} 11 | } 12 | } 13 | """ 14 | ``` 15 | 16 | ```python 17 | from pyeqs import QuerySet 18 | qs = QuerySet("127.0.0.1", query="cheese", index="my_index") 19 | print qs._query 20 | """ 21 | { 22 | 'query': { 23 | 'query_string': { 24 | 'query': 'cheese' 25 | } 26 | } 27 | } 28 | """ 29 | ``` 30 | 31 | #### Filtering 32 | 33 | ```python 34 | from pyeqs import QuerySet 35 | from pyeqs.dsl import Term, Type 36 | qs = QuerySet("127.0.0.1", index="my_index") 37 | qs.filter(Term("foo", "bar"), operator="or").filter(Type("baz")) 38 | print qs._query 39 | """ 40 | { 41 | 'query': { 42 | 'filtered': { 43 | 'filter': { 44 | 'or': [ 45 | { 46 | 'term': { 47 | 'foo': 'bar' 48 | } 49 | }, 50 | { 51 | 'type': { 52 | 'value': 'baz' 53 | } 54 | } 55 | ] 56 | }, 57 | 'query': { 58 | 'match_all': {} 59 | } 60 | } 61 | } 62 | } 63 | """ 64 | ``` 65 | 66 | #### Boolean Filters 67 | 68 | ```python 69 | from pyeqs import QuerySet, Bool 70 | from pyeqs.dsl import Sort 71 | qs = QuerySet("127.0.0.1", index="my_index") 72 | b = Bool() 73 | b.must(Term("foo", "bar")) 74 | qs.filter(b) 75 | ``` 76 | 77 | #### Sorting 78 | 79 | ```python 80 | from pyeqs import QuerySet 81 | from pyeqs.dsl import Sort 82 | qs = QuerySet("127.0.0.1", index="my_index") 83 | qs.order_by(Sort("_id", order="desc")) 84 | ``` 85 | 86 | #### Location Sorting 87 | 88 | Assuming a set of records with a `location` field mapped as a `geo_point`: 89 | 90 | ```python 91 | from pyeqs import QuerySet 92 | from pyeqs.dsl import Sort 93 | qs = QuerySet("127.0.0.1", index="my_index") 94 | qs.order_by(Sort("location", location=[40.0, 74.5])) 95 | ``` 96 | 97 | The parameter passed to the `location` kwarg can be any format that elasticsearch accepts. 98 | 99 | 100 | #### Scoring 101 | 102 | Single Scoring Functions 103 | 104 | ```python 105 | from pyeqs import QuerySet 106 | from pyeqs.dsl import ScriptScore 107 | qs = QuerySet("127.0.0.1", index="my_index") 108 | qs.score(ScriptScore("score = foo + bar;", lang="mvel", params={"bar": 1})) 109 | ``` 110 | 111 | Multiple Scoring Functions 112 | 113 | 114 | ```python 115 | from pyeqs import QuerySet 116 | from pyeqs.dsl import ScriptScore 117 | qs = QuerySet("127.0.0.1", index="my_index") 118 | qs.score(ScriptScore("score = foo + bar;", lang="mvel", params={"bar": 1})) 119 | qs.score({"boost": 10, "filter": {"term": {"foo": "bar"}}}) 120 | ``` 121 | 122 | #### Wrapping Results 123 | 124 | PyEQS allows you to transform the JSON results returned from Elasticsearch. This can be used to extract source fields, serialize objects, and perform additional cleanup inside the iterator abstraction. 125 | 126 | Wrapper functions have a simple interface. They should expect a list of results from Elasticsearch (`["hits"]["hits"]`) where each element is the dictionary representation of a record. 127 | 128 | ```python 129 | def id_wrapper(results): 130 | return map(lambda x: x['_id'], results) 131 | ``` 132 | 133 | You can have multiple wrappers on a PyEQS object, and they will be applied in the order they were applied to the queryset. Each wrapper will act on the output of the previous wrapper. The wrappers are stored in an array in `self._wrappers` if additional manipulation is required. 134 | 135 | ```python 136 | def int_wrapper(results): 137 | return map(int, results) 138 | ``` 139 | 140 | ```python 141 | from pyeqs import QuerySet 142 | from pyeqs.dsl import Term 143 | from wrappers import id_wrapper, int_wrapper 144 | qs = QuerySet("127.0.0.1", index="my_index") 145 | qs.filter(Term("foo", 1)) 146 | qs.wrapper(id_wrapper) 147 | qs.wrapper(int_wrapper) 148 | ``` 149 | 150 | #### Running Post Query Actions 151 | 152 | Post Query Actions are functions you can pass to PyEQS that can interact with all of the JSON returned from Elasticsearch (not just the hits). These functions should expect a simple method signature: 153 | 154 | ```python 155 | def simple_action(self, raw_results, start, stop) 156 | logger("Query Time: {}".format(raw_results['took'])) 157 | logger("Cache Size: {}".format(len(self._cache)) 158 | logger("Request Page Start: {}".format(start)) 159 | logger("Request Page Sop: {}".format(stop)) 160 | ``` 161 | 162 | Do not `post_query_actions` to modify the returned results (use wrappers). Instead, use it to attach logging and debugging info to your requests. 163 | 164 | ```python 165 | from pyeqs import QuerySet 166 | from pyeqs.dsl import Term 167 | from actions import simple_action 168 | qs = QuerySet("127.0.0.1", index="my_index") 169 | qs.filter(Term("foo", 1)) 170 | qs.post_query_actions(simple_action) 171 | ``` 172 | 173 | 174 | #### Limiting Returned Fields 175 | 176 | ```python 177 | from pyeqs import QuerySet 178 | qs = QuerySet("127.0.0.1", index="my_index") 179 | qs.only('_id') 180 | ``` 181 | 182 | #### Reusing Querysets 183 | 184 | ```python 185 | from pyeqs import QuerySet 186 | from pyeqs.dsl import Terms, Term 187 | qs = QuerySet("127.0.0.1", index="my_index") 188 | qs.filter(Terms("foo", ["bar", "baz"])) 189 | 190 | # Duplicate the Queryset and do more filters 191 | only_bar = qs.objects.filter(Term("foo", "bar")) 192 | 193 | only_baz = qs.objects.filter(Term("foo", "baz")) 194 | ``` 195 | 196 | #### Slicing Querysets 197 | 198 | ```python 199 | from pyeqs import QuerySet 200 | from pyeqs.dsl import Term 201 | qs = QuerySet("127.0.0.1", index="my_index") 202 | qs.filter(Term("foo", "bar")) 203 | results = qs[0:10] # Uses from/size in the background 204 | ``` 205 | 206 | #### Iterating over Quersets 207 | 208 | ```python 209 | from pyeqs import QuerySet 210 | from pyeqs.dsl import Term 211 | qs = QuerySet("127.0.0.1", index="my_index") 212 | qs.filter(Term("foo", "bar")) 213 | for result in qs: 214 | print result['_source'] 215 | # Builds a cache of 10 results at a time and iterates 216 | ``` 217 | 218 | #### Getting Counts 219 | 220 | ```python 221 | from pyeqs import QuerySet 222 | from pyeqs.dsl import Term 223 | qs = QuerySet("127.0.0.1", index="my_index") 224 | qs.filter(Term("foo", "bar")) 225 | qs.count() # None, since we haven't queried 226 | qs[0:10] 227 | qs.count() # Returns number of hits 228 | ``` 229 | 230 | #### Calculating Aggregations 231 | 232 | ```python 233 | from pyeqs import QuerySet 234 | from pyeqs.dsl import Aggregations 235 | qs = QuerySet("127.0.0.1", index="my_index") 236 | qs.aggregate(Aggregations(agg_name="foo", field_name="bar", metric="stats")) 237 | qs.aggregations() # None, since we haven't queried 238 | qs[0:10] 239 | qs.aggregations() # Returns the aggregation data requested 240 | ``` 241 | -------------------------------------------------------------------------------- /tests/functional/test_queryset.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | from __future__ import unicode_literals 3 | 4 | from sure import scenario 5 | 6 | from pyeqs import QuerySet 7 | from pyeqs.dsl import Term, ScriptScore, Sort 8 | from tests.helpers import prepare_data, cleanup_data, add_document 9 | 10 | 11 | @scenario(prepare_data, cleanup_data) 12 | def test_simple_search(context): 13 | """ 14 | Search with match_all query 15 | """ 16 | # When create a queryset 17 | t = QuerySet("localhost", index="foo") 18 | 19 | # And there are records 20 | add_document("foo", {"bar": "baz"}) 21 | 22 | # And I do a search 23 | results = t[0:1] 24 | 25 | # Then I get a the expected results 26 | len(results).should.equal(1) 27 | results[0]['_source'].should.equal({"bar": "baz"}) 28 | 29 | 30 | @scenario(prepare_data, cleanup_data) 31 | def test_string_search(context): 32 | """ 33 | Search with string query 34 | """ 35 | # When create a queryset 36 | t = QuerySet("localhost", query="cheese", index="foo") 37 | 38 | # And there are records 39 | add_document("foo", {"bar": "banana"}) 40 | add_document("foo", {"bar": "cheese"}) 41 | 42 | # And I do a search 43 | results = t[0:10] 44 | 45 | # Then I get a the expected results 46 | len(results).should.equal(1) 47 | results[0]['_source'].should.equal({"bar": "cheese"}) 48 | 49 | 50 | @scenario(prepare_data, cleanup_data) 51 | def test_search_with_filter(context): 52 | """ 53 | Search with match_all query and filter 54 | """ 55 | # When create a queryset 56 | t = QuerySet("localhost", index="foo") 57 | 58 | # And there are records 59 | add_document("foo", {"bar": "baz"}) 60 | add_document("foo", {"bar": "bazbaz"}) 61 | 62 | # And I do a search 63 | t.filter(Term("bar", "baz")) 64 | results = t[0:10] 65 | 66 | # Then I get a the expected results 67 | len(results).should.equal(1) 68 | results[0]['_source'].should.equal({"bar": "baz"}) 69 | 70 | 71 | @scenario(prepare_data, cleanup_data) 72 | def test_search_with_filter_and_scoring(context): 73 | """ 74 | Search with match_all query, filter and scoring 75 | """ 76 | # When create a queryset 77 | t = QuerySet("localhost", index="foo") 78 | 79 | # And there are records 80 | add_document("foo", {"bar": "baz", "scoring_field": 1}) 81 | add_document("foo", {"bar": "baz", "scoring_field": 2}) 82 | add_document("foo", {"bar": "bazbaz", "scoring_field": 3}) 83 | 84 | # And I do a search 85 | t.filter(Term("bar", "baz")) 86 | score = ScriptScore("final_score = 0 + doc['scoring_field'].value;") 87 | t.score(score) 88 | results = t[0:10] 89 | 90 | # Then I get a the expected results 91 | len(results).should.equal(2) 92 | results[0]['_source'].should.equal({"bar": "baz", "scoring_field": 2}) 93 | results[1]['_source'].should.equal({"bar": "baz", "scoring_field": 1}) 94 | 95 | 96 | @scenario(prepare_data, cleanup_data) 97 | def test_search_with_scoring_min_score_and_track_scores(context): 98 | """ 99 | Search with match_all query and scoring with min_score and track_scores 100 | """ 101 | # When create a queryset 102 | t = QuerySet("localhost", index="foo") 103 | 104 | # And there are records 105 | add_document("foo", {"bar": "baz", "scoring_field": 1}) 106 | add_document("foo", {"bar": "baz", "scoring_field": 2}) 107 | add_document("foo", {"bar": "baz", "scoring_field": 3}) 108 | 109 | # And I do a search 110 | score = ScriptScore("final_score = 0 + doc['scoring_field'].value;") 111 | t.score(score, min_score=2, track_scores=True) 112 | results = t[0:10] 113 | 114 | # Then I get a the expected results 115 | len(results).should.equal(2) 116 | results[0]['_source'].should.equal({"bar": "baz", "scoring_field": 3}) 117 | results[1]['_source'].should.equal({"bar": "baz", "scoring_field": 2}) 118 | 119 | 120 | @scenario(prepare_data, cleanup_data) 121 | def test_search_with_filter_and_scoring_and_sorting(context): 122 | """ 123 | Search with match_all query, filter, scoring, and sorting 124 | """ 125 | # When create a queryset 126 | t = QuerySet("localhost", index="foo") 127 | 128 | # And there are records 129 | add_document("foo", {"bar": "baz", "scoring_field": 0, "sorting_field": 30}) 130 | add_document("foo", {"bar": "baz", "scoring_field": 1, "sorting_field": 20}) 131 | add_document("foo", {"bar": "baz", "scoring_field": 2, "sorting_field": 10}) 132 | add_document("foo", {"bar": "bazbaz", "scoring_field": 3, "sorting_field": 0}) 133 | 134 | # And I do a search 135 | t.filter(Term("bar", "baz")) 136 | score = ScriptScore("final_score = 0 + doc['scoring_field'].value;") 137 | t.score(score) 138 | sorting = Sort("sorting_field", order="desc") 139 | t.order_by(sorting) 140 | results = t[0:10] 141 | 142 | # Then I get a the expected results 143 | len(results).should.equal(3) 144 | results[0]['_source'].should.equal({"bar": "baz", "scoring_field": 0, "sorting_field": 30}) 145 | results[1]['_source'].should.equal({"bar": "baz", "scoring_field": 1, "sorting_field": 20}) 146 | results[2]['_source'].should.equal({"bar": "baz", "scoring_field": 2, "sorting_field": 10}) 147 | 148 | 149 | @scenario(prepare_data, cleanup_data) 150 | def test_search_with_filter_and_scoring_and_sorting_and_fields(context): 151 | """ 152 | Search with match_all query, filter, scoring, sorting, and fields 153 | """ 154 | # When create a queryset 155 | t = QuerySet("localhost", index="foo") 156 | 157 | # And there are records 158 | add_document("foo", {"bar": "baz", "scoring_field": 0, "sorting_field": 30}) 159 | add_document("foo", {"bar": "baz", "scoring_field": 1, "sorting_field": 20}) 160 | add_document("foo", {"bar": "baz", "scoring_field": 2, "sorting_field": 10}) 161 | add_document("foo", {"bar": "bazbaz", "scoring_field": 3, "sorting_field": 0}) 162 | 163 | # And I do a search 164 | t.filter(Term("bar", "baz")) 165 | score = ScriptScore("final_score = 0 + doc['scoring_field'].value;") 166 | t.score(score) 167 | sorting = Sort("sorting_field", order="desc") 168 | t.order_by(sorting) 169 | t.only(["bar"]) 170 | results = t[0:10] 171 | 172 | # Then I get a the expected results 173 | len(results).should.equal(3) 174 | results[0]['fields'].should.equal({"bar": ["baz"]}) 175 | results[1]['fields'].should.equal({"bar": ["baz"]}) 176 | results[2]['fields'].should.equal({"bar": ["baz"]}) 177 | 178 | 179 | @scenario(prepare_data, cleanup_data) 180 | def test_wrappers(context): 181 | """ 182 | Search with wrapped match_all query 183 | """ 184 | # When create a queryset 185 | t = QuerySet("localhost", index="foo") 186 | 187 | # And there are records 188 | add_document("foo", {"bar": 1}) 189 | add_document("foo", {"bar": 2}) 190 | add_document("foo", {"bar": 3}) 191 | 192 | # And I do a search 193 | def wrapper_function(search_results): 194 | return list(map(lambda x: x['_source']['bar'] + 1, search_results)) 195 | t.wrappers(wrapper_function) 196 | t.order_by(Sort("bar", order="asc")) 197 | results = t[0:10] 198 | 199 | # Then I get a the expected results 200 | len(results).should.equal(3) 201 | results[0].should.equal(2) 202 | results[1].should.equal(3) 203 | results[2].should.equal(4) 204 | 205 | 206 | @scenario(prepare_data, cleanup_data) 207 | def test_search_as_queryset_with_filter(context): 208 | """ 209 | Search with match_all query and filter on a cloned queryset 210 | """ 211 | # When create a queryset 212 | t = QuerySet("localhost", index="foo") 213 | 214 | # And there are records 215 | add_document("foo", {"bar": "baz"}) 216 | add_document("foo", {"bar": "bazbaz"}) 217 | 218 | # And I do a filter on my new object 219 | my_search = t.objects.filter(Term("bar", "baz")) 220 | 221 | # And a different filter on my old object 222 | t.filter(Term("bar", "bazbaz")) 223 | 224 | # And I do a search 225 | results = my_search[0:10] 226 | 227 | # Then I get a the expected results 228 | len(results).should.equal(1) 229 | results[0]['_source'].should.equal({"bar": "baz"}) 230 | 231 | 232 | @scenario(prepare_data, cleanup_data) 233 | def test_search_with_iterator(context): 234 | """ 235 | Search using an iterator 236 | """ 237 | # When create a queryset 238 | t = QuerySet("localhost", index="foo") 239 | 240 | # And set iterator fetching to a small size 241 | t._per_request = 2 242 | 243 | # And there are records 244 | add_document("foo", {"bar": 0}) 245 | add_document("foo", {"bar": 1}) 246 | add_document("foo", {"bar": 2}) 247 | add_document("foo", {"bar": 3}) 248 | add_document("foo", {"bar": 4}) 249 | 250 | # And I do a filter on my new object 251 | 252 | # And a different filter on my old object 253 | t.order_by(Sort("bar", order="asc")) 254 | 255 | # Then I get the expected results 256 | for (counter, result) in enumerate(t): 257 | result['_source'].should.equal({"bar": counter}) 258 | 259 | len(t).should.equal(5) 260 | t.count().should.equal(5) 261 | t.max_score().should_not.be(None) 262 | 263 | 264 | @scenario(prepare_data, cleanup_data) 265 | def test_post_query_actions(context): 266 | """ 267 | Search with match_all query with post query actions 268 | """ 269 | # When create a queryset 270 | t = QuerySet("localhost", index="foo") 271 | 272 | # And there are records 273 | add_document("foo", {"bar": 1}) 274 | add_document("foo", {"bar": 2}) 275 | add_document("foo", {"bar": 3}) 276 | 277 | # And I have a post query action 278 | global my_global_var 279 | my_global_var = 1 280 | 281 | def action(self, results, start, stop): 282 | global my_global_var 283 | my_global_var += 1 284 | 285 | t.post_query_actions(action) 286 | 287 | # And I do a search 288 | t.order_by(Sort("bar", order="asc")) 289 | results = t[0:10] 290 | 291 | # Then I get a the expected results 292 | len(results).should.equal(3) 293 | results[0]["_source"]["bar"].should.equal(1) 294 | results[1]["_source"]["bar"].should.equal(2) 295 | results[2]["_source"]["bar"].should.equal(3) 296 | my_global_var.should.equal(2) 297 | -------------------------------------------------------------------------------- /tests/unit/test_aggregation.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | from __future__ import unicode_literals 3 | 4 | from pyeqs.dsl import Aggregations 5 | from tests.helpers import homogeneous 6 | 7 | 8 | def test_add_agg(): 9 | """ 10 | Create aggregations block 11 | """ 12 | # When add an agg block 13 | t = Aggregations("agg_name", "field_name", "metric") 14 | 15 | # Then I see correct json 16 | results = { 17 | "agg_name": { 18 | "metric": {"field": "field_name"} 19 | } 20 | } 21 | 22 | homogeneous(t, results) 23 | 24 | 25 | def test_add_agg_with_size(): 26 | """ 27 | Create aggregations block specifying size 28 | """ 29 | # When add a terms agg block w/ size 30 | t = Aggregations("agg_name", "field_name", "terms", size=1) 31 | 32 | # Then I see correct json 33 | results = { 34 | "agg_name": { 35 | "terms": { 36 | "field": "field_name", 37 | "order": {"_count": "desc"}, 38 | "min_doc_count": 1, 39 | "size": 1 40 | } 41 | } 42 | } 43 | 44 | homogeneous(t, results) 45 | 46 | 47 | def test_add_agg_with_order(): 48 | """ 49 | Create aggregations block specifying order type and direction 50 | """ 51 | # When add a terms agg block w/ size 52 | t = Aggregations("agg_name", "field_name", "terms", order_type="_term", order_dir="asc") 53 | 54 | # Then I see correct json 55 | results = { 56 | "agg_name": { 57 | "terms": { 58 | "field": "field_name", 59 | "order": {"_term": "asc"}, 60 | "min_doc_count": 1, 61 | "size": 0 62 | } 63 | } 64 | } 65 | 66 | homogeneous(t, results) 67 | 68 | 69 | def test_add_agg_with_min_doc_count(): 70 | """ 71 | Create aggregations block specifying the min_doc_count 72 | """ 73 | # When add a terms agg block w/ size 74 | t = Aggregations("agg_name", "field_name", "terms", min_doc_count=10) 75 | 76 | # Then I see correct json 77 | results = { 78 | "agg_name": { 79 | "terms": { 80 | "field": "field_name", 81 | "order": {"_count": "desc"}, 82 | "min_doc_count": 10, 83 | "size": 0 84 | } 85 | } 86 | } 87 | 88 | homogeneous(t, results) 89 | 90 | 91 | def test_add_agg_nested(): 92 | """ 93 | Create nested aggregations block 94 | """ 95 | # When add a nested_path with agg block 96 | t = Aggregations("agg_name", "field_name", "metric", nested_path="nested_doc") 97 | 98 | # The I see correct json 99 | results = { 100 | "nested_doc": { 101 | "nested": {"path": "nested_doc"}, 102 | "aggregations": { 103 | "agg_name": {"metric": {"field": "nested_doc.field_name"}}, 104 | } 105 | } 106 | } 107 | 108 | homogeneous(t, results) 109 | 110 | 111 | def test_add_agg_nested_with_size(): 112 | """ 113 | Create nested aggregations block specifying size 114 | """ 115 | # When add a nested_path with terms agg block w/ size 116 | t = Aggregations("agg_name", "field_name", "terms", size=1, 117 | nested_path="nested_doc") 118 | 119 | # The I see correct json 120 | results = { 121 | "nested_doc": { 122 | "nested": {"path": "nested_doc"}, 123 | "aggregations": { 124 | "agg_name": {"terms": { 125 | "field": "nested_doc.field_name", 126 | "order": {"_count": "desc"}, 127 | "min_doc_count": 1, 128 | "size": 1 129 | }} 130 | } 131 | } 132 | } 133 | 134 | homogeneous(t, results) 135 | 136 | 137 | def test_add_agg_nested_with_order(): 138 | """ 139 | Create nested aggregations block specifying order type and direction 140 | """ 141 | # When add a nested_path with terms agg block w/ size 142 | t = Aggregations("agg_name", "field_name", "terms", order_type="_term", order_dir="asc", 143 | nested_path="nested_doc") 144 | 145 | # The I see correct json 146 | results = { 147 | "nested_doc": { 148 | "nested": {"path": "nested_doc"}, 149 | "aggregations": { 150 | "agg_name": {"terms": { 151 | "field": "nested_doc.field_name", 152 | "order": {"_term": "asc"}, 153 | "min_doc_count": 1, 154 | "size": 0 155 | }} 156 | } 157 | } 158 | } 159 | 160 | homogeneous(t, results) 161 | 162 | 163 | def test_add_agg_nested_with_min_doc_count(): 164 | """ 165 | Create nested aggregations block specifying min_doc_count 166 | """ 167 | # When add a nested_path with terms agg block w/ size 168 | t = Aggregations("agg_name", "field_name", "terms", min_doc_count=10, 169 | nested_path="nested_doc") 170 | 171 | # The I see correct json 172 | results = { 173 | "nested_doc": { 174 | "nested": {"path": "nested_doc"}, 175 | "aggregations": { 176 | "agg_name": {"terms": { 177 | "field": "nested_doc.field_name", 178 | "order": {"_count": "desc"}, 179 | "min_doc_count": 10, 180 | "size": 0 181 | }} 182 | } 183 | } 184 | } 185 | 186 | homogeneous(t, results) 187 | 188 | 189 | def test_add_agg_filtered(): 190 | """ 191 | Create an aggregations block with filter 192 | """ 193 | # With a filter 194 | filter_value = {"filter_type": {"other_field": {"comparison": "value"}}} 195 | 196 | # When add a filtered agg block 197 | t = Aggregations("agg_name", "field_name", "metric", filter_val=filter_value, 198 | filter_name="filter_on_other") 199 | 200 | # Then I see correct json 201 | results = { 202 | "filter_on_other": { 203 | "filter": filter_value, 204 | "aggregations": { 205 | "agg_name": {"metric": {"field": "field_name"}} 206 | } 207 | } 208 | } 209 | 210 | homogeneous(t, results) 211 | 212 | 213 | def test_add_agg_global(): 214 | """ 215 | Create an aggregations block that is global 216 | """ 217 | # When add a global agg block 218 | t = Aggregations("agg_name", "field_name", "metric", global_name="global_agg") 219 | 220 | # Then I see correct json 221 | results = { 222 | "global_agg": { 223 | "global": {}, 224 | "aggregations": { 225 | "agg_name": {"metric": {"field": "field_name"}} 226 | } 227 | } 228 | } 229 | 230 | homogeneous(t, results) 231 | 232 | 233 | def test_add_agg_range(): 234 | """ 235 | Create an aggregations block for a range 236 | """ 237 | # When add an agg block w/ range 238 | range_list = [1, 2, 3] 239 | t = Aggregations("agg_name", "field_name", "metric", range_list=range_list, range_name="my_ranges") 240 | 241 | # Then I see the correct json 242 | results = { 243 | "my_ranges": { 244 | "range": { 245 | "field": "field_name", 246 | "ranges": [ 247 | {"to": 1}, 248 | {"from": 1, "to": 2}, 249 | {"from": 2, "to": 3}, 250 | {"from": 3} 251 | ] 252 | }} 253 | } 254 | 255 | homogeneous(t, results) 256 | 257 | # Also works without a given range_name 258 | t = Aggregations("agg_name", "field_name", "metric", range_list=range_list) 259 | 260 | # Then I see the correct json 261 | results = { 262 | "field_name_ranges": { 263 | "range": { 264 | "field": "field_name", 265 | "ranges": [ 266 | {"to": 1}, 267 | {"from": 1, "to": 2}, 268 | {"from": 2, "to": 3}, 269 | {"from": 3} 270 | ] 271 | }} 272 | } 273 | 274 | homogeneous(t, results) 275 | 276 | 277 | def test_add_agg_histogram(): 278 | """ 279 | Create an aggregations block w/ histogram intervals 280 | """ 281 | # Whan add an agg block w/ interval 282 | t = Aggregations("agg_name", "field_name", "metric", histogram_interval=20) 283 | 284 | # Then I see correct json 285 | results = { 286 | "agg_name": { 287 | "histogram": { 288 | "field": "field_name", 289 | "interval": 20, 290 | "order": {"_key": "desc"}, 291 | "min_doc_count": 1 292 | } 293 | } 294 | } 295 | 296 | homogeneous(t, results) 297 | 298 | 299 | def test_add_agg_histogram_with_order(): 300 | """ 301 | Create an aggregations block w/ histogram intervals and order type/direction 302 | """ 303 | # Whan add an agg block w/ interval 304 | t = Aggregations("agg_name", "field_name", "metric", histogram_interval=20, 305 | order_type="_count", order_dir="asc") 306 | 307 | # Then I see correct json 308 | results = { 309 | "agg_name": { 310 | "histogram": { 311 | "field": "field_name", 312 | "interval": 20, 313 | "order": {"_count": "asc"}, 314 | "min_doc_count": 1 315 | } 316 | } 317 | } 318 | 319 | homogeneous(t, results) 320 | 321 | 322 | def test_add_agg_histogram_with_min_doc_count(): 323 | """ 324 | Create an aggregations block w/ histogram intervals and min_doc_count 325 | """ 326 | # Whan add an agg block w/ interval 327 | t = Aggregations("agg_name", "field_name", "metric", histogram_interval=20, 328 | min_doc_count=10) 329 | 330 | # Then I see correct json 331 | results = { 332 | "agg_name": { 333 | "histogram": { 334 | "field": "field_name", 335 | "interval": 20, 336 | "order": {"_key": "desc"}, 337 | "min_doc_count": 10 338 | } 339 | } 340 | } 341 | 342 | homogeneous(t, results) 343 | -------------------------------------------------------------------------------- /pre-commit: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | import os 4 | import re 5 | import shlex 6 | import subprocess 7 | import sys 8 | 9 | os.environ['TESTING'] = "1" 10 | 11 | 12 | def wrap_text_with(color): 13 | return lambda msg: '{}{}{}'.format(color, msg, NC) 14 | 15 | HIGHLIGHT = '\033[1;37m' 16 | GREEN = '\033[1;32m' 17 | RED = '\033[1;31m' 18 | NC = '\033[0m' 19 | 20 | ok = wrap_text_with(GREEN) 21 | error = wrap_text_with(RED) 22 | highlight = wrap_text_with(HIGHLIGHT) 23 | 24 | CHECKS = [ 25 | # Pre-commit hooks 26 | # 27 | # It's a list of dictionaries containing instructions of how to 28 | # perform certain checks 29 | 30 | # There are plenty of examples below but it's worth mentioning the 31 | # command paramenter features specific to python files: 32 | 33 | # ~~~~~~ 34 | # 'command': '/path/to/bin/your-script {first_file_name} {filenames} {module_name}' 35 | # 36 | # WHERE: 37 | # 38 | # *first_file_name* the name of the first relevant python module 39 | # 40 | # *filenames*, a transitional alias to the 41 | # `file_name` parameter. 42 | # 43 | # *module_name*, the module name extracted from the 44 | # pattern. 45 | # 46 | # ...................................................................................................... 47 | # 48 | # CHECKS 49 | 50 | # { 51 | # # Test coverage hook 52 | # # 53 | # # Forbids commit if coverage is under 100% 54 | # # 55 | # # It might sound very restrictive but it actually makes sense 56 | # # under the following circumstances: 57 | # # 58 | # # * We will enforce 100% of unit test coverage starting from 59 | # # the smallest apps/libraries 60 | # # 61 | # # * The trigger only happens when there is a change in a unit 62 | # # test (in the future we could also have a django setting that 63 | # # lists mature apps that can enforce full coverage ALWAYS.) 64 | # # 65 | # # 66 | # 67 | # 68 | # 'output': 'Checking test coverage of module {module_name}...', 69 | # 'command': ('python scripts/coverage-check.py ' 70 | # '--module-name {module_name} ' 71 | # '--module-path {app_package_path} ' 72 | # '--module-path {app_module_path} ' 73 | # '--file-name {first_file_name} '), # 2> /dev/null 74 | # 'match_files': ['.*tests.unit.*?test\w*.*?\.py$'], 75 | # 'error_message': ( 76 | # 'Not enough coverage, \033[35mbaby\033[0n' 77 | # ) 78 | # }, 79 | { 80 | 'output': 'Checking for ipdbs and pdbs...', 81 | 'command': 'grep -EHIn "import i?pdb" "{file_name}"', 82 | 'match_files': ['.*\.py$'], 83 | 'ignore_noqa_lines': True, 84 | }, 85 | { 86 | 'output': 'Checking for print statements...', 87 | 'command': 'grep -HIn "^ *print \|^ *print(" "{file_name}"', 88 | 'match_files': ['.*\.py$'], 89 | 'ignore_files': ['.*migrations.*', '.*management/commands.*', '.*manage.py$', 90 | '^scripts/.*', '^gists/.*', '^terrain/.*', '^fabfile.*', '^conf/.*',], 91 | 'ignore_noqa_lines': True, 92 | }, 93 | { 94 | 'output': 'Checking for "import settings"...', 95 | 'command': 'grep -HIn "^import settings" "{file_name}"', 96 | 'match_files': ['.*\.py$'], 97 | 'error_message': ( 98 | "Did you mean 'from django.conf import settings'?" 99 | ) 100 | }, 101 | { 102 | 'output': 'Checking for "assert_called_once"...', 103 | 'command': 'grep -HIn "assert_called_once[(]" "{file_name}"', 104 | 'match_files': ['.*\.py$'], 105 | }, 106 | { 107 | 'output': 'Checking for unhandled merges...', 108 | 'command': 'grep -EHInr \'^(([<>]{{7}}\s.+)|(={{7}}))$\' "{file_name}"', 109 | }, 110 | { 111 | 'output': 'Checking for "debugger" inside js and html files...', 112 | 'command': 'grep -Hn "debugger" "{file_name}"', 113 | 'match_files': ['(?:.*/.*\.js$|.*\.html$)'], 114 | 'ignore_files': [], 115 | }, 116 | { 117 | 'output': 'Checking for html comments...', 118 | 'command': 'grep -Hn "<\!--" "{file_name}"', 119 | 'match_files': ['^.*\.html$'], 120 | 'error_message': ( 121 | "Avoid the use of HTML comments, it increases the size of the page and the DOM tree.\n" 122 | "Use django comments instead. Ex: '{# your comment #}'" 123 | ) 124 | }, 125 | { 126 | 'output': 'Checking for "=>" inside js test files...', 127 | 'command': 'grep -Hn "=>" "{file_name}"', 128 | 'match_files': ['^test-client/.*-test\.js$'], 129 | 'error_message': ( 130 | "focus rocket is a nice feature from busterjs\n" 131 | "but should never be committed since it\n" 132 | "limits the tests that are going to run." 133 | ) 134 | }, 135 | { 136 | 'output': 'Checking for "from __future__ import unicode_literals"', 137 | 'command': 'grep -L "from __future__ import unicode_literals" "{file_name}"', 138 | 'match_files': ['.*\.py$'], 139 | 'ignore_files': ['.*migrations.*'], 140 | }, 141 | { 142 | 'output': 'Checking for "# -*- coding: utf-8 -*-"', 143 | 'command': 'grep -L "coding: utf-8" "{file_name}"', 144 | 'match_files': ['.*\.py$'], 145 | 'ignore_files': ['.*migrations.*'], 146 | }, 147 | { 148 | 'output': 'Running pep8...', 149 | 'command': 'pep8 -r --ignore=E501,E502,W293,E121,E123,E124,E125,E126,E127,E128 "{file_name}"', 150 | 'match_files': ['.*\.py$'], 151 | 'ignore_files': ['.*migrations.*', '^gists/.*', '^conf/.*',], 152 | }, 153 | { 154 | # to see the complete list of tranformations/fixes: 155 | # run `2to3 -l` OR 156 | # see the complete list with description here: http://docs.python.org/2/library/2to3.html 157 | 'output': 'Running 2to3...', 158 | 'command': ('2to3 -f xreadlines -f types -f tuple_params -f throw -f sys_exc ' 159 | '-f set_literal -f renames -f raise -f paren -f urllib ' 160 | '-f operator -f ne -f methodattrs -f long -f isinstance -f intern -f input ' 161 | '-f import -f imports2 -f idioms -f getcwdu -f funcattrs -f exitfunc ' 162 | '-f execfile -f exec -f except -f buffer -f apply -f numliterals -f basestring ' 163 | '-f has_key -f reduce -f repr ' 164 | '"{file_name}" 2> /dev/null'), 165 | 'match_files': ['.*\.py$'], 166 | 'error_message': ( 167 | 'You probably have a code that will generate problems with python 3.\n' 168 | 'Take a look at http://packages.python.org/six/ to see a python 2 and 3 compatible way or\n' 169 | 'if the error relates to urllib, try to use requests: http://docs.python-requests.org/en/latest/\n' 170 | 'INFO: don\'t take the diff as an absolute truth.\n' 171 | 'More info:\n' 172 | '* http://docs.python.org/3/library/2to3.html\n' 173 | '* http://docs.pythonsprints.com/python3_porting/py-porting.html\n' 174 | '* http://docs.python.org/release/3.0/whatsnew/3.0.html' 175 | ) 176 | }, 177 | ] 178 | 179 | modified = re.compile('^[MA]\s+(?P.*)$') 180 | 181 | 182 | def matches_file(file_name, match_files): 183 | return any(re.compile(match_file).match(file_name) for match_file in match_files) 184 | 185 | 186 | def should_check_file(check, file_name): 187 | return (not 'match_files' in check or matches_file(file_name, check['match_files']))\ 188 | and (not 'ignore_files' in check or not matches_file(file_name, check['ignore_files'])) 189 | 190 | 191 | def get_relevant_module_infos(check_files): 192 | regex = re.compile( 193 | r'.*apps/(?P[\w/]+)' 194 | '/tests/(?P\w+)' 195 | '(?P[\w/]+)?' 196 | '(?Ptest.*?.py)$', 197 | ) 198 | 199 | get_module = lambda path: regex.search(path) 200 | get_clean_module_path = lambda info, group: info.group(group).strip('/').replace('/', '.') 201 | 202 | prettify_info = lambda info: { 203 | 'appname': info.group('appname'), 204 | 'app_package_path': 'apps/' + '/'.join(filter(bool, [ 205 | get_clean_module_path(info, 'appname'), 206 | get_clean_module_path(info, 'submodules_path'), 207 | ])).replace('.', '/') + '/__init__.py', 208 | 'app_module_path': 'apps/' + '/'.join(filter(bool, [ 209 | get_clean_module_path(info, 'appname'), 210 | get_clean_module_path(info, 'submodules_path'), 211 | ])).replace('.', '/') + '.py', 212 | 'submodules_path': info.group('submodules_path'), 213 | 'module_name': '.'.join(filter(bool, [ 214 | get_clean_module_path(info, 'appname'), 215 | get_clean_module_path(info, 'submodules_path'), 216 | ])), 217 | 'first_file_name': info.group('file_name'), 218 | 'test_type': info.group('type'), 219 | } 220 | module_infos = map(prettify_info, filter(bool, map(get_module, check_files))) 221 | return sorted(module_infos) 222 | 223 | possible_module_infos = set([ 224 | 'module_name', 225 | 'test_type', 226 | 'first_file_name', 227 | 'submodules_path', 228 | 'app_module_path', 229 | 'app_package_path', 230 | ]) 231 | 232 | 233 | def check_files(files, check): 234 | check_files = filter(lambda name: should_check_file(check, name), files) 235 | 236 | if check_files: 237 | if check.get('ignore_noqa_lines'): 238 | check['command'] += ' | grep -vi "#*noqa"' 239 | 240 | file_names = '" "'.join(check_files) 241 | relevant_module_infos = get_relevant_module_infos(check_files) 242 | 243 | extra_info = {} 244 | required_infos = set(re.findall(r'[{](\w+)[}]', check['command'])) 245 | 246 | command_requires_extra_info = not bool(required_infos.difference(possible_module_infos)) 247 | 248 | if command_requires_extra_info: 249 | if not relevant_module_infos: 250 | return 0 # the current command requires 251 | # relevant_module_infos but none is 252 | # available, let's forget this one 253 | 254 | extra_info = relevant_module_infos[0] 255 | output_label = highlight(check['output'].format(**extra_info)) 256 | 257 | else: 258 | output_label = highlight(check['output']) 259 | 260 | sys.stdout.write(output_label) 261 | sys.stdout.flush() 262 | 263 | command = check['command'].format(file_name=file_names, filenames=file_names, **extra_info) 264 | process = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True) 265 | out, err = process.communicate() 266 | 267 | if out or err: 268 | print error(' ✗') 269 | print ' ', '\n '.join(out.splitlines()) 270 | 271 | if err: 272 | print err 273 | 274 | if 'error_message' in check: 275 | print 276 | print highlight(check['error_message']) 277 | print 278 | 279 | return 1 280 | 281 | print ok(' ✔ ') 282 | 283 | return 0 284 | 285 | 286 | def run(test_name, command, env=None): 287 | 288 | sys.stdout.write(highlight('Running {}...'.format(test_name))) 289 | sys.stdout.flush() 290 | 291 | full_env = os.environ 292 | full_env.update(env or {}) 293 | 294 | try: 295 | subprocess.check_output( 296 | shlex.split(command), 297 | stderr=subprocess.STDOUT, 298 | env=full_env) 299 | except subprocess.CalledProcessError as e: 300 | print error(' ✗') 301 | print e.output 302 | raise SystemExit(e.returncode) 303 | else: 304 | print ok(' ✔ ') 305 | 306 | 307 | def check_call(command): 308 | subprocess.check_call(shlex.split(command), stdout=subprocess.PIPE, stderr=subprocess.PIPE) 309 | 310 | 311 | def get_committed_files(): 312 | files = [] 313 | out = subprocess.check_output(shlex.split('git status --porcelain')) 314 | for line in out.splitlines(): 315 | match = modified.match(line.strip()) 316 | if match: 317 | files.append(match.group('name')) 318 | 319 | return filter(lambda f: 'scripts/pre-commit' not in f, files) 320 | 321 | 322 | def main(files, is_git_hook): 323 | 324 | if is_git_hook: 325 | files = get_committed_files() 326 | # Stash any changes to the working tree that are not going to be committed 327 | print highlight('Git stashing untracked changes...') 328 | check_call('git stash --include-untracked --keep-index') 329 | 330 | any_changes_on_files_ending_with = lambda ext: any([f.endswith(ext) for f in files]) 331 | 332 | try: 333 | 334 | check_call('find . -name "*.pyc" -delete') 335 | 336 | if any_changes_on_files_ending_with('.py'): 337 | os.environ['TEST_TYPE'] = 'unit' 338 | run('python unit tests', 'make unit', env={'SKIP_DEPS': 'true'}) 339 | 340 | result = 0 341 | 342 | for check in CHECKS: 343 | result += check_files(files, check) 344 | if result and is_git_hook: 345 | break 346 | 347 | finally: 348 | 349 | if is_git_hook: 350 | # Unstash changes to the working tree that we had stashed 351 | check_call('git reset --hard') 352 | try: 353 | # This hook can be called by a simple git commit --amend 354 | # without anything to actually get from the stash. In this 355 | # case, this command will break and we can safely ignore it. 356 | check_call('git stash pop --quiet --index') 357 | except subprocess.CalledProcessError: 358 | pass 359 | 360 | sys.exit(result) 361 | 362 | 363 | if __name__ == '__main__': 364 | files = [] 365 | 366 | # if no arguments is passed it means that this 367 | # script is being called as a git hook 368 | is_git_hook = len(sys.argv) == 1 369 | all_files = '--all-files' in sys.argv 370 | 371 | if all_files: 372 | files = subprocess.check_output(shlex.split('git ls-files --full-name')).strip().split('\n') 373 | elif not is_git_hook: 374 | files = sys.argv[1:] 375 | 376 | main(files, is_git_hook) 377 | -------------------------------------------------------------------------------- /tests/functional/test_aggregations.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | from __future__ import unicode_literals 3 | 4 | from sure import scenario 5 | 6 | from pyeqs import QuerySet, Filter 7 | from pyeqs.dsl import Aggregations, Range 8 | from tests.helpers import prepare_data, cleanup_data, add_document, requires_es_gte 9 | 10 | 11 | @requires_es_gte('1.1.0') 12 | @scenario(prepare_data, cleanup_data) 13 | def test_search_aggregation(context): 14 | """ 15 | Search with aggregation 16 | """ 17 | # When create a queryset 18 | t = QuerySet("localhost", index="foo") 19 | 20 | # And there are records 21 | add_document("foo", {"bar": "baz"}) 22 | add_document("foo", {"bar": "bazbaz"}) 23 | 24 | # And I do an aggregated search 25 | t.aggregate(aggregation=Aggregations("foo_attrs", "bar", "terms")) 26 | t[0:10] 27 | 28 | # Then I get a the expected results 29 | t.aggregations().should.have.key('foo_attrs') 30 | t.aggregations()['foo_attrs'].should.have.key("buckets").being.equal([ 31 | {u'key': u'baz', u'doc_count': 1}, {u'key': u'bazbaz', u'doc_count': 1}]) 32 | 33 | 34 | @requires_es_gte('1.1.0') 35 | @scenario(prepare_data, cleanup_data) 36 | def test_search_terms_aggregation_with_size(context): 37 | """ 38 | Search with terms aggregation w/ specified size 39 | """ 40 | # When create a queryset 41 | t = QuerySet("localhost", index="foo") 42 | 43 | # And there are records 44 | add_document("foo", {"bar": "baz"}) 45 | add_document("foo", {"bar": "baz"}) 46 | add_document("foo", {"bar": "bazbaz"}) 47 | add_document("foo", {"bar": "bazbar"}) 48 | 49 | # And I do an aggregated search 50 | t.aggregate(aggregation=Aggregations("foo_attrs", "bar", "terms", size=1)) 51 | t[0:10] 52 | 53 | # Then I get a the expected results 54 | t.aggregations().should.have.key('foo_attrs') 55 | t.aggregations()['foo_attrs'].should.have.key("buckets").being.equal([ 56 | {u'key': u'baz', u'doc_count': 2}]) 57 | 58 | 59 | @requires_es_gte('1.1.0') 60 | @scenario(prepare_data, cleanup_data) 61 | def test_search_terms_aggregation_with_order_one(context): 62 | """ 63 | Search with terms aggregation ordered by count descending 64 | """ 65 | # When create a queryset 66 | t = QuerySet("localhost", index="foo") 67 | 68 | # And there are records 69 | add_document("foo", {"bar": "baz"}) 70 | add_document("foo", {"bar": "baz"}) 71 | add_document("foo", {"bar": "baz"}) 72 | add_document("foo", {"bar": "bazbaz"}) 73 | add_document("foo", {"bar": "bazbaz"}) 74 | add_document("foo", {"bar": "bazbar"}) 75 | 76 | # And I do an aggregated search 77 | t.aggregate(aggregation=Aggregations("foo_attrs", "bar", "terms", 78 | order_type="_count", order_dir="desc")) 79 | t[0:10] 80 | 81 | # Then I get a the expected results 82 | t.aggregations().should.have.key('foo_attrs') 83 | 84 | t.aggregations()['foo_attrs'].should.have.key("buckets").being.equal([ 85 | {u'key': u'baz', u'doc_count': 3}, 86 | {u'key': u'bazbaz', u'doc_count': 2}, 87 | {u'key': u'bazbar', u'doc_count': 1}]) 88 | 89 | 90 | @requires_es_gte('1.1.0') 91 | @scenario(prepare_data, cleanup_data) 92 | def test_search_terms_aggregation_with_order_two(context): 93 | """ 94 | Search with terms aggregation ordered by count ascending 95 | """ 96 | # When create a queryset 97 | t = QuerySet("localhost", index="foo") 98 | 99 | # And there are records 100 | add_document("foo", {"bar": "baz"}) 101 | add_document("foo", {"bar": "baz"}) 102 | add_document("foo", {"bar": "baz"}) 103 | add_document("foo", {"bar": "bazbaz"}) 104 | add_document("foo", {"bar": "bazbaz"}) 105 | add_document("foo", {"bar": "bazbar"}) 106 | 107 | # And I do an aggregated search 108 | t.aggregate(aggregation=Aggregations("foo_attrs", "bar", "terms", 109 | order_type="_count", order_dir="asc")) 110 | t[0:10] 111 | 112 | # Then I get a the expected results 113 | t.aggregations().should.have.key('foo_attrs') 114 | 115 | t.aggregations()['foo_attrs'].should.have.key("buckets").being.equal([ 116 | {u'key': u'bazbar', u'doc_count': 1}, 117 | {u'key': u'bazbaz', u'doc_count': 2}, 118 | {u'key': u'baz', u'doc_count': 3}]) 119 | 120 | 121 | @requires_es_gte('1.1.0') 122 | @scenario(prepare_data, cleanup_data) 123 | def test_search_terms_aggregation_with_order_three(context): 124 | """ 125 | Search with terms aggregation ordered by term descending 126 | """ 127 | # When create a queryset 128 | t = QuerySet("localhost", index="foo") 129 | 130 | # And there are records 131 | add_document("foo", {"bar": "baz"}) 132 | add_document("foo", {"bar": "baz"}) 133 | add_document("foo", {"bar": "baz"}) 134 | add_document("foo", {"bar": "bazbaz"}) 135 | add_document("foo", {"bar": "bazbaz"}) 136 | add_document("foo", {"bar": "bazbar"}) 137 | 138 | # And I do an aggregated search 139 | t.aggregate(aggregation=Aggregations("foo_attrs", "bar", "terms", 140 | order_type="_term", order_dir="desc")) 141 | t[0:10] 142 | 143 | # Then I get a the expected results 144 | t.aggregations().should.have.key('foo_attrs') 145 | 146 | t.aggregations()['foo_attrs'].should.have.key("buckets").being.equal([ 147 | {u'key': u'bazbaz', u'doc_count': 2}, 148 | {u'key': u'bazbar', u'doc_count': 1}, 149 | {u'key': u'baz', u'doc_count': 3}]) 150 | 151 | 152 | @requires_es_gte('1.1.0') 153 | @scenario(prepare_data, cleanup_data) 154 | def test_search_terms_aggregation_with_order_four(context): 155 | """ 156 | Search with terms aggregation ordered by term ascending 157 | """ 158 | # When create a queryset 159 | t = QuerySet("localhost", index="foo") 160 | 161 | # And there are records 162 | add_document("foo", {"bar": "baz"}) 163 | add_document("foo", {"bar": "baz"}) 164 | add_document("foo", {"bar": "baz"}) 165 | add_document("foo", {"bar": "bazbaz"}) 166 | add_document("foo", {"bar": "bazbaz"}) 167 | add_document("foo", {"bar": "bazbar"}) 168 | 169 | # And I do an aggregated search 170 | t.aggregate(aggregation=Aggregations("foo_attrs", "bar", "terms", 171 | order_type="_term", order_dir="asc")) 172 | t[0:10] 173 | 174 | # Then I get a the expected results 175 | t.aggregations().should.have.key('foo_attrs') 176 | 177 | t.aggregations()['foo_attrs'].should.have.key("buckets").being.equal([ 178 | {u'key': u'baz', u'doc_count': 3}, 179 | {u'key': u'bazbar', u'doc_count': 1}, 180 | {u'key': u'bazbaz', u'doc_count': 2}]) 181 | 182 | 183 | @requires_es_gte('1.1.0') 184 | @scenario(prepare_data, cleanup_data) 185 | def test_search_terms_aggregation_with_min_doc_count(context): 186 | """ 187 | Search with terms aggregation w/ a min_doc_count 188 | """ 189 | # When create a queryset 190 | t = QuerySet("localhost", index="foo") 191 | 192 | # And there are records 193 | add_document("foo", {"bar": "baz"}) 194 | add_document("foo", {"bar": "baz"}) 195 | add_document("foo", {"bar": "baz"}) 196 | add_document("foo", {"bar": "bazbaz"}) 197 | add_document("foo", {"bar": "bazbaz"}) 198 | add_document("foo", {"bar": "bazbar"}) 199 | 200 | # And I do an aggregated search 201 | t.aggregate(aggregation=Aggregations("foo_attrs", "bar", "terms", min_doc_count=2)) 202 | t[0:10] 203 | 204 | # Then I get a the expected results 205 | t.aggregations().should.have.key('foo_attrs') 206 | 207 | t.aggregations()['foo_attrs'].should.have.key("buckets").being.equal([ 208 | {u'key': u'baz', u'doc_count': 3}, 209 | {u'key': u'bazbaz', u'doc_count': 2}]) 210 | 211 | 212 | @requires_es_gte('1.1.0') 213 | @scenario(prepare_data, cleanup_data) 214 | def test_search_multi_aggregations(context): 215 | """ 216 | Search with multiple aggregations 217 | """ 218 | # When create a query block 219 | t = QuerySet("localhost", index="foo") 220 | 221 | # And there are records 222 | add_document("foo", {"bar": "baz", "foo": "foo"}) 223 | add_document("foo", {"bar": "bazbaz", "foo": "foo"}) 224 | add_document("foo", {"bar": "bazbaz"}) 225 | 226 | # And I do an aggregated search on two dimensions 227 | t.aggregate(aggregation=Aggregations("bar_attrs", "bar", "terms")) 228 | t.aggregate(aggregation=Aggregations("missing_foo", "foo", "missing")) 229 | t[0:10] 230 | 231 | # Then I get a the expected results 232 | t.aggregations().should.have.key("missing_foo").being.equal({u'doc_count': 1}) 233 | t.aggregations().should.have.key("bar_attrs") 234 | t.aggregations()['bar_attrs'].should.have.key("buckets").being.equal([ 235 | {u'key': u'bazbaz', u'doc_count': 2}, {u'key': u'baz', u'doc_count': 1}]) 236 | 237 | 238 | @requires_es_gte('1.1.0') 239 | @scenario(prepare_data, cleanup_data) 240 | def test_search_nested_aggregations(context): 241 | """ 242 | Search with nested aggregations 243 | """ 244 | # When create a query block 245 | t = QuerySet("localhost", index="foo") 246 | 247 | # And there are nested records 248 | add_document("foo", {"child": [{"stuff": "yep", "bazbaz": 10}], "foo": "foo"}) 249 | add_document("foo", {"child": [{"stuff": "nope", "bazbaz": 1}], "foo": "foofoo"}) 250 | 251 | # And I do a nested 252 | t.aggregate(aggregation=Aggregations("best_bazbaz", "bazbaz", "max", nested_path="child")) 253 | t[0:10] 254 | 255 | # The I get the expected results 256 | t.aggregations().should.have.key("child").being.equal({'best_bazbaz': {'value': 10.0}, 'doc_count': 2}) 257 | 258 | 259 | @requires_es_gte('1.1.0') 260 | @scenario(prepare_data, cleanup_data) 261 | def test_search_nested_terms_aggregations_with_size(context): 262 | """ 263 | Search with nested terms aggregation and a specified size 264 | """ 265 | # When create a query block 266 | t = QuerySet("localhost", index="foo") 267 | 268 | # And there are nested records 269 | add_document("foo", {"child": [{"stuff": "yep", "bazbaz": "type0"}], "foo": "foo"}) 270 | add_document("foo", {"child": [{"stuff": "yep", "bazbaz": "type0"}], "foo": "foo"}) 271 | add_document("foo", {"child": [{"stuff": "nope", "bazbaz": "type1"}], "foo": "foofoo"}) 272 | add_document("foo", {"child": [{"stuff": "nope", "bazbaz": "type2"}], "foo": "foofoo"}) 273 | 274 | # And I do a nested 275 | t.aggregate(aggregation=Aggregations("baz_types", "bazbaz", "terms", 276 | nested_path="child", size=1)) 277 | t[0:10] 278 | 279 | # The I get the expected results 280 | t.aggregations().should.have.key("child").being.equal({ 281 | u'baz_types': { 282 | u'buckets': [{u'doc_count': 2, u'key': u'type0'}] 283 | }, 284 | u'doc_count': 4 285 | }) 286 | 287 | 288 | @requires_es_gte('1.1.0') 289 | @scenario(prepare_data, cleanup_data) 290 | def test_search_filtered_aggregations(context): 291 | """ 292 | Search with filtered aggregations 293 | """ 294 | # When create a query block 295 | t = QuerySet("localhost", index="foo") 296 | 297 | # And there are records 298 | add_document("foo", {"bar": 0, "foo": "baz"}) 299 | add_document("foo", {"bar": 0, "foo": "baz"}) 300 | add_document("foo", {"bar": 2}) 301 | add_document("foo", {"bar": 3, "foo": "bazbaz"}) 302 | add_document("foo", {"bar": 5, "foo": "foobar"}) 303 | add_document("foo", {"bar": 5}) 304 | add_document("foo", {"bar": 5, "foo": "foobaz"}) 305 | add_document("foo", {"bar": 9}) 306 | 307 | # And I do a filtered 308 | f = Filter().filter(Range("bar", gte=5)) 309 | t.aggregate(aggregation=Aggregations("missing_foo", "foo", "missing", 310 | filter_val=f, filter_name="high_bar")) 311 | t[0:10] 312 | 313 | # Then I get the expected results 314 | t.aggregations().should.have.key("high_bar") 315 | t.aggregations()["high_bar"].should.have.key("missing_foo").being.equal( 316 | {u'doc_count': 2}) 317 | t.aggregations()["high_bar"]["doc_count"].should.equal(4) 318 | 319 | 320 | @requires_es_gte('1.1.0') 321 | @scenario(prepare_data, cleanup_data) 322 | def test_search_global_aggregations(context): 323 | """ 324 | Search with global aggregations 325 | """ 326 | # With a specific query 327 | # q = QueryBuilder(query_string=QueryString(query={"match": {"foo_attr": "yes"}})) 328 | 329 | # When create a query block 330 | t = QuerySet("localhost", index="foo") 331 | 332 | # And there are records 333 | add_document("foo", {"bar": "baz", "foo_attr": "yes"}) 334 | add_document("foo", {"bar": "bazbaz", "foo_attr": "no"}) 335 | add_document("foo", {"bar": "bazbaz"}) 336 | 337 | # And I do a global 338 | t.aggregate(aggregation=Aggregations("foo_attrs", "bar", "terms", global_name="all_bar")) 339 | t[0:10] 340 | 341 | # I get the expected results 342 | t.aggregations().should.have.key("all_bar").being.equal({ 343 | u'foo_attrs': {u'buckets': [ 344 | {u'key': u'bazbaz', u'doc_count': 2}, 345 | {u'key': u'baz', u'doc_count': 1} 346 | ]}, 347 | u'doc_count': 3}) 348 | 349 | 350 | @requires_es_gte('1.1.0') 351 | @scenario(prepare_data, cleanup_data) 352 | def test_search_range_aggregations(context): 353 | """ 354 | Search with aggregations for ranges 355 | """ 356 | # When create a query block 357 | t = QuerySet("localhost", index="foo") 358 | 359 | # And there are records 360 | add_document("foo", {"bar": 1, "foo": "foo"}) 361 | add_document("foo", {"bar": 2, "foo": "foo"}) 362 | add_document("foo", {"bar": 2}) 363 | 364 | # When I do a ranged aggregation 365 | range_list = [0, 1, 2, 3] 366 | t.aggregate(aggregation=Aggregations("bar_types", "bar", "metric", range_list=range_list)) 367 | t[0:10] 368 | 369 | # I get the expected results 370 | t.aggregations().should.have.key("bar_ranges") 371 | t.aggregations()["bar_ranges"].should.have.key("buckets").being.equal([ 372 | {u'to': 0.0, u'doc_count': 0}, 373 | {u'to': 1.0, u'from': 0.0, u'doc_count': 0}, 374 | {u'to': 2.0, u'from': 1.0, u'doc_count': 1}, 375 | {u'to': 3.0, u'from': 2.0, u'doc_count': 2}, 376 | {u'from': 3.0, u'doc_count': 0}]) 377 | 378 | # And I should be able to name it if I want 379 | t.aggregate(aggregation=Aggregations("bar_types", "bar", "metric", 380 | range_list=range_list, range_name="my_unique_range")) 381 | t[0:10] 382 | 383 | # I get the expected results 384 | t.aggregations().should.have.key("my_unique_range") 385 | t.aggregations()["my_unique_range"].should.have.key("buckets").being.equal([ 386 | {u'to': 0.0, u'doc_count': 0}, 387 | {u'to': 1.0, u'from': 0.0, u'doc_count': 0}, 388 | {u'to': 2.0, u'from': 1.0, u'doc_count': 1}, 389 | {u'to': 3.0, u'from': 2.0, u'doc_count': 2}, 390 | {u'from': 3.0, u'doc_count': 0}]) 391 | 392 | 393 | @requires_es_gte('1.1.0') 394 | @scenario(prepare_data, cleanup_data) 395 | def test_search_histogram_aggregations(context): 396 | """ 397 | Search with aggregations that have histograms 398 | """ 399 | # When create a query block 400 | t = QuerySet("localhost", index="foo") 401 | 402 | # And there are records 403 | add_document("foo", {"bar": 0, "foo": "baz"}) 404 | add_document("foo", {"bar": 0, "foo": "baz"}) 405 | add_document("foo", {"bar": 2}) 406 | add_document("foo", {"bar": 3, "foo": "bazbaz"}) 407 | add_document("foo", {"bar": 5, "foo": "foobar"}) 408 | add_document("foo", {"bar": 5}) 409 | add_document("foo", {"bar": 5, "foo": "foobaz"}) 410 | add_document("foo", {"bar": 9}) 411 | 412 | # When I do a histogram aggregation 413 | t.aggregate(aggregation=Aggregations("bar_buckets", "bar", "metric", histogram_interval=2)) 414 | t[0:10] 415 | 416 | # I get the expected results 417 | t.aggregations().should.have.key("bar_buckets") 418 | t.aggregations()["bar_buckets"].should.have.key("buckets").being.equal([ 419 | {u'key': 8, u'doc_count': 1}, 420 | {u'key': 4, u'doc_count': 3}, 421 | {u'key': 2, u'doc_count': 2}, 422 | {u'key': 0, u'doc_count': 2}]) 423 | 424 | 425 | @requires_es_gte('1.1.0') 426 | @scenario(prepare_data, cleanup_data) 427 | def test_search_histogram_aggregations_with_order(context): 428 | """ 429 | Search with aggregations that have histograms in order 430 | """ 431 | # When create a query block 432 | t = QuerySet("localhost", index="foo") 433 | 434 | # And there are records 435 | add_document("foo", {"bar": 0, "foo": "baz"}) 436 | add_document("foo", {"bar": 0, "foo": "baz"}) 437 | add_document("foo", {"bar": 2}) 438 | add_document("foo", {"bar": 3, "foo": "bazbaz"}) 439 | add_document("foo", {"bar": 5, "foo": "foobar"}) 440 | add_document("foo", {"bar": 5}) 441 | add_document("foo", {"bar": 5, "foo": "foobaz"}) 442 | add_document("foo", {"bar": 9}) 443 | 444 | # When I do a histogram aggregation 445 | t.aggregate(aggregation=Aggregations("bar_buckets", "bar", "metric", histogram_interval=2, 446 | order_type="_count", order_dir="asc")) 447 | t[0:10] 448 | 449 | # I get the expected results 450 | t.aggregations().should.have.key("bar_buckets") 451 | t.aggregations()["bar_buckets"].should.have.key("buckets").being.equal([ 452 | {u'key': 8, u'doc_count': 1}, 453 | {u'key': 0, u'doc_count': 2}, 454 | {u'key': 2, u'doc_count': 2}, 455 | {u'key': 4, u'doc_count': 3}]) 456 | 457 | 458 | @requires_es_gte('1.1.0') 459 | @scenario(prepare_data, cleanup_data) 460 | def test_search_histogram_aggregations_with_min_doc_count(context): 461 | """ 462 | Search with aggregations that have histograms with min_doc_count 463 | """ 464 | # When create a query block 465 | t = QuerySet("localhost", index="foo") 466 | 467 | # And there are records 468 | add_document("foo", {"bar": 0, "foo": "baz"}) 469 | add_document("foo", {"bar": 0, "foo": "baz"}) 470 | add_document("foo", {"bar": 2}) 471 | add_document("foo", {"bar": 3, "foo": "bazbaz"}) 472 | add_document("foo", {"bar": 5, "foo": "foobar"}) 473 | add_document("foo", {"bar": 5}) 474 | add_document("foo", {"bar": 5, "foo": "foobaz"}) 475 | add_document("foo", {"bar": 9}) 476 | 477 | # When I do a histogram aggregation 478 | t.aggregate(aggregation=Aggregations("bar_buckets", "bar", "metric", histogram_interval=2, 479 | min_doc_count=2)) 480 | t[0:10] 481 | 482 | # I get the expected results 483 | t.aggregations().should.have.key("bar_buckets") 484 | t.aggregations()["bar_buckets"].should.have.key("buckets").being.equal([ 485 | {u'key': 4, u'doc_count': 3}, 486 | {u'key': 2, u'doc_count': 2}, 487 | {u'key': 0, u'doc_count': 2}]) 488 | -------------------------------------------------------------------------------- /tests/unit/test_queryset.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | from __future__ import unicode_literals 3 | 4 | import httpretty 5 | import json 6 | 7 | from pyeqs import QuerySet, Filter 8 | from pyeqs.dsl import Aggregations, QueryString, ScriptScore, Sort, Term 9 | from tests.helpers import homogeneous 10 | 11 | 12 | def test_create_queryset(): 13 | """ 14 | Create Default QuerySet 15 | """ 16 | # When create a queryset 17 | t = QuerySet("foobar") 18 | 19 | # Then I see the appropriate JSON 20 | results = { 21 | "query": {"match_all": {}} 22 | } 23 | 24 | homogeneous(t._query, results) 25 | 26 | 27 | def test_create_queryset_with_query_string(): 28 | """ 29 | Create QuerySet with QueryString 30 | """ 31 | # When create a queryset 32 | q = QueryString("foo") 33 | t = QuerySet("foobar", query=q) 34 | 35 | # Then I see the appropriate JSON 36 | results = { 37 | "query": { 38 | "query_string": { 39 | "query": "foo" 40 | } 41 | } 42 | } 43 | 44 | homogeneous(t._query, results) 45 | 46 | 47 | def test_create_queryset_with_filter(): 48 | """ 49 | Create QuerySet with Filter 50 | """ 51 | # When create a query block 52 | t = QuerySet("foobar") 53 | 54 | # And I add a filter 55 | t.filter(Term("foo", "bar")) 56 | 57 | # Then I see the appropriate JSON 58 | results = { 59 | "query": { 60 | "filtered": { 61 | "query": {"match_all": {}}, 62 | "filter": { 63 | "and": [ 64 | { 65 | "term": { 66 | "foo": "bar" 67 | } 68 | } 69 | ] 70 | } 71 | } 72 | } 73 | } 74 | 75 | homogeneous(t._query, results) 76 | 77 | 78 | def test_create_queryset_with_filters(): 79 | """ 80 | Create QuerySet with Multiple Filters 81 | """ 82 | # When create a query block 83 | t = QuerySet("foobar") 84 | 85 | # And I add a filter 86 | t.filter(Term("foo", "bar")) 87 | t.filter(Term("foobar", "foobar")) 88 | 89 | # Then I see the appropriate JSON 90 | results = { 91 | "query": { 92 | "filtered": { 93 | "query": {"match_all": {}}, 94 | "filter": { 95 | "and": [ 96 | { 97 | "term": { 98 | "foo": "bar" 99 | } 100 | }, 101 | { 102 | "term": { 103 | "foobar": "foobar" 104 | } 105 | } 106 | ] 107 | } 108 | } 109 | } 110 | } 111 | 112 | homogeneous(t._query, results) 113 | 114 | 115 | def test_create_queryset_with_filter_block(): 116 | """ 117 | Create QuerySet with Filter Block 118 | """ 119 | # When create a query block 120 | t = QuerySet("foobar") 121 | 122 | # And I add a filter 123 | f = Filter("or").filter(Term("foo", "bar")) 124 | t.filter(f) 125 | 126 | # Then I see the appropriate JSON 127 | results = { 128 | "query": { 129 | "filtered": { 130 | "query": {"match_all": {}}, 131 | "filter": { 132 | "or": [ 133 | { 134 | "term": { 135 | "foo": "bar" 136 | } 137 | } 138 | ] 139 | } 140 | } 141 | } 142 | } 143 | 144 | homogeneous(t._query, results) 145 | 146 | 147 | def test_create_queryset_with_sorting(): 148 | """ 149 | Create QuerySet with Sorting 150 | """ 151 | # When create a query block 152 | t = QuerySet("foobar") 153 | 154 | # And I add sorting 155 | s = Sort("_id", order="asc") 156 | t.order_by(s) 157 | 158 | # Then I see the appropriate JSON 159 | results = { 160 | "query": { 161 | "match_all": {} 162 | }, 163 | "sort": [ 164 | { 165 | "_id": { 166 | "order": "asc" 167 | } 168 | } 169 | ] 170 | } 171 | 172 | homogeneous(t._query, results) 173 | 174 | 175 | def test_create_queryset_with_multiple_sorting(): 176 | """ 177 | Create QuerySet with Multiple Sorting 178 | """ 179 | # When create a query block 180 | t = QuerySet("foobar") 181 | 182 | # And I add sorting 183 | s = Sort("_id", order="asc") 184 | t.order_by(s) 185 | 186 | ss = Sort("_id", order="desc") 187 | t.order_by(ss) 188 | 189 | # Then I see the appropriate JSON 190 | results = { 191 | "sort": [ 192 | { 193 | "_id": { 194 | "order": "asc" 195 | } 196 | }, 197 | { 198 | "_id": { 199 | "order": "desc" 200 | } 201 | } 202 | ], 203 | "query": { 204 | "match_all": {} 205 | } 206 | } 207 | 208 | homogeneous(t._query, results) 209 | 210 | 211 | def test_create_queryset_with_scoring(): 212 | """ 213 | Create QuerySet with Scoring 214 | """ 215 | # When create a query block 216 | t = QuerySet("foobar") 217 | 218 | # And I add scoring 219 | s = ScriptScore("foo = 0.0") 220 | t.score(s) 221 | 222 | # Then I see the appropriate JSON 223 | results = { 224 | "query": { 225 | "function_score": { 226 | "functions": [ 227 | { 228 | "script_score": { 229 | "script": "foo = 0.0" 230 | } 231 | } 232 | ], 233 | "query": {"match_all": {}}, 234 | "boost_mode": "replace", 235 | "score_mode": "multiply" 236 | } 237 | } 238 | } 239 | 240 | homogeneous(t._query, results) 241 | 242 | 243 | def test_create_queryset_with_scoring_min_score_track_score(): 244 | """ 245 | Create QuerySet with Scoring, Minimum Score and Track Scores 246 | """ 247 | # When create a query block 248 | t = QuerySet("foobar") 249 | 250 | # And I add scoring 251 | s = ScriptScore("foo = 0.0") 252 | t.score(s, min_score=0, track_scores=True) 253 | 254 | # Then I see the appropriate JSON 255 | results = { 256 | "min_score": 0, 257 | "track_scores": True, 258 | "query": { 259 | "function_score": { 260 | "functions": [ 261 | { 262 | "script_score": { 263 | "script": "foo = 0.0" 264 | } 265 | } 266 | ], 267 | "query": {"match_all": {}}, 268 | "boost_mode": "replace", 269 | "score_mode": "multiply" 270 | } 271 | } 272 | } 273 | 274 | homogeneous(t._query, results) 275 | 276 | 277 | def test_create_queryset_with_multiple_scoring(): 278 | """ 279 | Create QuerySet with Multiple Scoring 280 | """ 281 | # When create a query block 282 | t = QuerySet("foobar") 283 | 284 | # And I add scoring 285 | s = ScriptScore("foo = 0.0") 286 | t.score(s) 287 | 288 | # And I add more scoring 289 | boost = { 290 | "boost_factor": "3", 291 | "filter": Term("foo", "bar") 292 | } 293 | t.score(boost) 294 | 295 | # Then I see the appropriate JSON 296 | results = { 297 | "query": { 298 | "function_score": { 299 | "query": {"match_all": {}}, 300 | "functions": [ 301 | { 302 | "script_score": { 303 | "script": "foo = 0.0" 304 | } 305 | }, 306 | { 307 | "boost_factor": "3", 308 | "filter": { 309 | "term": { 310 | "foo": "bar" 311 | } 312 | } 313 | } 314 | ], 315 | "boost_mode": "replace", 316 | "score_mode": "multiply" 317 | } 318 | } 319 | } 320 | 321 | homogeneous(t._query, results) 322 | 323 | 324 | def test_create_queryset_with_scoring_and_filtering(): 325 | """ 326 | Create QuerySet with Scoring and Filtering 327 | """ 328 | # When create a query block 329 | t = QuerySet("foobar") 330 | 331 | # And I add scoring 332 | s = ScriptScore("foo = 0.0") 333 | t.score(s) 334 | 335 | # And I add filtering 336 | t.filter(Term("foo", "bar")) 337 | 338 | # Then I see the appropriate JSON 339 | results = { 340 | "query": { 341 | "function_score": { 342 | "query": { 343 | "filtered": { 344 | "query": {"match_all": {}}, 345 | "filter": { 346 | "and": [ 347 | { 348 | "term": { 349 | "foo": "bar" 350 | } 351 | } 352 | ] 353 | } 354 | } 355 | }, 356 | "functions": [ 357 | { 358 | "script_score": { 359 | "script": "foo = 0.0" 360 | } 361 | } 362 | ], 363 | "boost_mode": "replace", 364 | "score_mode": "multiply" 365 | } 366 | } 367 | } 368 | 369 | homogeneous(t._query, results) 370 | 371 | 372 | def test_create_queryset_with_scoring_and_filtering_from_object(): 373 | """ 374 | Create QuerySet with Scoring and Filter Object 375 | """ 376 | # When create a query block 377 | t = QuerySet("foobar") 378 | 379 | # And I add scoring 380 | s = ScriptScore("foo = 0.0") 381 | t.score(s) 382 | 383 | # And I add filtering 384 | f = Filter("and").filter(Term("foo", "bar")) 385 | t.filter(f) 386 | 387 | # Then I see the appropriate JSON 388 | results = { 389 | "query": { 390 | "function_score": { 391 | "query": { 392 | "filtered": { 393 | "query": {"match_all": {}}, 394 | "filter": { 395 | "and": [ 396 | { 397 | "term": { 398 | "foo": "bar" 399 | } 400 | } 401 | ] 402 | } 403 | } 404 | }, 405 | "functions": [ 406 | { 407 | "script_score": { 408 | "script": "foo = 0.0" 409 | } 410 | } 411 | ], 412 | "boost_mode": "replace", 413 | "score_mode": "multiply" 414 | } 415 | } 416 | } 417 | 418 | homogeneous(t._query, results) 419 | 420 | 421 | def test_create_queryset_with_filters_and_scoring(): 422 | """ 423 | Create QuerySet with Scoring and Multiple Filters 424 | """ 425 | # When create a query block 426 | t = QuerySet("foobar") 427 | 428 | # And I add filtering 429 | t.filter(Term("foo", "bar")) 430 | 431 | # And I add scoring 432 | s = ScriptScore("foo = 0.0") 433 | t.score(s) 434 | 435 | # And I add a second filter 436 | t.filter(Term("foobar", "foobar")) 437 | 438 | # Then I see the appropriate JSON 439 | results = { 440 | "query": { 441 | "function_score": { 442 | "query": { 443 | "filtered": { 444 | "query": {"match_all": {}}, 445 | "filter": { 446 | "and": [ 447 | { 448 | "term": { 449 | "foo": "bar" 450 | } 451 | }, 452 | { 453 | "term": { 454 | "foobar": "foobar" 455 | } 456 | } 457 | ] 458 | } 459 | } 460 | }, 461 | "functions": [ 462 | { 463 | "script_score": { 464 | "script": "foo = 0.0" 465 | } 466 | } 467 | ], 468 | "boost_mode": "replace", 469 | "score_mode": "multiply" 470 | } 471 | } 472 | } 473 | 474 | homogeneous(t._query, results) 475 | 476 | 477 | def test_create_queryset_with_only_block(): 478 | """ 479 | Create QuerySet with Only block 480 | """ 481 | # When create a query block 482 | t = QuerySet("foobar") 483 | 484 | # And I add an 'only' block 485 | t.only("_id") 486 | 487 | # Then I see the appropriate JSON 488 | results = { 489 | "query": {"match_all": {}}, 490 | "fields": ["_id"] 491 | } 492 | 493 | homogeneous(t._query, results) 494 | 495 | 496 | def test_queryset_count(): 497 | """ 498 | Get QuerySet Count 499 | """ 500 | # When I create a query block 501 | t = QuerySet("foobar") 502 | 503 | # Then I get an appropriate Count 504 | t.count().should.equal(None) 505 | 506 | 507 | def test_queryset_max_score(): 508 | """ 509 | Get QuerySet Max Score 510 | """ 511 | # When I create a query block 512 | t = QuerySet("foobar") 513 | 514 | # Then I get an appropriate max score 515 | t.max_score().should.equal(None) 516 | 517 | 518 | def test_queryset_string(): 519 | """ 520 | Create QuerySet with String query 521 | """ 522 | # When I create a query block 523 | t = QuerySet("foobar", query="foo") 524 | 525 | # Then I see the appropriate JSON 526 | results = { 527 | "query": { 528 | "query_string": { 529 | "query": "foo" 530 | } 531 | } 532 | } 533 | 534 | homogeneous(t._query, results) 535 | 536 | 537 | def test_create_queryset_with_aggregation(): 538 | """ 539 | Create QuerySet with an Aggregation 540 | """ 541 | # When I create a query block 542 | t = QuerySet("foobar") 543 | 544 | # And I aggregate 545 | a = Aggregations("agg_name", "field_name", "metric") 546 | t.aggregate(a) 547 | 548 | results = { 549 | "query": { 550 | "match_all": {} 551 | }, 552 | "aggregations": { 553 | "agg_name": {"metric": {"field": "field_name"}} 554 | } 555 | } 556 | homogeneous(t._query, results) 557 | 558 | # And I can do it as many times as I want 559 | a1 = Aggregations("other_agg_name", "other_field_name", "terms", size=1) 560 | t.aggregate(a1) 561 | 562 | results = { 563 | "query": { 564 | "match_all": {} 565 | }, 566 | "aggregations": { 567 | "agg_name": {"metric": {"field": "field_name"}}, 568 | "other_agg_name": {"terms": { 569 | "field": "other_field_name", 570 | "order": {"_count": "desc"}, 571 | "min_doc_count": 1, 572 | "size": 1 573 | }} 574 | } 575 | } 576 | homogeneous(t._query, results) 577 | 578 | 579 | @httpretty.activate 580 | def test_queryset_getitem(): 581 | """ 582 | Fetch from QuerySet with __getitem__ 583 | """ 584 | # When I create a query block 585 | t = QuerySet("localhost", index="bar") 586 | 587 | # And I have records 588 | response = { 589 | "took": 12, 590 | "hits": { 591 | "total": 1, 592 | "max_score": 10, 593 | "hits": [ 594 | { 595 | "_index": "bar", 596 | "_type": "baz", 597 | "_id": "1", 598 | "_score": 10, 599 | "_source": { 600 | "foo": "bar" 601 | }, 602 | "sort": [ 603 | 1395687078000 604 | ] 605 | } 606 | ] 607 | } 608 | } 609 | httpretty.register_uri(httpretty.GET, "http://localhost:9200/bar/_search", 610 | body=json.dumps(response), 611 | content_type="application/json") 612 | 613 | results = t[0:1] 614 | len(results).should.equal(1) 615 | t.count().should.equal(1) 616 | 617 | 618 | @httpretty.activate 619 | def test_queryset_getitem_with_wrapper(): 620 | """ 621 | Fetch from QuerySet with __getitem__ and wrapper 622 | """ 623 | # When I create a query block 624 | t = QuerySet("localhost", index="bar") 625 | wrapper = lambda y: list(map(lambda x: x['_id'], y)) 626 | t.wrappers(wrapper) 627 | 628 | # And I have records 629 | response = { 630 | "took": 12, 631 | "hits": { 632 | "total": 1, 633 | "max_score": 10, 634 | "hits": [ 635 | { 636 | "_index": "bar", 637 | "_type": "baz", 638 | "_id": "1", 639 | "_score": 10, 640 | "_source": { 641 | "foo": "bar" 642 | }, 643 | "sort": [ 644 | 1395687078000 645 | ] 646 | } 647 | ] 648 | } 649 | } 650 | httpretty.register_uri(httpretty.GET, "http://localhost:9200/bar/_search", 651 | body=json.dumps(response), 652 | content_type="application/json") 653 | 654 | results = t[0:1] 655 | len(results).should.equal(1) 656 | t.count().should.equal(1) 657 | t.max_score().should.equal(10) 658 | int(results[0]).should.equal(1) 659 | 660 | 661 | @httpretty.activate 662 | def test_queryset_getitem_multiple(): 663 | """ 664 | Fetch from QuerySet with __getitem__ multiple times 665 | """ 666 | # When I create a query block 667 | t = QuerySet("localhost", index="bar") 668 | wrapper = lambda y: list(map(lambda x: x['_id'], y)) 669 | t.wrappers(wrapper) 670 | # And I have a record 671 | response = { 672 | "took": 12, 673 | "hits": { 674 | "total": 1, 675 | "max_score": 10, 676 | "hits": [ 677 | { 678 | "_index": "bar", 679 | "_type": "baz", 680 | "_id": "1", 681 | "_score": 10, 682 | "_source": { 683 | "foo": "bar" 684 | }, 685 | "sort": [ 686 | 1395687078000 687 | ] 688 | } 689 | ] 690 | } 691 | } 692 | httpretty.register_uri(httpretty.GET, "http://localhost:9200/bar/_search", 693 | body=json.dumps(response), 694 | content_type="application/json") 695 | 696 | results = t[0:1] 697 | len(results).should.equal(1) 698 | t.count().should.equal(1) 699 | 700 | results = t[0:1] 701 | len(results).should.equal(1) 702 | t.count().should.equal(1) 703 | 704 | 705 | @httpretty.activate 706 | def test_queryset_iteration(): 707 | """ 708 | Fetch results with QuerySet via __iter__ 709 | """ 710 | 711 | # When I create a query block 712 | t = QuerySet("foobar", index="bar") 713 | wrapper = lambda y: list(map(lambda x: x['_id'], y)) 714 | t.wrappers(wrapper) 715 | 716 | # And I have a record 717 | response = { 718 | "took": 12, 719 | "hits": { 720 | "total": 1, 721 | "max_score": 10, 722 | "hits": [ 723 | { 724 | "_index": "bar", 725 | "_type": "baz", 726 | "_id": "1", 727 | "_score": 10, 728 | "_source": { 729 | "foo": "bar" 730 | }, 731 | "sort": [ 732 | 1395687078000 733 | ] 734 | } 735 | ] 736 | } 737 | } 738 | httpretty.register_uri(httpretty.GET, "http://foobar:9200/bar/_search", 739 | body=json.dumps(response), 740 | content_type="application/json") 741 | 742 | results = [] 743 | for result in t: 744 | results.append(result) 745 | len(results).should.equal(1) 746 | len(t).should.equal(1) 747 | t.count().should.equal(1) 748 | 749 | 750 | @httpretty.activate 751 | def test_queryset_iteration_with_no_results(): 752 | """ 753 | Fetch results with QuerySet via __iter__ with no results 754 | """ 755 | 756 | # When I create a query block 757 | t = QuerySet("foobar", index="bar") 758 | wrapper = lambda y: list(map(lambda x: x['_id'], y)) 759 | t.wrappers(wrapper) 760 | 761 | # And I have no records 762 | response = { 763 | "took": 12, 764 | "hits": { 765 | "total": 0, 766 | "max_score": 0, 767 | "hits": [] 768 | } 769 | } 770 | httpretty.register_uri(httpretty.GET, "http://foobar:9200/bar/_search", 771 | body=json.dumps(response), 772 | content_type="application/json") 773 | 774 | results = [] 775 | for result in t: 776 | results.append(result) 777 | len(results).should.equal(0) 778 | t.count().should.equal(0) 779 | 780 | 781 | @httpretty.activate 782 | def test_queryset_iteration_with_multiple_cache_fetches(): 783 | """ 784 | Fetch results with QuerySet via __iter__ with multiple cache fetches 785 | """ 786 | 787 | # When I create a query block 788 | t = QuerySet("foobar", index="bar") 789 | wrapper = lambda y: list(map(lambda x: x['_id'], y)) 790 | t.wrappers(wrapper) 791 | 792 | # And we lower the per request to force multiple fetches 793 | t._per_request = 2 794 | 795 | # And I have records 796 | first_response = { 797 | "took": 12, 798 | "hits": { 799 | "total": 3, 800 | "max_score": 10, 801 | "hits": [ 802 | { 803 | "_index": "bar", 804 | "_type": "baz", 805 | "_id": "1", 806 | "_score": 10, 807 | "_source": { 808 | "foo": "bar" 809 | }, 810 | "sort": [ 811 | 1395687078000 812 | ] 813 | }, 814 | { 815 | "_index": "bar", 816 | "_type": "baz", 817 | "_id": "2", 818 | "_score": 10, 819 | "_source": { 820 | "foo": "barbar" 821 | }, 822 | "sort": [ 823 | 1395687078000 824 | ] 825 | } 826 | ] 827 | } 828 | } 829 | 830 | second_response = { 831 | "took": 12, 832 | "hits": { 833 | "total": 3, 834 | "max_score": 10, 835 | "hits": [ 836 | { 837 | "_index": "bar", 838 | "_type": "baz", 839 | "_id": "3", 840 | "_score": 10, 841 | "_source": { 842 | "foo": "barbarbar" 843 | }, 844 | "sort": [ 845 | 1395687078000 846 | ] 847 | } 848 | ] 849 | } 850 | } 851 | httpretty.register_uri(httpretty.GET, "http://foobar:9200/bar/_search", 852 | responses=[ 853 | httpretty.Response(body=json.dumps(first_response), content_type="application/json"), 854 | httpretty.Response(body=json.dumps(second_response), content_type="application/json"), 855 | ]) 856 | 857 | # Then I should eventually get all records 858 | results = [] 859 | for result in t: 860 | results.append(result) 861 | len(results).should.equal(3) 862 | t.count().should.equal(3) 863 | results[0].should.equal("1") 864 | results[1].should.equal("2") 865 | results[2].should.equal("3") 866 | 867 | 868 | @httpretty.activate 869 | def test_queryset_getitem_with_post_query_action(): 870 | """ 871 | Fetch from QuerySet with __getitem__ and post query action 872 | """ 873 | # When I create a query block 874 | t = QuerySet("localhost", index="bar") 875 | 876 | # And I have a post query action 877 | global my_global_var 878 | my_global_var = 1 879 | 880 | def action(self, results, start, stop): 881 | global my_global_var 882 | my_global_var += 1 883 | 884 | t.post_query_actions(action) 885 | 886 | # And I have records 887 | response = { 888 | "took": 12, 889 | "timed_out": False, 890 | "_shards": { 891 | "total": 5, 892 | "successful": 5, 893 | "failed": 0 894 | }, 895 | "hits": { 896 | "total": 1, 897 | "max_score": 10, 898 | "hits": [ 899 | { 900 | "_index": "bar", 901 | "_type": "baz", 902 | "_id": "1", 903 | "_score": 10, 904 | "_source": { 905 | "foo": "bar" 906 | }, 907 | "sort": [ 908 | 1395687078000 909 | ] 910 | } 911 | ] 912 | } 913 | } 914 | httpretty.register_uri(httpretty.GET, "http://localhost:9200/bar/_search", 915 | body=json.dumps(response), 916 | content_type="application/json") 917 | 918 | results = t[0:1] 919 | len(results).should.equal(1) 920 | t.count().should.equal(1) 921 | 922 | # Then I see the correct results 923 | results[0]['_id'].should.equal('1') 924 | my_global_var.should.equal(2) 925 | 926 | 927 | @httpretty.activate 928 | def test_queryset_aggregations(): 929 | """ 930 | Fetch aggregation data from QuerySet with #aggregations 931 | """ 932 | # When I create a query block 933 | t = QuerySet("localhost", index="bar") 934 | 935 | # And I aggregate something 936 | a = Aggregations("agg_name", "field_name", "metric") 937 | t.aggregate(a) 938 | 939 | # And I have records 940 | response = { 941 | "took": 12, 942 | "hits": { 943 | "total": 1, 944 | "max_score": 10, 945 | "hits": [ 946 | {"_index": "bar", 947 | "_type": "baz", 948 | "_id": "1", 949 | "_score": 10, 950 | "_source": { 951 | "foo": "bar" 952 | }, 953 | "sort": [ 954 | 1395687078000 955 | ]} 956 | ] 957 | }, 958 | "aggregations": { 959 | "agg_name": {"metric": {"field": "field_name"}} 960 | } 961 | } 962 | httpretty.register_uri(httpretty.GET, "http://localhost:9200/bar/_search", 963 | body=json.dumps(response), 964 | content_type="application/json") 965 | 966 | t[0:1] 967 | t.aggregations().should.have.key("agg_name").being.equal({"metric": {"field": "field_name"}}) 968 | --------------------------------------------------------------------------------