├── .gitignore
├── LICENCE
├── README.md
├── setup.py
├── sqlalchemy_elasticquery
    ├── __init__.py
    └── elastic_query.py
└── test
    └── test.py


/.gitignore:
--------------------------------------------------------------------------------
1 | build/
2 | dist/
3 | *.egg-info/
4 | 


--------------------------------------------------------------------------------
/LICENCE:
--------------------------------------------------------------------------------
 1 | The MIT License (MIT)
 2 | 
 3 | Copyright (c) <year> <copyright holders>
 4 | 
 5 | Permission is hereby granted, free of charge, to any person obtaining a copy
 6 | of this software and associated documentation files (the "Software"), to deal
 7 | in the Software without restriction, including without limitation the rights
 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 9 | copies of the Software, and to permit persons to whom the Software is
10 | furnished to do so, subject to the following conditions:
11 | 
12 | The above copyright notice and this permission notice shall be included in
13 | all copies or substantial portions of the Software.
14 | 
15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
21 | THE SOFTWARE.
22 | 


--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
 1 | # sqlalchemy-elasticquery
 2 | 
 3 | Use [ElasticSearch](http://www.elasticsearch.org/) syntax for search in [SQLAlchemy](http://www.sqlalchemy.org/).
 4 | 
 5 | WARNING: ElasticQuery is currently under active development.  Thus the format of the API and this module may change drastically.
 6 | 
 7 | # Install
 8 | ```
 9 | pip install sqlalchemy-elasticquery
10 | ```
11 | # Quick start example
12 | Import module
13 | ```python
14 | from sqlalchemy_elasticquery import elastic_query
15 | ```
16 | 
17 | SQLAlchemy imports
18 | ```python
19 | from sqlalchemy import create_engine, Column, Integer, String
20 | from sqlalchemy.ext.declarative import declarative_base
21 | from sqlalchemy.orm import sessionmaker
22 | from sqlalchemy_elasticquery import elastic_query
23 | ```
24 | 
25 | Setup SQLAlchemy
26 | ```python
27 | engine = create_engine('sqlite:///:memory:', echo=False)
28 | Session = sessionmaker(bind=engine)
29 | session = Session()
30 | Base = declarative_base()
31 | ```
32 | 
33 | Model example
34 | ```python
35 | class User(Base):
36 | 	__tablename__ = 'users'
37 | 	id = Column(Integer, primary_key=True)
38 | 	name = Column(String)
39 | 	lastname = Column(String)
40 | 	uid = Column(Integer)
41 | ```
42 | 
43 | Create DB and add mock data
44 | ```python
45 | Base.metadata.create_all(bind=engine)
46 | session.add_all([
47 | 	User(name='Jhon', lastname='Galt', uid='19571957'),
48 | 	User(name='Steve', lastname='Jobs', uid='20092009'),
49 | 	User(name='Iron', lastname='Man', uid='19571957')
50 | ])
51 | session.commit()
52 | ```
53 | 
54 | ElasticQuery example
55 | ```python
56 | query_string = '{"filter":{"or":{"name":"Jhon","lastname":"Galt"},"and":{"uid":"19571957"}}}'
57 | print elastic_query(User, query_string, session)
58 | SELECT users.id AS users_id, users.name AS users_name, users.lastname AS users_lastname, users.uid AS users_uid FROM users WHERE users.uid = :uid_1 AND (users.lastname = :lastname_1 OR users.name = :name_1)
59 | ```
60 | # Querying
61 | * ***Nested search***: You can search for nested properties(Two levels, now). Ex: 
62 | 
63 | # Options
64 | 
65 | * **enabled_fields**: It's a list of fields allowed for work, for default all fields are allowed.
66 | 
67 | # Using with Flask
68 | 
69 | ElasticQuery example
70 | ```python
71 | from sqlalchemy_elasticquery import elastic_query
72 | 
73 | query_string = '{"filter":{"or":{"name":"Jhon","lastname":"Galt"},"and":{"uid":"19571957"}}}'
74 | print elastic_query(User, query_string)
75 | SELECT users.id AS users_id, users.name AS users_name, users.lastname AS users_lastname, users.uid AS users_uid FROM users WHERE users.uid = :uid_1 AND (users.lastname = :lastname_1 OR users.name = :name_1)
76 | ```
77 | 
78 | # TODO:
79 |  - Improve documentation
80 |  - Improve tests
81 |  - Errors emit


--------------------------------------------------------------------------------
/setup.py:
--------------------------------------------------------------------------------
 1 | import os
 2 | from setuptools import setup
 3 | 
 4 | 
 5 | def read(*paths):
 6 |     """Build a file path from *paths* and return the contents."""
 7 |     with open(os.path.join(*paths), 'r') as f:
 8 |         return f.read()
 9 | 
10 | setup(
11 |     name='sqlalchemy-elasticquery',
12 |     version='0.0.3',
13 |     description='Use ElasticSearch query search in SQLAlchemy.',
14 |     url='https://github.com/loverajoel/sqlalchemy-elasticquery',
15 |     license='MIT',
16 |     author='Joel Lovera',
17 |     author_email='joelalover@gmail.com',
18 |     packages=['sqlalchemy_elasticquery'],
19 |     include_package_data=True,
20 |     install_requires=[
21 |         'Flask>=0.10',
22 |         'SQLAlchemy>=0.7.8',
23 |     ],
24 |     classifiers=[
25 |         'Development Status :: 4 - Beta',
26 |         'Environment :: Plugins',
27 |         'Intended Audience :: Developers',
28 |         'License :: OSI Approved :: MIT License',
29 |         'Operating System :: OS Independent',
30 |         'Programming Language :: Python',
31 |         'Topic :: Database'
32 |         'Topic :: Software Development :: Libraries :: Python Modules'
33 |     ]
34 | )
35 | 


--------------------------------------------------------------------------------
/sqlalchemy_elasticquery/__init__.py:
--------------------------------------------------------------------------------
1 | from .elastic_query import elastic_query
2 | 


--------------------------------------------------------------------------------
/sqlalchemy_elasticquery/elastic_query.py:
--------------------------------------------------------------------------------
  1 | """
  2 |     SQLAlchemy-ElasticQuery
  3 |     ~~~~~~~~~~~~~~~~~~~~~~~~~
  4 |     This extension allow you use the ElasticSearch syntax for search in SQLAlchemy.
  5 |     Get a query string and return a SQLAlchemy query
  6 | 
  7 |     Example query string:
  8 | 
  9 |         {
 10 |             "filter" : {
 11 |                 "or" : {
 12 |                     "firstname" : {
 13 |                         "equals" : "Jhon"
 14 |                     },
 15 |                     "lastname" : "Galt",
 16 |                     "uid" : {
 17 |                         "like" : "111111"
 18 |                     }
 19 |                 },
 20 |                 "and" : {
 21 |                     "status" : "active",
 22 |                     "age" : {
 23 |                         "gt" : 18
 24 |                     }
 25 |                 }
 26 |             },
 27 |             "sort" : {
 28 |                 { "firstname" : "asc" },
 29 |                 { "age" : "desc" }
 30 |             }
 31 |         }
 32 | 
 33 | """
 34 | import json
 35 | from sqlalchemy import and_, or_, desc, asc
 36 | 
 37 | __version__ = '0.0.1'
 38 | 
 39 | 
 40 | def elastic_query(model, query, session=None, enabled_fields=None):
 41 |     """ Public method for init the class ElasticQuery
 42 |         :model: SQLAlchemy model
 43 |         :query: valid string like a ElasticSearch
 44 |         :session: SQLAlchemy session *optional
 45 |         :enabled_fields: Fields allowed for make a query *optional
 46 |     """
 47 |     # TODO: make session to optional
 48 |     instance = ElasticQuery(model, query, session, enabled_fields)
 49 |     return instance.search()
 50 | 
 51 | """ Valid operators """
 52 | OPERATORS = {
 53 |     'like': lambda f, a: f.like(a),
 54 |     'equals': lambda f, a: f == a,
 55 |     'is_null': lambda f: f is None,
 56 |     'is_not_null': lambda f: f is not None,
 57 |     'gt': lambda f, a: f > a,
 58 |     'gte': lambda f, a: f >= a,
 59 |     'lt': lambda f, a: f < a,
 60 |     'lte': lambda f, a: f <= a,
 61 |     'in': lambda f, a: f.in_(a),
 62 |     'not_in': lambda f, a: ~f.in_(a),
 63 |     'not_equal_to': lambda f, a: f != a
 64 |     }
 65 | 
 66 | 
 67 | class ElasticQuery(object):
 68 |     """ Magic method """
 69 | 
 70 |     def __init__(self, model, query, session=None, enabled_fields=None):
 71 |         """ Initializator of the class 'ElasticQuery' """
 72 |         self.model = model
 73 |         self.query = query
 74 |         if hasattr(model, 'query'):
 75 |             self.model_query = model.query
 76 |         else:
 77 |             self.model_query = session.query(self.model)
 78 |         self.enabled_fields = enabled_fields
 79 | 
 80 |     def search(self):
 81 |         """ This is the most important method """
 82 |         try:
 83 |             filters = json.loads(self.query)
 84 |         except ValueError:
 85 |             return False
 86 | 
 87 |         result = self.model_query
 88 |         if 'filter'in filters.keys():
 89 |             result = self.parse_filter(filters['filter'])
 90 |         if 'sort'in filters.keys():
 91 |             result = result.order_by(*self.sort(filters['sort']))
 92 | 
 93 |         return result
 94 | 
 95 |     def parse_filter(self, filters):
 96 |         """ This method process the filters """
 97 |         for filter_type in filters:
 98 |             if filter_type == 'or' or filter_type == 'and':
 99 |                 conditions = []
100 |                 for field in filters[filter_type]:
101 |                     if self.is_field_allowed(field):
102 |                         conditions.append(self.create_query(self.parse_field(field, filters[filter_type][field])))
103 |                 if filter_type == 'or':
104 |                     self.model_query = self.model_query.filter(or_(*conditions))
105 |                 elif filter_type == 'and':
106 |                     self.model_query = self.model_query.filter(and_(*conditions))
107 |             else:
108 |                 if self.is_field_allowed(filter_type):
109 |                     conditions = self.create_query(self.parse_field(filter_type, filters[filter_type]))
110 |                     self.model_query = self.model_query.filter(conditions)
111 |         return self.model_query
112 | 
113 |     def parse_field(self, field, field_value):
114 |         """ Parse the operators and traduce: ES to SQLAlchemy operators """
115 |         if type(field_value) is dict:
116 |             # TODO: check operators and emit error
117 |             operator = list(field_value)[0]
118 |             if self.verify_operator(operator) is False:
119 |                 return "Error: operator does not exist", operator
120 |             value = field_value[operator]
121 |         elif type(field_value) is unicode:
122 |             operator = u'equals'
123 |             value = field_value
124 |         return field, operator, value
125 | 
126 |     @staticmethod
127 |     def verify_operator(operator):
128 |         """ Verify if the operator is valid """
129 |         try:
130 |             if hasattr(OPERATORS[operator], '__call__'):
131 |                 return True
132 |             else:
133 |                 return False
134 |         except ValueError:
135 |             return False
136 | 
137 |     def is_field_allowed(self, field):
138 |         if self.enabled_fields:
139 |             return field in self.enabled_fields
140 |         else:
141 |             return True
142 | 
143 |     def create_query(self, attr):
144 |         """ Mix all values and make the query """
145 |         field = attr[0]
146 |         operator = attr[1]
147 |         value = attr[2]
148 |         model = self.model
149 | 
150 |         if '.' in field:
151 |             field_items = field.split('.')
152 |             field_name = getattr(model, field_items[0], None)
153 |             class_name = field_name.property.mapper.class_
154 |             new_model = getattr(class_name, field_items[1])
155 |             return field_name.has(OPERATORS[operator](new_model, value))
156 | 
157 |         return OPERATORS[operator](getattr(model, field, None), value)
158 | 
159 |     def sort(self, sort_list):
160 |         """ Sort """
161 |         order = []
162 |         for sort in sort_list:
163 |             if sort_list[sort] == "asc":
164 |                 order.append(asc(getattr(self.model, sort, None)))
165 |             elif sort_list[sort] == "desc":
166 |                 order.append(desc(getattr(self.model, sort, None)))
167 |         return order
168 | 


--------------------------------------------------------------------------------
/test/test.py:
--------------------------------------------------------------------------------
  1 | import unittest
  2 | from sqlalchemy import create_engine, Column, Integer, String, ForeignKey
  3 | from sqlalchemy.ext.declarative import declarative_base
  4 | from flask.ext.sqlalchemy import SQLAlchemy
  5 | from sqlalchemy.orm import sessionmaker, relationship
  6 | from sqlalchemy_elasticquery import elastic_query
  7 | from flask import Flask
  8 | 
  9 | Base = declarative_base()
 10 | 
 11 | 
 12 | class City(Base):
 13 |     __tablename__ = 'city'
 14 | 
 15 |     id = Column(Integer, primary_key=True)
 16 |     name = Column(String)
 17 | 
 18 |     def __repr__(self):
 19 |         return str(self.id)
 20 | 
 21 | 
 22 | class User(Base):
 23 |     __tablename__ = 'users'
 24 | 
 25 |     id = Column(Integer, primary_key=True)
 26 |     name = Column(String)
 27 |     lastname = Column(String)
 28 |     uid = Column(Integer)
 29 |     city_id = Column(Integer, ForeignKey(City.id))
 30 |     city = relationship(City)
 31 | 
 32 |     def __repr__(self):
 33 |         return str(self.id)
 34 | 
 35 | 
 36 | class TestCase(unittest.TestCase):
 37 | 
 38 |     def setUp(self):
 39 |         """ Initial setup for the test"""
 40 |         global engine
 41 |         engine = create_engine('sqlite:///:memory:', echo=False)
 42 |         global Session
 43 |         Session = sessionmaker(bind=engine)
 44 |         global session
 45 |         session = Session()
 46 |         session._model_changes = {}
 47 | 
 48 |         Base.metadata.create_all(bind=engine)
 49 | 
 50 |         session.add_all([
 51 |             User(name='Jhon', lastname='Galt', uid='19571957', city_id=1),
 52 |             User(name='Steve', lastname='Jobs', uid='20092009', city_id=2),
 53 |             User(name='Iron', lastname='Man', uid='19571957', city_id=1),
 54 |             City(name='Cordoba'),
 55 |             City(name='New York')
 56 |         ])
 57 |         session.commit()
 58 | 
 59 |     def test_setup_is_ok(self):
 60 |         """ Demo test """
 61 |         assert(session.query(User).count() == 3)
 62 | 
 63 |     def test_simple_query(self):
 64 |         """ test simple query """
 65 |         query_string = '{"filter" : {"uid" : {"like" : "%1957%"} } }'
 66 |         assert(elastic_query(User, query_string, session).count() == 2)
 67 |         query_string = '{"filter" : {"name" : {"like" : "%Jho%"}, "lastname" : "Galt" } }'
 68 |         assert(elastic_query(User, query_string, session).count() == 1)
 69 | 
 70 |     def test_and_operator(self):
 71 |         """ test and operator """
 72 |         query_string = \
 73 |             '{"filter" : {"and" : {"name" : {"like" : "%Jho%"}, "lastname" : "Galt", "uid" : {"like" : "%1957%"} } } }'
 74 |         assert(elastic_query(User, query_string, session).count() == 1)
 75 | 
 76 |     def test_or_operator(self):
 77 |         """ test or operator """
 78 |         query_string = \
 79 |             '{"filter" : {"or" : { "name" : "Jobs", "lastname" : "Man", "uid" : "19571957" } } }'
 80 |         assert(elastic_query(User, query_string, session).count() == 2)
 81 | 
 82 |     def test_or_and_operator(self):
 83 |         """ test or and operator """
 84 |         query_string = \
 85 |             '{"filter" : {"or" : { "name" : "Jhon", "lastname" : "Galt" }, "and" : { "uid" : "19571957" } } }'
 86 |         assert(elastic_query(User, query_string, session).count() == 1)
 87 | 
 88 |     def test_sorting(self):
 89 |         """ test operator levels """
 90 |         query_string = \
 91 |             '{"filter" : {"or" : { "name" : "Jhon", "lastname" : "Man" } }, "sort": { "name" : "asc" } }'
 92 |         results = elastic_query(User, query_string, session).all()
 93 |         assert(results[0].name == 'Iron')
 94 | 
 95 |     def test_in_operator(self):
 96 |         """ test operator in """
 97 |         query_string = '{"filter" : {"name" : {"in" : ["Jhon", "Peter", "Iron"] } } }'
 98 |         assert(elastic_query(User, query_string, session).count() == 2)
 99 | 
100 |         query_string = '{"filter" : {"name" : {"in" :["Jhon", "Peter", "Iron"]}, "lastname" : "Galt" } }'
101 |         assert(elastic_query(User, query_string, session).count() == 1)
102 | 
103 |     def test_flask(self):
104 |         app = Flask(__name__)
105 |         db = SQLAlchemy(app)
106 |         app.config['SQLALCHEMY_DATABASE_URI'] = 'sqlite:///:memory:'
107 | 
108 |         class Cities(db.Model):
109 |             __tablename__ = 'users'
110 | 
111 |             id = Column(Integer, primary_key=True)
112 |             name = Column(String)
113 |             population = Column(Integer)
114 | 
115 |             def __init__(self, name, population):
116 |                 self.name = name
117 |                 self.population = population
118 |                 
119 |         app.config['TESTING'] = True
120 |         app = app.test_client()
121 |         db.create_all()
122 | 
123 |         city = Cities("Cordoba", 1000000)
124 |         db.session.add(city)
125 |         city = Cities("Rafaela", 99000)
126 |         db.session.add(city)
127 |         db.session.commit()
128 | 
129 |         query_string = '{ "sort": { "population" : "desc" } }'
130 |         results = elastic_query(Cities, query_string)
131 |         assert(results[0].name == 'Cordoba')
132 | 
133 |     def test_allow_fields_option(self):
134 |         """ test allow_fields option """
135 |         query_string = '{"filter" : {"or" : { "name" : "Jhon", "lastname" : "Man" } }, "sort": { "name" : "asc" } }'
136 |         enabled_fields = ['name']
137 |         results = elastic_query(User, query_string, session, enabled_fields=enabled_fields).all()
138 |         assert(results[0].name == 'Jhon')
139 | 
140 |     def test_search_for_levels(self):
141 |         """ test search for levels """
142 |         query_string = \
143 |             '{"filter" : {"or" : { "city.name" : "New York", "lastname" : "Man" } }, "sort": { "name" : "asc" } }'
144 |         results = elastic_query(User, query_string, session).all()
145 |         assert(results[0].name == 'Iron')
146 | 
147 |         query_string = '{"filter" : { "city.name" : "New York" } }'
148 |         results = elastic_query(User, query_string, session).all()
149 |         assert(results[0].name == 'Steve')
150 | 
151 |         query_string = '{"filter" : { "city.name" : {"like" : "%New%"} } }'
152 |         results = elastic_query(User, query_string, session).all()
153 |         assert(results[0].name == 'Steve')
154 | 
155 | 
156 | unittest.main()
157 | 


--------------------------------------------------------------------------------