├── .gitignore ├── README.md ├── __init__.py ├── es_monitor.py ├── es_sql ├── README.md ├── __init__.py ├── __main__.py ├── es_query.py ├── executors │ ├── __init__.py │ ├── select_from_leaf_executor.py │ ├── select_from_system.py │ ├── select_inside_executor.py │ └── translators │ │ ├── __init__.py │ │ ├── bucket_script_translator.py │ │ ├── case_when_translator.py │ │ ├── doc_script_translator.py │ │ ├── filter_translator.py │ │ ├── group_by_translator.py │ │ ├── join_translator.py │ │ ├── metric_translator.py │ │ └── sort_translator.py ├── sqlparse │ ├── __init__.py │ ├── compat.py │ ├── datetime_evaluator.py │ ├── datetime_evaluator_test.py │ ├── engine │ │ ├── __init__.py │ │ ├── filter.py │ │ └── grouping.py │ ├── exceptions.py │ ├── filters.py │ ├── formatter.py │ ├── functions.py │ ├── keywords.py │ ├── lexer.py │ ├── ordereddict.py │ ├── pipeline.py │ ├── sql.py │ ├── sql_select.py │ ├── sql_select_test.py │ ├── tokens.py │ └── utils.py └── tests │ ├── __init__.py │ ├── execute_sql_test.py │ ├── join │ ├── __init__.py │ ├── client_size_join_test.py │ └── server_side_join_test.py │ ├── select_from_leaf │ ├── __init__.py │ ├── order_by_test.py │ ├── projections_test.py │ └── where_test.py │ ├── select_inside_branch │ ├── __init__.py │ ├── group_by_test.py │ ├── having_test.py │ ├── order_by_test.py │ ├── projection_test.py │ ├── response_test.py │ └── where_test.py │ └── select_inside_leaf │ ├── __init__.py │ ├── group_by_test.py │ ├── having_test.py │ ├── metric_test.py │ ├── order_by_test.py │ └── response_test.py ├── explorer-ui ├── .babelrc ├── .eslintrc.js ├── .gitignore ├── README.md ├── build │ ├── dev-client.js │ ├── dev-server.js │ ├── karma.conf.js │ ├── webpack.base.conf.js │ ├── webpack.dev.conf.js │ └── webpack.prod.conf.js ├── package.json ├── src │ ├── App.vue │ ├── assets │ │ └── logo.png │ ├── components │ │ └── Query.vue │ ├── index.html │ └── main.js └── test │ └── unit │ ├── Hello.spec.js │ └── index.js ├── explorer ├── __init__.py ├── __main__.py ├── app.py ├── index.html ├── requirement.txt └── static ├── plugin.sh ├── sample ├── import-githubarchive.py ├── import-quote.py ├── import-symbol.py └── symbol │ ├── nasdaq.csv │ ├── nyse mkt.csv │ └── nyse.csv └── setup.py /.gitignore: -------------------------------------------------------------------------------- 1 | /.idea 2 | *.pyc 3 | *.pyo 4 | /sample/quote.zip 5 | /sample/githubarchive 6 | /log 7 | /build 8 | /dist 9 | /*.egg-info 10 | *.iml 11 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | [![Gitter](https://badges.gitter.im/taowen/es-monitor.svg)](https://gitter.im/taowen/es-monitor?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge) 2 | 3 | A set of tools to query Elasticsearch with SQL 4 | 5 | Tutorial in Chinese: https://segmentfault.com/a/1190000003502849 6 | 7 | # As Monitor Plugin 8 | 9 | ``` 10 | ./plugin.sh https://url-to-params 11 | ``` 12 | 13 | The url points to a file with format： 14 | 15 | * first line: elasticsearch http url 16 | * remaining lines: sql 17 | 18 | For example 19 | 20 | ``` 21 | http://es_hosts 22 | 23 | SELECT count(*) AS value FROM gs_plutus_debug 24 | WHERE "timestamp" > now() - INTERVAL '5 minutes'; 25 | SAVE RESULT AS gs_plutus_debug.count; 26 | 27 | SELECT count(*) AS value FROM gs_api_track 28 | WHERE "@timestamp" > now() - INTERVAL '5 minutes'; 29 | SAVE RESULT AS gs_api_track.count; 30 | ``` 31 | 32 | Basic authentication is supported 33 | 34 | ``` 35 | cat << EOF | python -m es_sql http://xxx:8000/ 36 | VAR username=hello; 37 | VAR password=world; 38 | SELECT count(*) FROM my_index 39 | EOF 40 | ``` 41 | 42 | Can also use SQL to query Elasticsearch cluster health stats 43 | ``` 44 | SELECT * FROM _cluster_health 45 | SELECT * FROM _cluster_state 46 | SELECT * FROM _cluster_stats 47 | SELECT * FROM _cluster_pending_tasks 48 | SELECT * FROM _cluster_reroute 49 | SELECT * FROM _nodes_stats 50 | SELECT * FROM _nodes_info 51 | SELECT * FROM _indices_stats 52 | SELECT * FROM _indices_stats.all 53 | SELECT * FROM _indices_stats.[index_name] 54 | ``` 55 | 56 | The output will be a JSON array containing data points 57 | 58 | # As Console Command 59 | 60 | For example 61 | 62 | ``` 63 | cat << EOF | python -m es_sql http://es_hosts 64 | SELECT "user", "oid", max("@timestamp") as value FROM gs_api_track_ 65 | GROUP BY "user", "oid" WHERE "@timestamp" > 1454239084000 66 | EOF 67 | ``` 68 | 69 | ```python -m es_sql``` can be ```es-sql``` if ```pip install es-sql``` 70 | 71 | # As Python Library 72 | 73 | ``` 74 | pip install es-sql 75 | ``` 76 | ``` 77 | import es_sql 78 | es_sql.execute_sql( 79 | 'http://127.0.0.1:9200', 80 | 'SELECT COUNT(*) FROM your_index WHERE field=%(param)s', 81 | arguments={'param': 'value'}) 82 | ``` 83 | For more information: https://github.com/taowen/es-monitor/tree/master/es_sql 84 | 85 | # As HTTP Api 86 | 87 | Start http server (gunicorn) 88 | ``` 89 | python -m explorer 90 | ``` 91 | Translate SQL to Elasticsearch DSL request 92 | ``` 93 | $ cat << EOF | curl -X POST -d @- http://127.0.0.1:8000/translate 94 | SELECT * FROM quote WHERE symbol='AAPL' 95 | EOF 96 | 97 | { 98 | "data": { 99 | "indices": "quote*", 100 | "query": { 101 | "term": { 102 | "symbol": "AAPL" 103 | } 104 | } 105 | }, 106 | "error": null 107 | } 108 | ``` 109 | 110 | Use SQL to query Elasticsearch 111 | ``` 112 | $ cat << EOF | curl -X POST -d @- http://127.0.0.1:8000/search?elasticsearch=http://127.0.0.1:9200 113 | SELECT COUNT(*) FROM quote WHERE symbol='AAPL' 114 | EOF 115 | 116 | { 117 | "data": { 118 | "result": [ 119 | { 120 | "COUNT(*)": 8790 121 | } 122 | ] 123 | }, 124 | "error": null 125 | } 126 | ``` 127 | 128 | Use SQL with arguments 129 | ``` 130 | $ cat << EOF | curl -X POST -d @- http://127.0.0.1:8000/search_with_arguments 131 | { 132 | "elasticsearch":"http://127.0.0.1:9200", 133 | "sql":"SELECT COUNT(*) FROM quote WHERE symbol=%(param1)s", 134 | "arguments":{"param1":"AAPL"} 135 | } 136 | EOF 137 | { 138 | "data": { 139 | "result": [ 140 | { 141 | "COUNT(*)": 8790 142 | } 143 | ] 144 | }, 145 | "error": null 146 | } 147 | ``` -------------------------------------------------------------------------------- /__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/taowen/es-monitor/c4deceb4964857f495d13bfaf2d92f36734c9e1c/__init__.py -------------------------------------------------------------------------------- /es_monitor.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | 3 | """ 4 | Report odin series using elasticsearch query 5 | 6 | """ 7 | import base64 8 | import json 9 | import os 10 | import sys 11 | import time 12 | import urllib2 13 | import logging 14 | import logging.handlers 15 | 16 | LOGGER = logging.getLogger(__name__) 17 | 18 | from es_sql import es_query 19 | 20 | 21 | def to_str(val): 22 | if isinstance(val, basestring): 23 | return val 24 | else: 25 | return str(val) 26 | 27 | 28 | def query_datapoints(config): 29 | lines = config.splitlines() 30 | es_hosts = lines[0] 31 | sql = ''.join(lines[1:]) 32 | datapoints = [] 33 | ts = int(time.time()) 34 | try: 35 | result_map = es_query.execute_sql(es_hosts, sql) 36 | except urllib2.HTTPError as e: 37 | LOGGER.exception(e.read()) 38 | sys.exit(1) 39 | except: 40 | LOGGER.exception('read datapoint failed') 41 | sys.exit(1) 42 | for metric_name, rows in result_map.iteritems(): 43 | for row in rows or []: 44 | datapoint = {} 45 | datapoint['timestamp'] = ts 46 | datapoint['name'] = row.pop('_metric_name', None) or metric_name 47 | datapoint['value'] = row.pop('value', 0) 48 | if row: 49 | tags = {} 50 | for k, v in row.iteritems(): 51 | k = to_str(k) 52 | if not k.startswith('_'): 53 | tags[k] = to_str(v) 54 | datapoint['tags'] = tags 55 | datapoints.append(datapoint) 56 | LOGGER.info('read datapoints: %s' % len(datapoints)) 57 | return datapoints 58 | 59 | 60 | if __name__ == "__main__": 61 | logging.getLogger().setLevel(logging.INFO) 62 | home = os.getenv('HOME') 63 | if not home: 64 | home = '/tmp' 65 | handler = logging.handlers.RotatingFileHandler(os.path.join(home, '.es-monitor.log'), maxBytes=1024 * 1024, backupCount=0) 66 | handler.setFormatter(logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')) 67 | logging.getLogger().addHandler(handler) 68 | try: 69 | url = sys.argv[1] if len(sys.argv) > 1 else 'file:///home/rd/es-monitor/current.conf' 70 | if url.startswith('file:///'): 71 | with open(url.replace('file:///', '/'), 'r') as f: 72 | content = f.read() 73 | else: 74 | cache_key = '/tmp/es-monitor-%s' % base64.b64encode(url) 75 | if os.path.exists(cache_key): 76 | with open(cache_key) as f: 77 | content = f.read() 78 | else: 79 | resp = urllib2.urlopen(url) 80 | content = resp.read() 81 | with open(cache_key, 'w') as f: 82 | f.write(content) 83 | print json.dumps(query_datapoints(content)) 84 | except: 85 | LOGGER.exception('failed to run') 86 | sys.exit(1) -------------------------------------------------------------------------------- /es_sql/__init__.py: -------------------------------------------------------------------------------- 1 | from es_query import execute_sql 2 | -------------------------------------------------------------------------------- /es_sql/__main__.py: -------------------------------------------------------------------------------- 1 | import json 2 | import sys 3 | import logging 4 | import urllib2 5 | import sys 6 | 7 | from . import es_query 8 | 9 | 10 | def main(): 11 | logging.basicConfig(level=logging.DEBUG) 12 | sql = sys.stdin.read() 13 | result_map = es_query.execute_sql(sys.argv[1], sql) 14 | print('=====') 15 | for result_name, rows in result_map.iteritems(): 16 | for row in rows: 17 | print json.dumps(row) 18 | 19 | 20 | if __name__ == "__main__": 21 | try: 22 | main() 23 | except urllib2.HTTPError as e: 24 | print(e.read()) 25 | sys.exit(1) 26 | 27 | -------------------------------------------------------------------------------- /es_sql/es_query.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | """ 4 | Query elasticsearch using SQL 5 | """ 6 | import logging 7 | import re 8 | 9 | from .executors import SelectFromLeafExecutor 10 | from .executors import SelectInsideBranchExecutor 11 | from .executors import SelectInsideLeafExecutor 12 | from .executors import SqlParameter 13 | from .sqlparse.sql_select import SqlSelect 14 | 15 | LOGGER = logging.getLogger(__name__) 16 | 17 | def execute_sql(es_url, sql, arguments=None): 18 | arguments = arguments or {} 19 | current_sql_selects = [] 20 | result_map = {} 21 | for sql_select in sql.split(';'): 22 | sql_select = sql_select.strip() 23 | if not sql_select: 24 | continue 25 | is_sql = re.match(r'^(WITH|SELECT)\s+', sql_select, re.IGNORECASE) 26 | if is_sql: 27 | current_sql_selects.append(sql_select) 28 | else: 29 | is_var = re.match(r'^VAR\s+(.*)=(.*)$', sql_select, re.IGNORECASE | re.DOTALL) 30 | is_save = re.match(r'^SAVE\s+RESULT\s+AS\s+(.*)$', sql_select, re.IGNORECASE | re.DOTALL) 31 | is_remove = re.match(r'^REMOVE\s+RESULT\s+(.*)$', sql_select, re.IGNORECASE | re.DOTALL) 32 | if is_var: 33 | arguments[is_var.group(1)] = is_var.group(2) 34 | elif is_save: 35 | result_name = is_save.group(1) 36 | result_map[result_name] = create_executor(current_sql_selects, result_map).execute(es_url, arguments) 37 | current_sql_selects = [] 38 | elif is_remove: 39 | if current_sql_selects: 40 | result_map['result'] = create_executor(current_sql_selects, result_map).execute(es_url, arguments) 41 | current_sql_selects = [] 42 | result_map.pop(is_remove.group(1)) 43 | else: 44 | exec sql_select in {'result_map': result_map}, {} 45 | if current_sql_selects: 46 | result_map['result'] = create_executor(current_sql_selects, result_map).execute(es_url, arguments) 47 | return result_map 48 | 49 | 50 | def create_executor(sql_selects, joinable_results=None): 51 | executor_map = {} 52 | if not isinstance(sql_selects, list): 53 | sql_selects = [sql_selects] 54 | root_executor = None 55 | level = 0 56 | for sql_select in sql_selects: 57 | level += 1 58 | executor_name = 'level%s' % level 59 | if not isinstance(sql_select, SqlSelect): 60 | sql_select = sql_select.strip() 61 | if not sql_select: 62 | continue 63 | match = re.match(r'^WITH\s+(.*)\s+AS\s+\((.*)\)\s*$', sql_select, re.IGNORECASE | re.DOTALL) 64 | if match: 65 | executor_name = match.group(1).strip() 66 | sql_select = match.group(2).strip() 67 | sql_select = SqlSelect.parse(sql_select, joinable_results, executor_map) 68 | if not isinstance(sql_select.from_table, basestring): 69 | raise Exception('nested SELECT is not supported') 70 | if sql_select.from_table in executor_map: 71 | parent_executor = executor_map[sql_select.from_table] 72 | executor = SelectInsideBranchExecutor(sql_select, executor_name) 73 | parent_executor.add_child(executor) 74 | else: 75 | if sql_select.is_select_inside: 76 | executor = SelectInsideLeafExecutor(sql_select) 77 | else: 78 | executor = SelectFromLeafExecutor(sql_select) 79 | if root_executor: 80 | if executor.sql_select.join_table != root_executor[0]: 81 | raise Exception('multiple root executor is not supported') 82 | root_executor = (executor_name, executor) 83 | executor_map[executor_name] = executor 84 | if not root_executor: 85 | raise Exception('sql not found in %s' % sql_selects) 86 | root_executor[1].build_request() 87 | update_placeholder(root_executor[1].request, root_executor[1].request) 88 | return root_executor[1] 89 | 90 | 91 | 92 | def update_placeholder(request, obj, path=None): 93 | path = path or [] 94 | if isinstance(obj, dict): 95 | for k, v in obj.items(): 96 | obj[k] = update_placeholder(request, v, path + [k]) 97 | return obj 98 | elif isinstance(obj, (tuple, list)): 99 | for i, e in enumerate(list(obj)): 100 | obj[i] = update_placeholder(request, e, path + [i]) 101 | return obj 102 | elif isinstance(obj, SqlParameter): 103 | request['_parameters_'] = request.get('_parameters_', {}) 104 | request['_parameters_'][obj.parameter_name] = { 105 | 'path': path 106 | } 107 | if obj.field_hint: 108 | request['_parameters_'][obj.parameter_name]['field_hint'] = obj.field_hint 109 | return str(obj) 110 | else: 111 | return obj 112 | -------------------------------------------------------------------------------- /es_sql/executors/__init__.py: -------------------------------------------------------------------------------- 1 | from .select_from_leaf_executor import SelectFromLeafExecutor 2 | from .select_inside_executor import SelectInsideBranchExecutor 3 | from .select_inside_executor import SelectInsideLeafExecutor 4 | from .translators.filter_translator import SqlParameter 5 | 6 | -------------------------------------------------------------------------------- /es_sql/executors/select_from_leaf_executor.py: -------------------------------------------------------------------------------- 1 | import functools 2 | import logging 3 | import time 4 | import urllib2 5 | import json 6 | import base64 7 | 8 | from es_sql.sqlparse import sql as stypes 9 | from es_sql.sqlparse import tokens as ttypes 10 | from .translators import filter_translator 11 | from .translators import join_translator 12 | from .translators import sort_translator 13 | from . import select_from_system 14 | 15 | LOGGER = logging.getLogger(__name__) 16 | 17 | class SelectFromLeafExecutor(object): 18 | def __init__(self, sql_select): 19 | self.sql_select = sql_select 20 | self.request = self.build_request() 21 | self.selectors = [] 22 | for projection_name, projection in self.sql_select.projections.iteritems(): 23 | if projection.ttype == ttypes.Wildcard: 24 | self.selectors.append(select_wildcard) 25 | elif projection.ttype == ttypes.Name: 26 | self.selectors.append(functools.partial( 27 | select_name, projection_name=projection_name, projection=projection)) 28 | else: 29 | python_script = translate_projection_to_python(projection) 30 | python_code = compile(python_script, '', 'eval') 31 | self.selectors.append(functools.partial( 32 | select_by_python_code, projection_name=projection_name, python_code=python_code)) 33 | 34 | def execute(self, es_url, arguments=None): 35 | url = self.sql_select.generate_url(es_url) 36 | response = select_from_system.execute(es_url, self.sql_select) or search_es(url, self.request, arguments) 37 | return self.select_response(response) 38 | 39 | def build_request(self): 40 | request = {} 41 | if self.sql_select.order_by: 42 | request['sort'] = sort_translator.translate_sort(self.sql_select) 43 | if self.sql_select.limit: 44 | request['size'] = self.sql_select.limit 45 | if self.sql_select.where: 46 | request['query'] = filter_translator.create_compound_filter(self.sql_select.where.tokens[1:]) 47 | if self.sql_select.join_table: 48 | join_filters = join_translator.translate_join(self.sql_select) 49 | if len(join_filters) == 1: 50 | request['query'] = { 51 | 'bool': {'filter': [request.get('query', {}), join_filters[0]]}} 52 | else: 53 | request['query'] = { 54 | 'bool': {'filter': request.get('query', {}), 55 | 'should': join_filters}} 56 | return request 57 | 58 | def select_response(self, response): 59 | rows = [] 60 | for input in response['hits']['hits']: 61 | row = {} 62 | for selector in self.selectors: 63 | selector(input, row) 64 | rows.append(row) 65 | return rows 66 | 67 | 68 | def search_es(url, request, arguments=None, http_opener=None): 69 | arguments = arguments or {} 70 | parameters = request.pop('_parameters_', {}) 71 | if parameters: 72 | pset = set(parameters.keys()) 73 | aset = set(arguments.keys()) 74 | if pset - aset: 75 | raise Exception('not all parameters have been specified: %s' % (pset - aset)) 76 | if aset - pset: 77 | raise Exception('too many arguments specified: %s' % (aset - pset)) 78 | for param_name, param in parameters.iteritems(): 79 | level = request 80 | for p in param['path'][:-1]: 81 | level = level[p] 82 | level[param['path'][-1]] = arguments[param_name] 83 | request_id = time.time() 84 | if LOGGER.isEnabledFor(logging.DEBUG): 85 | LOGGER.debug('[%s] === send request to: %s\n%s' % (request_id, url, json.dumps(request, indent=2))) 86 | headers = { 87 | 'kbn-xsrf-token': arguments.get( 88 | 'kbn-xsrf-token', 'e6d4b4da34778b7ec0f25aae7480b5091caf39758b2c4f08f6ffe6e794b6e6fd') 89 | } 90 | if arguments.get('username'): 91 | auth_token = base64.encodestring('%s:%s' % (arguments['username'], arguments['password'])).replace('\n', '') 92 | headers['Authorization'] = 'Basic %s' % auth_token 93 | http_request = urllib2.Request(url, headers=headers, data=json.dumps(request)) 94 | if http_opener: 95 | resp = http_opener.open(http_request) 96 | else: 97 | resp = urllib2.urlopen(http_request) 98 | response = json.loads(resp.read()) 99 | if LOGGER.isEnabledFor(logging.DEBUG): 100 | LOGGER.debug('[%s] === received response:\n%s' % (request_id, json.dumps(response, indent=2))) 101 | return response 102 | 103 | 104 | def select_wildcard(input, row): 105 | row.update(input['_source']) 106 | if '_id' in input: 107 | row['_id'] = input['_id'] 108 | row['_type'] = input['_type'] 109 | row['_index'] = input['_index'] 110 | 111 | 112 | def select_name(input, row, projection_name, projection): 113 | projection_as_str = str(projection) 114 | if projection_as_str in input: 115 | row[projection_name] = input[projection_as_str] 116 | elif projection_as_str in input['_source']: 117 | row[projection_name] = input['_source'][projection_as_str] 118 | else: 119 | row[projection_name] = None 120 | 121 | 122 | def select_by_python_code(input, row, projection_name, python_code): 123 | row[projection_name] = eval(python_code, {}, input['_source']) 124 | 125 | 126 | def translate_projection_to_python(projection): 127 | if isinstance(projection, stypes.DotName): 128 | return translate_symbol(str(projection)) 129 | if isinstance(projection, stypes.TokenList): 130 | tokens = list(projection.flatten()) 131 | else: 132 | tokens = [projection] 133 | translated = [] 134 | for token in tokens: 135 | if token.ttype == ttypes.String.Symbol: 136 | translated.append(translate_symbol(token.value[1:-1])) 137 | else: 138 | translated.append(str(token)) 139 | return ''.join(translated) 140 | 141 | 142 | def translate_symbol(value): 143 | path = value.split('.') 144 | if len(path) == 1: 145 | return value 146 | else: 147 | return ''.join([path[0], "['", "']['".join(path[1:]), "']"]) 148 | -------------------------------------------------------------------------------- /es_sql/executors/select_from_system.py: -------------------------------------------------------------------------------- 1 | import urllib2 2 | import json 3 | 4 | def execute(es_url, sql_select): 5 | response = None 6 | if sql_select.from_table.startswith('_cluster_health'): 7 | response = json.loads(urllib2.urlopen('%s/_cluster/health' % es_url).read()) 8 | response = {'hits': {'hits': [{'_source': response}]}} 9 | elif sql_select.from_table.startswith('_cluster_state'): 10 | _, _, metric = sql_select.from_table.partition('.') 11 | if metric == 'nodes': 12 | response = json.loads(urllib2.urlopen('%s/_cluster/state/nodes' % es_url).read()) 13 | nodes = [] 14 | for node_id, node in response['nodes'].iteritems(): 15 | node['node_id'] = node_id 16 | nodes.append({'_source': node}) 17 | response = {'hits': {'hits': nodes}} 18 | elif metric == 'blocks': 19 | response = json.loads(urllib2.urlopen('%s/_cluster/state/blocks' % es_url).read()) 20 | blocks = [] 21 | for index_name, index_blocks in response['blocks'].get('indices', {}).iteritems(): 22 | for block_no, block in index_blocks.iteritems(): 23 | block['block_no'] = block_no 24 | block['index_name'] = index_name 25 | blocks.append({'_source': block}) 26 | response = {'hits': {'hits': blocks}} 27 | elif metric == 'routing_table': 28 | response = json.loads(urllib2.urlopen('%s/_cluster/state/routing_table' % es_url).read()) 29 | routing_tables = [] 30 | for index_name, index_shards in response['routing_table'].get('indices', {}).iteritems(): 31 | for shard_index, shard_tables in index_shards.get('shards', {}).iteritems(): 32 | for shard_table in shard_tables: 33 | shard_table['shard_index'] = shard_index 34 | shard_table['index_name'] = index_name 35 | routing_tables.append({'_source': shard_table}) 36 | response = {'hits': {'hits': routing_tables}} 37 | elif metric == 'routing_nodes': 38 | response = json.loads(urllib2.urlopen('%s/_cluster/state/routing_nodes' % es_url).read()) 39 | routing_nodes = [] 40 | for node_no, node_routing_nodes in response['routing_nodes'].get('nodes', {}).iteritems(): 41 | for routing_node in node_routing_nodes: 42 | routing_node['is_assigned'] = True 43 | routing_node['node_no'] = node_no 44 | routing_nodes.append({'_source': routing_node}) 45 | for routing_node in response['routing_nodes'].get('unassigned', []): 46 | routing_node['is_assigned'] = False 47 | routing_node['node_no'] = None 48 | routing_nodes.append({'_source': routing_node}) 49 | response = {'hits': {'hits': routing_nodes}} 50 | else: 51 | response = json.loads(urllib2.urlopen('%s/_cluster/state' % es_url).read()) 52 | response = {'hits': {'hits': [{'_source': response}]}} 53 | elif sql_select.from_table.startswith('_cluster_stats'): 54 | response = json.loads(urllib2.urlopen('%s/_cluster/stats' % es_url).read()) 55 | rows = [] 56 | collect_stats_rows(rows, response, ['cluster']) 57 | response = {'hits': {'hits': rows}} 58 | elif sql_select.from_table.startswith('_cluster_pending_tasks'): 59 | response = json.loads(urllib2.urlopen('%s/_cluster/pending_tasks' % es_url).read()) 60 | response = {'hits': {'hits': [{'_source': task} for task in response.get('tasks', [])]}} 61 | elif sql_select.from_table.startswith('_cluster_reroute'): 62 | response = json.loads(urllib2.urlopen('%s/_cluster/reroute' % es_url).read()) 63 | commands = [] 64 | for command in response.get('commands', []): 65 | for command_name, command_args in command.iteritems(): 66 | command_args['command_name'] = command_name 67 | commands.append({'_source': command_args}) 68 | response = {'hits': {'hits': commands}} 69 | elif sql_select.from_table.startswith('_nodes_stats'): 70 | response = json.loads(urllib2.urlopen('%s/_nodes/stats' % es_url).read()) 71 | all_rows = [] 72 | for node_id, node in response.get('nodes', {}).iteritems(): 73 | node_name = node.pop('name', None) 74 | node_transport_address = node.pop('transport_address', None) 75 | node_host = node.pop('host', None) 76 | node.pop('ip', None) 77 | rows = [] 78 | collect_stats_rows(rows, node, ['nodes']) 79 | for row in rows: 80 | row['_source']['node_id'] = node_id 81 | row['_source']['node_name'] = node_name 82 | row['_source']['node_transport_address'] = node_transport_address 83 | row['_source']['node_host'] = node_host 84 | all_rows.extend(rows) 85 | response = {'hits': {'hits': all_rows}} 86 | elif sql_select.from_table.startswith('_nodes_info'): 87 | response = json.loads(urllib2.urlopen('%s/_nodes' % es_url).read()) 88 | nodes = [] 89 | for node_id, node in response.get('nodes', {}).iteritems(): 90 | node['node_id'] = node_id 91 | nodes.append({'_source': node}) 92 | response = {'hits': {'hits': nodes}} 93 | elif sql_select.from_table.startswith('_indices_stats'): 94 | _, _, target_index_name = sql_select.from_table.partition('.') 95 | response = json.loads(urllib2.urlopen('%s/_stats' % es_url).read()) 96 | all_rows = [] 97 | if target_index_name: 98 | for index_name, index_stats in response.get('indices', {}).iteritems(): 99 | if target_index_name != 'all' and target_index_name != index_name: 100 | continue 101 | rows = [] 102 | collect_stats_rows(rows, index_stats, ['indices', 'per_index']) 103 | for row in rows: 104 | row['_source']['index_name'] = index_name 105 | all_rows.extend(rows) 106 | else: 107 | rows = [] 108 | collect_stats_rows(rows, response.get('_shards', {}), ['indices', 'shards']) 109 | all_rows.extend(rows) 110 | rows = [] 111 | collect_stats_rows(rows, response.get('_all', {}), ['indices', 'all']) 112 | all_rows.extend(rows) 113 | response = {'hits': {'hits': all_rows}} 114 | if sql_select.where and response and response['hits']['hits']: 115 | columns = sorted(response['hits']['hits'][0]['_source'].keys()) 116 | import sqlite3 117 | with sqlite3.connect(':memory:') as conn: 118 | conn.execute('CREATE TABLE temp(%s)' % (', '.join(columns))) 119 | rows = [[hit['_source'][column] for column in columns] for hit in response['hits']['hits']] 120 | conn.executemany('INSERT INTO temp VALUES (%s)' % (', '.join(['?'] * len(columns))), rows) 121 | filtered_rows = [] 122 | for row in conn.execute('SELECT * FROM temp %s' % sql_select.where): 123 | filtered_rows.append({'_source': dict(zip(columns, row))}) 124 | return {'hits': {'hits': filtered_rows}} 125 | return response 126 | 127 | def collect_stats_rows(rows, response, path): 128 | if isinstance(response, dict): 129 | for k, v in response.iteritems(): 130 | collect_stats_rows(rows, v, path + [k]) 131 | elif isinstance(response, (tuple, list)): 132 | for e in response: 133 | collect_stats_rows(rows, e, path) 134 | else: 135 | rows.append({'_source': { 136 | '_metric_name': '.'.join(path), 137 | 'value': response 138 | }}) 139 | -------------------------------------------------------------------------------- /es_sql/executors/select_inside_executor.py: -------------------------------------------------------------------------------- 1 | from es_sql.sqlparse.ordereddict import OrderedDict 2 | from .translators import bucket_script_translator 3 | from .translators import filter_translator 4 | from .translators import group_by_translator 5 | from .translators import join_translator 6 | from .translators import metric_translator 7 | from .translators import sort_translator 8 | from .select_from_leaf_executor import search_es 9 | 10 | 11 | class SelectInsideExecutor(object): 12 | def __init__(self, sql_select): 13 | self.sql_select = sql_select 14 | self.request = {'query': {}} 15 | self.children = [] 16 | 17 | def add_child(self, executor): 18 | self.children.append(executor) 19 | 20 | def build_request(self): 21 | buckets_names = self.list_buckets_names() 22 | for buckets_name, buckets_path in buckets_names.iteritems(): 23 | buckets_names[buckets_name] = self.format_buckets_path(buckets_path) 24 | self.sql_select.buckets_names = buckets_names 25 | self.metric_request, self.metric_selector = metric_translator.translate_metrics(self.sql_select) 26 | self.request['size'] = 0 # do not need hits in response 27 | self.add_aggs_to_request() 28 | self.add_children_aggs() 29 | 30 | def list_buckets_names(self): 31 | buckets_names = {} 32 | for projection_name, projection in self.sql_select.projections.iteritems(): 33 | if bucket_script_translator.is_count_star(projection): 34 | buckets_names[projection_name] = ['_count'] 35 | else: 36 | buckets_names[projection_name] = [projection_name] 37 | for group_by_name in self.sql_select.group_by.keys(): 38 | buckets_names[group_by_name] = ['_key'] 39 | for child_executor in self.children: 40 | child_prefix = child_executor.sql_select.group_by.keys() 41 | child_buckets_names = child_executor.list_buckets_names() 42 | for buckets_name, buckets_path in child_buckets_names.iteritems(): 43 | buckets_names[buckets_name] = child_prefix + buckets_path 44 | return buckets_names 45 | 46 | def format_buckets_path(self, buckets_path): 47 | prefix = '>'.join(buckets_path[:-1]) 48 | if prefix: 49 | return '.'.join([prefix, buckets_path[-1]]) 50 | else: 51 | return buckets_path[-1] 52 | 53 | def add_children_aggs(self): 54 | if not self.children: 55 | return 56 | aggs = self.request['aggs'] 57 | for bucket_key in self.sql_select.group_by.keys(): 58 | aggs = aggs[bucket_key]['aggs'] 59 | for child_executor in self.children: 60 | child_executor.build_request() 61 | child_aggs = child_executor.request['aggs'] 62 | aggs.update(child_aggs) 63 | 64 | def select_response(self, response): 65 | group_by_names = self.sql_select.group_by.keys() if self.sql_select.group_by else [] 66 | buckets = self.select_buckets(response) 67 | all_rows = [] 68 | for bucket, inner_row in buckets: 69 | rows = [] 70 | self.collect_records(rows, bucket, group_by_names, inner_row) 71 | all_rows.extend(rows) 72 | all_rows = self.pass_response_to_children(all_rows) 73 | for row in all_rows: 74 | filtered = row.pop('_filtered_', {}) 75 | filtered.pop('_bucket_', None) 76 | row.update(filtered) 77 | return all_rows 78 | 79 | def pass_response_to_children(self, all_rows): 80 | filter_children = [] 81 | drill_down_children = [] 82 | for child_executor in self.children: 83 | if child_executor.is_filter_only(): 84 | filter_children.append(child_executor) 85 | else: 86 | drill_down_children.append(child_executor) 87 | for child_executor in filter_children: 88 | child_executor.select_response(all_rows) 89 | if drill_down_children: 90 | children_rows = [] 91 | for child_executor in drill_down_children: 92 | children_rows.extend(child_executor.select_response(all_rows)) 93 | return children_rows 94 | else: 95 | return all_rows 96 | 97 | def select_buckets(self, response): 98 | raise Exception('base class') 99 | 100 | def add_aggs_to_request(self): 101 | self.request['aggs'], tail_aggs = group_by_translator.translate_group_by(self.sql_select.group_by) 102 | if self.metric_request: 103 | tail_aggs.update(self.metric_request) 104 | if self.sql_select.having: 105 | tail_aggs['having'] = { 106 | 'bucket_selector': bucket_script_translator.translate_script(self.sql_select, self.sql_select.having) 107 | } 108 | if self.sql_select.order_by or self.sql_select.limit: 109 | if len(self.sql_select.group_by or {}) != 1: 110 | raise Exception('order by can only be applied on single group by') 111 | group_by_name = self.sql_select.group_by.keys()[0] 112 | aggs = self.request['aggs'][group_by_name] 113 | agg_names = set(aggs.keys()) - set(['aggs']) 114 | if len(agg_names) != 1: 115 | raise Exception('order by can only be applied on single group by') 116 | agg_type = list(agg_names)[0] 117 | agg = aggs[agg_type] 118 | if self.sql_select.order_by: 119 | agg['order'] = sort_translator.translate_sort(self.sql_select) 120 | if self.sql_select.limit: 121 | agg['size'] = self.sql_select.limit 122 | 123 | def collect_records(self, rows, parent_bucket, group_by_names, props): 124 | if group_by_names: 125 | current_response = parent_bucket[group_by_names[0]] 126 | if 'buckets' in current_response: 127 | child_buckets = current_response['buckets'] 128 | if isinstance(child_buckets, dict): 129 | for child_bucket_key, child_bucket in child_buckets.iteritems(): 130 | child_props = dict(props, **{group_by_names[0]: child_bucket_key}) 131 | self.collect_records(rows, child_bucket, group_by_names[1:], child_props) 132 | else: 133 | for child_bucket in child_buckets: 134 | child_bucket_key = child_bucket['key_as_string'] if 'key_as_string' in child_bucket else \ 135 | child_bucket['key'] 136 | child_props = dict(props, **{group_by_names[0]: child_bucket_key}) 137 | self.collect_records(rows, child_bucket, group_by_names[1:], child_props) 138 | else: 139 | self.collect_records(rows, current_response, group_by_names[1:], props) 140 | else: 141 | record = props 142 | for key, value in parent_bucket.iteritems(): 143 | if isinstance(value, dict) and 'value' in value: 144 | record[key] = value['value'] 145 | for metric_name, get_metric in self.metric_selector.iteritems(): 146 | record[metric_name] = get_metric(parent_bucket) 147 | record['_bucket_'] = parent_bucket 148 | rows.append(record) 149 | 150 | 151 | class SelectInsideBranchExecutor(SelectInsideExecutor): 152 | def __init__(self, sql_select, executor_name): 153 | super(SelectInsideBranchExecutor, self).__init__(sql_select) 154 | self.executor_name = executor_name 155 | self._is_filter_only = len(self.sql_select.group_by) == 0 156 | if self.sql_select.where: 157 | old_group_by = self.sql_select.group_by 158 | self.sql_select.group_by = OrderedDict() 159 | self.sql_select.group_by[self.executor_name] = self.sql_select.where 160 | for key in old_group_by.keys(): 161 | self.sql_select.group_by[key] = old_group_by[key] 162 | self.sql_select.where = None 163 | 164 | def is_filter_only(self): 165 | return self._is_filter_only and all(child_executor.is_filter_only() for child_executor in self.children) 166 | 167 | def select_buckets(self, response): 168 | # response is selected from inner executor 169 | if self.is_filter_only(): 170 | buckets = [] 171 | for parent_row in response: 172 | bucket = parent_row.get('_bucket_') 173 | parent_row['_filtered_'] = parent_row.get('_filtered_', {}) 174 | buckets.append((bucket, parent_row['_filtered_'])) 175 | return buckets 176 | else: 177 | buckets = [] 178 | for parent_row in response: 179 | child_row = dict(parent_row) 180 | child_row['_bucket_path'] = list(child_row.get('_bucket_path', [])) 181 | child_row['_bucket_path'].append(self.executor_name) 182 | bucket = child_row.get('_bucket_') 183 | buckets.append((bucket, child_row)) 184 | return buckets 185 | 186 | 187 | class SelectInsideLeafExecutor(SelectInsideExecutor): 188 | def __init__(self, sql_select): 189 | super(SelectInsideLeafExecutor, self).__init__(sql_select) 190 | 191 | def execute(self, es_url, arguments=None): 192 | url = self.sql_select.generate_url(es_url) 193 | response = search_es(url, self.request, arguments) 194 | return self.select_response(response) 195 | 196 | def select_response(self, response): 197 | rows = super(SelectInsideLeafExecutor, self).select_response(response) 198 | for row in rows: 199 | row.pop('_bucket_', None) 200 | return rows 201 | 202 | def build_request(self): 203 | super(SelectInsideLeafExecutor, self).build_request() 204 | if self.sql_select.where: 205 | self.request['query'] = filter_translator.create_compound_filter(self.sql_select.where.tokens[1:]) 206 | if self.sql_select.join_table: 207 | join_filters = join_translator.translate_join(self.sql_select) 208 | if len(join_filters) == 1: 209 | self.request['query'] = { 210 | 'bool': {'filter': [self.request.get('query', {}), join_filters[0]]}} 211 | else: 212 | self.request['query'] = { 213 | 'bool': {'filter': self.request.get('query', {}), 214 | 'should': join_filters}} 215 | 216 | def select_buckets(self, response): 217 | # response is returned from elasticsearch 218 | buckets = [] 219 | bucket = response.get('aggregations', {}) 220 | bucket['doc_count'] = response['hits']['total'] 221 | buckets.append((bucket, {})) 222 | return buckets 223 | -------------------------------------------------------------------------------- /es_sql/executors/translators/__init__.py: -------------------------------------------------------------------------------- 1 | # build elasticsearch request from sql 2 | # translate elasticsearch response back to sql row concept -------------------------------------------------------------------------------- /es_sql/executors/translators/bucket_script_translator.py: -------------------------------------------------------------------------------- 1 | from es_sql.sqlparse import sql as stypes 2 | from es_sql.sqlparse import tokens as ttypes 3 | 4 | 5 | def translate_script(sql_select, tokens): 6 | agg = {'buckets_path': {}, 'script': {'lang': 'expression', 'inline': ''}} 7 | _translate(sql_select.buckets_names, agg, tokens) 8 | return agg 9 | 10 | 11 | def _translate(buckets_names, agg, tokens): 12 | for token in tokens: 13 | if token.ttype == ttypes.Keyword and 'AND' == token.value.upper(): 14 | agg['script']['inline'] = '%s%s' % ( 15 | agg['script']['inline'], '&&') 16 | elif token.ttype == ttypes.Keyword and 'OR' == token.value.upper(): 17 | agg['script']['inline'] = '%s%s' % ( 18 | agg['script']['inline'], '||') 19 | elif token.is_group(): 20 | _translate(buckets_names, agg, token.tokens) 21 | else: 22 | if token.is_field(): 23 | buckets_name = token.as_field_name() 24 | bucket_path = buckets_names.get(buckets_name) 25 | if not bucket_path: 26 | raise Exception( 27 | 'having clause referenced variable must exist in select clause: %s' % buckets_name) 28 | agg['buckets_path'][buckets_name] = bucket_path 29 | agg['script']['inline'] = '%s%s' % ( 30 | agg['script']['inline'], buckets_name) 31 | else: 32 | agg['script']['inline'] = '%s%s' % ( 33 | agg['script']['inline'], token.value) 34 | 35 | 36 | def is_count_star(projection): 37 | return isinstance(projection, stypes.Function) \ 38 | and 'COUNT' == projection.tokens[0].as_field_name().upper() \ 39 | and 1 == len(projection.get_parameters()) \ 40 | and ttypes.Wildcard == projection.get_parameters()[0].ttype 41 | -------------------------------------------------------------------------------- /es_sql/executors/translators/case_when_translator.py: -------------------------------------------------------------------------------- 1 | from es_sql.sqlparse import sql as stypes 2 | from es_sql.sqlparse import tokens as ttypes 3 | from . import filter_translator 4 | 5 | 6 | def translate_case_when(case_when): 7 | case_when_translator = CaseWhenNumericRangeTranslator() 8 | try: 9 | case_when_aggs = case_when_translator.on_CASE(case_when.tokens[1:]) 10 | except: 11 | case_when_translator = CaseWhenFiltersTranslator() 12 | case_when_aggs = case_when_translator.on_CASE(case_when.tokens[1:]) 13 | return case_when_aggs 14 | 15 | 16 | class CaseWhenNumericRangeTranslator(object): 17 | def __init__(self): 18 | self.ranges = [] 19 | self.field = None 20 | 21 | def on_CASE(self, tokens): 22 | idx = 0 23 | while idx < len(tokens): 24 | token = tokens[idx] 25 | idx += 1 26 | if token.is_whitespace(): 27 | continue 28 | if 'WHEN' == token.value.upper(): 29 | idx = self.on_WHEN(tokens, idx) 30 | elif 'ELSE' == token.value.upper(): 31 | idx = self.on_ELSE(tokens, idx) 32 | elif 'END' == token.value.upper(): 33 | break 34 | else: 35 | raise Exception('unexpected: %s' % token) 36 | return self.build() 37 | 38 | def on_WHEN(self, tokens, idx): 39 | current_range = {} 40 | idx = skip_whitespace(tokens, idx) 41 | token = tokens[idx] 42 | self.parse_comparison(current_range, token) 43 | idx = skip_whitespace(tokens, idx + 1) 44 | token = tokens[idx] 45 | if 'AND' == token.value.upper(): 46 | idx = skip_whitespace(tokens, idx + 1) 47 | token = tokens[idx] 48 | self.parse_comparison(current_range, token) 49 | idx = skip_whitespace(tokens, idx + 1) 50 | token = tokens[idx] 51 | if 'THEN' != token.value.upper(): 52 | raise Exception('unexpected: %s' % token) 53 | idx = skip_whitespace(tokens, idx + 1) 54 | token = tokens[idx] 55 | idx += 1 56 | current_range['key'] = eval(token.value) 57 | self.ranges.append(current_range) 58 | return idx 59 | 60 | def parse_comparison(self, current_range, token): 61 | if isinstance(token, stypes.Comparison): 62 | operator = str(token.token_next_by_type(0, ttypes.Comparison)) 63 | if '>=' == operator: 64 | current_range['from'] = filter_translator.eval_value(token.right) 65 | elif '<' == operator: 66 | current_range['to'] = filter_translator.eval_value(token.right) 67 | else: 68 | raise Exception('unexpected: %s' % token) 69 | self.set_field(token.left.as_field_name()) 70 | else: 71 | raise Exception('unexpected: %s' % token) 72 | 73 | def on_ELSE(self, tokens, idx): 74 | raise Exception('else is not supported') 75 | 76 | def set_field(self, field): 77 | if self.field is None: 78 | self.field = field 79 | elif self.field != field: 80 | raise Exception('can only case when on single field: %s %s' % (self.field, field)) 81 | else: 82 | self.field = field 83 | 84 | def build(self): 85 | if not self.field or not self.ranges: 86 | raise Exception('internal error') 87 | return { 88 | 'range': { 89 | 'field': self.field, 90 | 'ranges': self.ranges 91 | } 92 | } 93 | 94 | 95 | class CaseWhenFiltersTranslator(object): 96 | def __init__(self): 97 | self.filters = {} 98 | self.other_bucket_key = None 99 | 100 | def on_CASE(self, tokens): 101 | idx = 0 102 | while idx < len(tokens): 103 | token = tokens[idx] 104 | idx += 1 105 | if token.is_whitespace(): 106 | continue 107 | if 'WHEN' == token.value.upper(): 108 | idx = self.on_WHEN(tokens, idx) 109 | elif 'ELSE' == token.value.upper(): 110 | idx = self.on_ELSE(tokens, idx) 111 | elif 'END' == token.value.upper(): 112 | break 113 | else: 114 | raise Exception('unexpected: %s' % token) 115 | return self.build() 116 | 117 | def on_WHEN(self, tokens, idx): 118 | filter_tokens = [] 119 | bucket_key = None 120 | while idx < len(tokens): 121 | token = tokens[idx] 122 | idx += 1 123 | if token.is_whitespace(): 124 | continue 125 | if ttypes.Keyword == token.ttype and 'THEN' == token.value.upper(): 126 | idx = skip_whitespace(tokens, idx + 1) 127 | bucket_key = eval(tokens[idx].value) 128 | idx += 1 129 | break 130 | filter_tokens.append(token) 131 | if not filter_tokens: 132 | raise Exception('case when can not have empty filter') 133 | self.filters[bucket_key] = filter_translator.create_compound_filter(filter_tokens) 134 | return idx 135 | 136 | def on_ELSE(self, tokens, idx): 137 | idx = skip_whitespace(tokens, idx + 1) 138 | self.other_bucket_key = eval(tokens[idx].value) 139 | idx += 1 140 | return idx 141 | 142 | def set_field(self, field): 143 | if self.field is None: 144 | self.field = field 145 | elif self.field != field: 146 | raise Exception('can only case when on single field: %s %s' % (self.field, field)) 147 | else: 148 | self.field = field 149 | 150 | def build(self): 151 | if not self.filters: 152 | raise Exception('internal error') 153 | agg = {'filters': {'filters': self.filters}} 154 | if self.other_bucket_key: 155 | agg['filters']['other_bucket_key'] = self.other_bucket_key 156 | return agg 157 | 158 | 159 | def skip_whitespace(tokens, idx): 160 | while idx < len(tokens): 161 | token = tokens[idx] 162 | if token.is_whitespace(): 163 | idx += 1 164 | continue 165 | else: 166 | break 167 | return idx 168 | -------------------------------------------------------------------------------- /es_sql/executors/translators/doc_script_translator.py: -------------------------------------------------------------------------------- 1 | from es_sql.sqlparse import sql as stypes 2 | from es_sql.sqlparse import tokens as ttypes 3 | 4 | 5 | def translate_script(tokens): 6 | agg = translate_as_multiple_value(tokens) 7 | if len(agg['fields']) == 0: 8 | raise Exception('doc script does not reference any field') 9 | elif len(agg['fields']) == 1: 10 | return translate_as_single_value(tokens) 11 | else: 12 | agg.pop('fields') 13 | return agg 14 | 15 | 16 | def translate_as_multiple_value(tokens): 17 | agg = {'fields': set(), 'script': {'lang': 'expression', 'inline': ''}} 18 | _translate(agg, tokens, is_single_value=False) 19 | return agg 20 | 21 | 22 | def translate_as_single_value(tokens): 23 | agg = {'fields': set(), 'script': {'lang': 'expression', 'inline': ''}} 24 | _translate(agg, tokens, is_single_value=True) 25 | agg['field'] = list(agg.pop('fields'))[0] 26 | return agg 27 | 28 | 29 | def _translate(agg, tokens, is_single_value): 30 | for token in tokens: 31 | if token.ttype == ttypes.Keyword and 'AND' == token.value.upper(): 32 | agg['script']['inline'] = '%s%s' % ( 33 | agg['script']['inline'], '&&') 34 | elif token.ttype == ttypes.Keyword and 'OR' == token.value.upper(): 35 | agg['script']['inline'] = '%s%s' % ( 36 | agg['script']['inline'], '||') 37 | elif isinstance(token, stypes.Function): 38 | agg['script']['inline'] = '%s%s(' % ( 39 | agg['script']['inline'], token.get_function_name()) 40 | _translate(agg, token.get_parameters(), is_single_value) 41 | agg['script']['inline'] = '%s)' % ( 42 | agg['script']['inline']) 43 | elif token.is_group(): 44 | _translate(agg, token.tokens, is_single_value) 45 | else: 46 | if token.is_field(): 47 | field_name = token.as_field_name() 48 | agg['fields'].add(field_name) 49 | if is_single_value: 50 | agg['script']['inline'] = '%s%s' % ( 51 | agg['script']['inline'], '_value') 52 | else: 53 | agg['script']['inline'] = '%s%s' % ( 54 | agg['script']['inline'], "doc['%s'].value" % field_name) 55 | else: 56 | agg['script']['inline'] = '%s%s' % ( 57 | agg['script']['inline'], token.value) 58 | -------------------------------------------------------------------------------- /es_sql/executors/translators/filter_translator.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import re 3 | import time 4 | 5 | import datetime 6 | 7 | from es_sql.sqlparse import datetime_evaluator 8 | from es_sql.sqlparse import sql as stypes 9 | from es_sql.sqlparse import tokens as ttypes 10 | 11 | LOGGER = logging.getLogger(__name__) 12 | 13 | 14 | def create_compound_filter(tokens, tables=None): 15 | tables = tables or {} 16 | idx = 0 17 | current_filter = None 18 | last_filter = None 19 | logic_op = None 20 | is_not = False 21 | while idx < len(tokens): 22 | token = tokens[idx] 23 | idx += 1 24 | if token.is_whitespace(): 25 | continue 26 | if isinstance(token, stypes.Comparison) or isinstance(token, stypes.Parenthesis): 27 | if isinstance(token, stypes.Comparison): 28 | new_filter = create_comparision_filter(token, tables) 29 | elif isinstance(token, stypes.Parenthesis): 30 | new_filter = create_compound_filter(token.tokens[1:-1]) 31 | else: 32 | raise Exception('unexpected: %s' % token) 33 | try: 34 | if not logic_op and not current_filter: 35 | if is_not: 36 | is_not = False 37 | new_filter = {'bool': {'must_not': [new_filter]}} 38 | current_filter = new_filter 39 | elif 'OR' == logic_op: 40 | if is_not: 41 | is_not = False 42 | new_filter = {'bool': {'must_not': [new_filter]}} 43 | current_filter = {'bool': {'should': [current_filter, new_filter]}} 44 | elif 'AND' == logic_op: 45 | merged_filter = try_merge_filter(new_filter, last_filter) 46 | if merged_filter: 47 | last_filter.clear() 48 | last_filter.update(merged_filter) 49 | elif is_not: 50 | is_not = False 51 | if isinstance(current_filter, dict) and ['bool'] == current_filter.keys(): 52 | current_filter['bool']['must_not'] = current_filter['bool'].get('must_not', []) 53 | current_filter['bool']['must_not'].append(new_filter) 54 | else: 55 | current_filter = {'bool': {'filter': [current_filter], 'must_not': [new_filter]}} 56 | else: 57 | if isinstance(current_filter, dict) and ['bool'] == current_filter.keys(): 58 | current_filter['bool']['filter'] = current_filter['bool'].get('filter', []) 59 | current_filter['bool']['filter'].append(new_filter) 60 | else: 61 | current_filter = {'bool': {'filter': [current_filter, new_filter]}} 62 | else: 63 | raise Exception('unexpected: %s' % token) 64 | finally: 65 | last_filter = current_filter 66 | elif ttypes.Keyword == token.ttype: 67 | if 'OR' == token.value.upper(): 68 | if logic_op == 'AND': 69 | raise Exception('OR can only follow OR/NOT, otherwise use () to compound') 70 | logic_op = 'OR' 71 | elif 'AND' == token.value.upper(): 72 | if logic_op == 'OR': 73 | raise Exception('AND can only follow AND/NOT, otherwise use () to compound') 74 | logic_op = 'AND' 75 | elif 'NOT' == token.value.upper(): 76 | is_not = True 77 | else: 78 | raise Exception('unexpected: %s' % token) 79 | else: 80 | raise Exception('unexpected: %s' % token) 81 | return current_filter 82 | 83 | 84 | def try_merge_filter(new_filter, last_filter): 85 | if ['range'] == new_filter.keys() and ['range'] == last_filter.keys(): 86 | for k in new_filter['range']: 87 | for o in new_filter['range'][k]: 88 | if last_filter['range'].get(k, {}).get(o): 89 | return None 90 | for o in new_filter['range'][k]: 91 | last_filter['range'][k] = last_filter['range'].get(k, {}) 92 | last_filter['range'][k][o] = new_filter['range'][k][o] 93 | return dict(last_filter) 94 | if ['type'] == last_filter.keys() and ['ids'] == new_filter.keys(): 95 | last_filter, new_filter = new_filter, last_filter 96 | if ['type'] == new_filter.keys() and ['ids'] == last_filter.keys(): 97 | if not last_filter['ids'].get('type'): 98 | last_filter['ids']['type'] = new_filter['type']['value'] 99 | return dict(last_filter) 100 | return None 101 | 102 | 103 | def create_comparision_filter(comparison, tables=None): 104 | tables = tables or {} 105 | if not isinstance(comparison, stypes.Comparison): 106 | raise Exception('unexpected: %s' % comparison) 107 | operator = comparison.operator 108 | right_operand = comparison.right 109 | left_operand = comparison.left 110 | if operator in ('>', '>=', '<', '<='): 111 | right_operand_as_value = eval_value(right_operand) 112 | left_operand_as_value = eval_value(left_operand) 113 | if left_operand.is_field() and right_operand_as_value is not None: 114 | operator_as_str = {'>': 'gt', '>=': 'gte', '<': 'lt', '<=': 'lte'}[operator] 115 | add_field_hint_to_parameter(right_operand_as_value, left_operand.as_field_name()) 116 | return { 117 | 'range': {left_operand.as_field_name(): {operator_as_str: right_operand_as_value}}} 118 | elif right_operand.is_field() and left_operand_as_value is not None: 119 | operator_as_str = {'>': 'lte', '>=': 'lt', '<': 'gte', '<=': 'gt'}[operator] 120 | add_field_hint_to_parameter(left_operand_as_value, right_operand.as_field_name()) 121 | return { 122 | 'range': {right_operand.as_field_name(): {operator_as_str: left_operand_as_value}}} 123 | else: 124 | raise Exception('complex range condition not supported: %s' % comparison) 125 | elif '=' == operator: 126 | cross_table_eq = eval_cross_table_eq(tables, left_operand, right_operand) 127 | if cross_table_eq: 128 | return cross_table_eq 129 | right_operand_as_value = eval_value(right_operand) 130 | left_operand_as_value = eval_value(left_operand) 131 | if left_operand.is_field() and right_operand_as_value is not None: 132 | pass 133 | elif right_operand.is_field() and left_operand_as_value is not None: 134 | right_operand_as_value = left_operand_as_value 135 | left_operand, right_operand = right_operand, left_operand 136 | else: 137 | raise Exception('complex equal condition not supported: %s' % comparison) 138 | field = left_operand.as_field_name() 139 | if '_type' == field: 140 | return {'type': {'value': right_operand_as_value}} 141 | elif '_id' == field: 142 | return {'ids': {'value': [right_operand_as_value]}} 143 | else: 144 | add_field_hint_to_parameter(right_operand_as_value, field) 145 | return {'term': {field: right_operand_as_value}} 146 | elif operator.upper() in ('LIKE', 'ILIKE'): 147 | right_operand = eval_value(right_operand) 148 | field = left_operand.as_field_name() 149 | if isinstance(right_operand, SqlParameter): 150 | add_field_hint_to_parameter(right_operand, field) 151 | return {'wildcard': {field: right_operand}} 152 | else: 153 | return {'wildcard': {field: right_operand.replace('%', '*').replace('_', '?')}} 154 | elif operator in ('!=', '<>'): 155 | if right_operand.is_field(): 156 | left_operand, right_operand = right_operand, left_operand 157 | elif left_operand.is_field(): 158 | pass 159 | else: 160 | raise Exception('complex not equal condition not supported: %s' % comparison) 161 | right_operand = eval_value(right_operand) 162 | add_field_hint_to_parameter(right_operand, left_operand.as_field_name()) 163 | return {'bool': {'must_not': {'term': {left_operand.as_field_name(): right_operand}}}} 164 | elif 'IN' == operator.upper(): 165 | values = eval_value(right_operand) 166 | if not isinstance(values, SqlParameter): 167 | if not isinstance(values, tuple): 168 | values = (values,) 169 | values = list(values) 170 | if '_id' == left_operand.as_field_name(): 171 | return {'ids': {'value': values}} 172 | else: 173 | add_field_hint_to_parameter(values, left_operand.as_field_name()) 174 | return {'terms': {left_operand.as_field_name(): values}} 175 | elif re.match('IS\s+NOT', operator.upper()): 176 | if 'NULL' != right_operand.value.upper(): 177 | raise Exception('unexpected: %s' % repr(right_operand)) 178 | return {'exists': {'field': left_operand.as_field_name()}} 179 | elif 'IS' == operator.upper(): 180 | if 'NULL' != right_operand.value.upper(): 181 | raise Exception('unexpected: %s' % repr(right_operand)) 182 | return {'bool': {'must_not': {'exists': {'field': left_operand.as_field_name()}}}} 183 | else: 184 | raise Exception('unexpected operator: %s' % operator) 185 | 186 | 187 | def eval_cross_table_eq(tables, left, right): 188 | if isinstance(right, stypes.DotName) and len(right.tokens) == 3 and '.' == right.tokens[1].value: 189 | if isinstance(left, stypes.DotName) and len(left.tokens) == 3 and '.' == left.tokens[1].value: 190 | right_table = right.tokens[0].as_field_name() 191 | left_table = left.tokens[0].as_field_name() 192 | if True == tables.get(right_table): 193 | field_ref = FieldRef(left.tokens[0].as_field_name(), left.tokens[2].as_field_name()) 194 | return {'term': {right.tokens[2].as_field_name(): field_ref}} 195 | elif True == tables.get(left_table): 196 | field_ref = FieldRef(right.tokens[0].as_field_name(), right.tokens[2].as_field_name()) 197 | return {'term': {left.tokens[2].as_field_name(): field_ref}} 198 | return None 199 | 200 | 201 | class FieldRef(object): 202 | def __init__(self, table, field): 203 | self.table = table 204 | self.field = field 205 | 206 | def __repr__(self): 207 | return '${%s.%s}' % (self.table, self.field) 208 | 209 | def __str__(self): 210 | return repr(self) 211 | 212 | def __unicode__(self): 213 | return repr(self) 214 | 215 | 216 | def add_field_hint_to_parameter(parameter, field): 217 | if isinstance(parameter, SqlParameter): 218 | parameter.field_hint = field 219 | 220 | 221 | def eval_value(token): 222 | val = str(token) 223 | if token.ttype == ttypes.Name.Placeholder: 224 | return SqlParameter(token.value[2:-2]) 225 | try: 226 | val = eval(val, {}, datetime_evaluator.datetime_functions()) 227 | if isinstance(val, datetime.datetime): 228 | return long(time.mktime(val.timetuple()) * 1000) 229 | return val 230 | except: 231 | return None 232 | 233 | 234 | class SqlParameter(object): 235 | def __init__(self, parameter_name): 236 | self.parameter_name = parameter_name 237 | self.field_hint = None 238 | 239 | def __repr__(self): 240 | return ''.join(['%(', self.parameter_name, ')s']) 241 | 242 | def __unicode__(self): 243 | return repr(self) 244 | 245 | def __str__(self): 246 | return repr(self) 247 | -------------------------------------------------------------------------------- /es_sql/executors/translators/group_by_translator.py: -------------------------------------------------------------------------------- 1 | from es_sql.sqlparse import sql as stypes 2 | from . import case_when_translator 3 | from . import doc_script_translator 4 | from . import filter_translator 5 | 6 | 7 | def translate_group_by(group_by_map): 8 | aggs = {} 9 | tail_aggs = aggs 10 | for group_by_name, group_by in group_by_map.iteritems(): 11 | if isinstance(group_by, stypes.Parenthesis): 12 | if len(group_by.tokens > 3): 13 | raise Exception('unexpected: %s' % group_by) 14 | group_by = group_by.tokens[1] 15 | if group_by.is_field(): 16 | tail_aggs = append_terms_aggs(tail_aggs, group_by, group_by_name) 17 | elif isinstance(group_by, stypes.Case): 18 | tail_aggs = append_range_aggs(tail_aggs, group_by, group_by_name) 19 | elif isinstance(group_by, stypes.Function): 20 | sql_function_name = group_by.tokens[0].value.upper() 21 | if sql_function_name in ('DATE_TRUNC', 'TO_CHAR'): 22 | tail_aggs = append_date_histogram_aggs(tail_aggs, group_by, group_by_name) 23 | elif 'HISTOGRAM' == sql_function_name: 24 | tail_aggs = append_histogram_aggs(tail_aggs, group_by, group_by_name) 25 | else: 26 | tail_aggs = append_terms_aggs_with_script(tail_aggs, group_by, group_by_name) 27 | elif isinstance(group_by, stypes.Expression): 28 | tail_aggs = append_terms_aggs_with_script(tail_aggs, group_by, group_by_name) 29 | elif isinstance(group_by, stypes.Where): 30 | tail_aggs = append_filter_aggs(tail_aggs, group_by, group_by_name) 31 | else: 32 | raise Exception('unexpected: %s' % repr(group_by)) 33 | return aggs, tail_aggs 34 | 35 | 36 | def append_terms_aggs(tail_aggs, group_by, group_by_name): 37 | new_tail_aggs = {} 38 | tail_aggs[group_by_name] = { 39 | 'terms': {'field': group_by.as_field_name(), 'size': 0}, 40 | 'aggs': new_tail_aggs 41 | } 42 | return new_tail_aggs 43 | 44 | 45 | def append_terms_aggs_with_script(tail_aggs, group_by, group_by_name): 46 | new_tail_aggs = {} 47 | script = doc_script_translator.translate_script([group_by]) 48 | script['size'] = 0 49 | tail_aggs[group_by_name] = { 50 | 'terms': script, 51 | 'aggs': new_tail_aggs 52 | } 53 | return new_tail_aggs 54 | 55 | 56 | def append_date_histogram_aggs(tail_aggs, group_by, group_by_name): 57 | new_tail_aggs = {} 58 | date_format = None 59 | if 'TO_CHAR' == group_by.tokens[0].value.upper(): 60 | to_char_params = list(group_by.get_parameters()) 61 | sql_function = to_char_params[0] 62 | date_format = to_char_params[1].value[1:-1]\ 63 | .replace('%Y', 'yyyy')\ 64 | .replace('%m', 'MM')\ 65 | .replace('%d', 'dd')\ 66 | .replace('%H', 'hh')\ 67 | .replace('%M', 'mm')\ 68 | .replace('%S', 'ss') 69 | else: 70 | sql_function = group_by 71 | if 'DATE_TRUNC' == sql_function.tokens[0].value.upper(): 72 | parameters = list(sql_function.get_parameters()) 73 | if len(parameters) != 2: 74 | raise Exception('incorrect parameters count: %s' % list(parameters)) 75 | interval, field = parameters 76 | tail_aggs[group_by_name] = { 77 | 'date_histogram': { 78 | 'field': field.as_field_name(), 79 | 'time_zone': '+08:00', 80 | 'interval': eval(interval.value) 81 | }, 82 | 'aggs': new_tail_aggs 83 | } 84 | if date_format: 85 | tail_aggs[group_by_name]['date_histogram']['format'] = date_format 86 | else: 87 | raise Exception('unexpected: %s' % repr(sql_function)) 88 | return new_tail_aggs 89 | 90 | 91 | def append_histogram_aggs(tail_aggs, group_by, group_by_name): 92 | new_tail_aggs = {} 93 | parameters = tuple(group_by.get_parameters()) 94 | historgram = {'field': parameters[0].as_field_name(), 'interval': eval(parameters[1].value)} 95 | if len(parameters) == 3: 96 | historgram.update(eval(eval(parameters[2].value))) 97 | tail_aggs[group_by_name] = { 98 | 'histogram': historgram, 99 | 'aggs': new_tail_aggs 100 | } 101 | return new_tail_aggs 102 | 103 | 104 | def append_filter_aggs(tail_aggs, where, group_by_name): 105 | new_tail_aggs = {} 106 | filter = filter_translator.create_compound_filter(where.tokens[1:]) 107 | tail_aggs[group_by_name] = { 108 | 'filter': filter, 109 | 'aggs': new_tail_aggs 110 | } 111 | return new_tail_aggs 112 | 113 | 114 | def append_range_aggs(tail_aggs, case_when, group_by_name): 115 | new_tail_aggs = {} 116 | case_when_aggs = case_when_translator.translate_case_when(case_when) 117 | tail_aggs[group_by_name] = case_when_aggs 118 | tail_aggs[group_by_name]['aggs'] = new_tail_aggs 119 | return new_tail_aggs 120 | -------------------------------------------------------------------------------- /es_sql/executors/translators/join_translator.py: -------------------------------------------------------------------------------- 1 | from . import filter_translator 2 | 3 | 4 | def translate_join(sql_select): 5 | join_table = sql_select.join_table 6 | if join_table in sql_select.joinable_results: 7 | return translate_client_side_join(join_table, sql_select) 8 | elif join_table in sql_select.joinable_queries: 9 | other_executor = sql_select.joinable_queries[join_table] 10 | template_filter = filter_translator.create_compound_filter( 11 | sql_select.join_conditions, sql_select.tables()) 12 | term_filter = template_filter.get('term') 13 | if not term_filter: 14 | raise Exception('server side join can only on simple equal condition') 15 | if len(term_filter.keys()) > 1: 16 | raise Exception('server side join can only on simple equal condition') 17 | field = term_filter.keys()[0] 18 | field_ref = term_filter[field] 19 | return [{ 20 | 'filterjoin': { 21 | field: { 22 | 'indices': '%s*' % other_executor.sql_select.from_table, 23 | 'path': field_ref.field, 24 | 'query': other_executor.request['query'] 25 | } 26 | } 27 | }] 28 | else: 29 | raise Exception('join table not found: %s' % join_table) 30 | 31 | 32 | def translate_client_side_join(join_table, sql_select): 33 | template_filter = filter_translator.create_compound_filter( 34 | sql_select.join_conditions, sql_select.tables()) 35 | rows = sql_select.joinable_results[join_table] 36 | terms_filter = optimize_as_terms(template_filter, rows) 37 | if terms_filter: 38 | return [terms_filter] 39 | template_filter_str = repr(template_filter) 40 | join_filters = [] 41 | for row in rows: 42 | this_filter_as_str = template_filter_str 43 | for k, v in row.iteritems(): 44 | variable_name = '${%s.%s}' % (join_table, k) 45 | this_filter_as_str = this_filter_as_str.replace(variable_name, 46 | "'%s'" % v if isinstance(v, basestring) else v) 47 | join_filters.append(eval(this_filter_as_str)) 48 | return join_filters 49 | 50 | 51 | def optimize_as_terms(template_filter, rows): 52 | if not template_filter.get('term'): 53 | return None 54 | term_filter = template_filter['term'] 55 | if len(term_filter) > 1: 56 | return None 57 | field = term_filter.keys()[0] 58 | field_ref = term_filter[field] 59 | terms = [row[field_ref.field] for row in rows] 60 | return {'terms': {field: terms}} 61 | -------------------------------------------------------------------------------- /es_sql/executors/translators/metric_translator.py: -------------------------------------------------------------------------------- 1 | import json 2 | 3 | from es_sql.sqlparse import sql as stypes 4 | from es_sql.sqlparse import tokens as ttypes 5 | from . import bucket_script_translator 6 | 7 | 8 | def translate_metrics(sql_select): 9 | metric_request = {} 10 | metric_selector = {} 11 | for projection_name, projection in sql_select.projections.iteritems(): 12 | if projection_name in sql_select.group_by: 13 | continue 14 | if ttypes.Wildcard == projection.ttype: 15 | continue 16 | request, selector = translate_metric(sql_select, projection, projection_name) 17 | if request: 18 | projection_mapped_to = request.pop('_projection_mapped_to_', None) 19 | if projection_mapped_to: 20 | bucket_name = projection_mapped_to[0] 21 | metric_request[bucket_name] = request 22 | sql_select.projection_mapping[projection_name] = '.'.join(projection_mapped_to) 23 | else: 24 | metric_request[projection_name] = request 25 | if selector: 26 | metric_selector[projection_name] = selector 27 | return metric_request, metric_selector 28 | 29 | 30 | def translate_metric(sql_select, sql_function, projection_name): 31 | buckets_names = sql_select.buckets_names 32 | if isinstance(sql_function, stypes.Function): 33 | sql_function_name = sql_function.tokens[0].value.upper() 34 | if 'COUNT' == sql_function_name: 35 | return translate_count(buckets_names, sql_function, projection_name) 36 | elif sql_function_name in ('MAX', 'MIN', 'AVG', 'SUM'): 37 | return translate_min_max_avg_sum(buckets_names, sql_function, projection_name) 38 | elif sql_function_name in ('CSUM', 'DERIVATIVE'): 39 | return translate_csum_derivative(buckets_names, sql_function, projection_name) 40 | elif sql_function_name in ('MOVING_AVG', 'SERIAL_DIFF'): 41 | return translate_moving_avg_serial_diff(buckets_names, sql_function, projection_name) 42 | elif sql_function_name in ( 43 | 'SUM_OF_SQUARES', 'VARIANCE', 'STD_DEVIATION', 'STD_DEVIATION_UPPER_BOUND', 'STD_DEVIATION_LOWER_BOUND'): 44 | return translate_extended_stats(buckets_names, sql_function, projection_name) 45 | else: 46 | raise Exception('unsupported function: %s' % repr(sql_function)) 47 | elif isinstance(sql_function, stypes.Expression): 48 | return translate_expression(sql_select, sql_function, projection_name) 49 | else: 50 | raise Exception('unexpected: %s' % repr(sql_function)) 51 | 52 | 53 | def translate_expression(sql_select, sql_function, projection_name): 54 | selector = lambda bucket: bucket[projection_name]['value'] 55 | return {'bucket_script': bucket_script_translator.translate_script(sql_select, sql_function.tokens)}, selector 56 | 57 | def translate_count(buckets_names, sql_function, projection_name): 58 | params = sql_function.get_parameters() 59 | if len(params) == 1 and ttypes.Wildcard == params[0].ttype: 60 | selector = lambda bucket: bucket['doc_count'] 61 | return None, selector 62 | else: 63 | count_keyword = sql_function.tokens[1].token_next_by_type(0, ttypes.Keyword) 64 | selector = lambda bucket: bucket[projection_name]['value'] 65 | if count_keyword: 66 | if 'DISTINCT' == count_keyword.value.upper(): 67 | request = {'cardinality': {'field': params[-1].as_field_name()}} 68 | return request, selector 69 | else: 70 | raise Exception('unexpected: %s' % repr(count_keyword)) 71 | else: 72 | request = {'value_count': {'field': params[0].as_field_name()}} 73 | return request, selector 74 | 75 | 76 | def translate_min_max_avg_sum(buckets_names, sql_function, projection_name): 77 | params = sql_function.get_parameters() 78 | sql_function_name = sql_function.tokens[0].value.upper() 79 | if len(params) != 1: 80 | raise Exception('unexpected: %s' % repr(sql_function)) 81 | selector = lambda bucket: bucket[projection_name]['value'] 82 | field_name = params[0].as_field_name() 83 | buckets_path = buckets_names.get(field_name) 84 | if buckets_path: 85 | request = {'%s_bucket' % sql_function_name.lower(): {'buckets_path': buckets_path}} 86 | else: 87 | request = {sql_function_name.lower(): {'field': field_name}} 88 | return request, selector 89 | 90 | 91 | def translate_csum_derivative(buckets_names, sql_function, projection_name): 92 | sql_function_name = sql_function.tokens[0].value.upper() 93 | params = sql_function.get_parameters() 94 | selector = lambda bucket: bucket[projection_name]['value'] if projection_name in bucket else None 95 | field_name = params[0].as_field_name() 96 | buckets_path = buckets_names.get(field_name) 97 | if not buckets_path: 98 | raise Exception('field not found: %s' % field_name) 99 | metric_type = { 100 | 'CSUM': 'cumulative_sum', 101 | 'CUMULATIVE_SUM': 'cumulative_sum', 102 | 'DERIVATIVE': 'derivative' 103 | }[sql_function_name] 104 | request = {metric_type: {'buckets_path': buckets_path}} 105 | return request, selector 106 | 107 | 108 | def translate_moving_avg_serial_diff(buckets_names, sql_function, projection_name): 109 | sql_function_name = sql_function.tokens[0].value.upper() 110 | params = sql_function.get_parameters() 111 | selector = lambda bucket: bucket[projection_name]['value'] if projection_name in bucket else None 112 | field_name = params[0].as_field_name() 113 | buckets_path = buckets_names.get(field_name) 114 | if not buckets_path: 115 | raise Exception('field not found: %s' % field_name) 116 | request = {sql_function_name.lower(): {'buckets_path': buckets_path}} 117 | if len(params) > 1: 118 | request[sql_function_name.lower()].update(json.loads(params[1].value[1:-1])) 119 | return request, selector 120 | 121 | def translate_extended_stats(buckets_names, sql_function, projection_name): 122 | sql_function_name = sql_function.tokens[0].value.upper() 123 | params = sql_function.get_parameters() 124 | if len(params) != 1: 125 | raise Exception('unexpected: %s' % str(sql_function)) 126 | if not params[0].is_field(): 127 | raise Exception('unexpected: %s' % str(sql_function)) 128 | field = params[0].as_field_name() 129 | overridden_bucket_name = '%s_extended_stats' % field 130 | if 'STD_DEVIATION_UPPER_BOUND' == sql_function_name: 131 | _projection_mapped_to_ = (overridden_bucket_name, 'std_deviation_bounds.upper') 132 | selector = lambda bucket: bucket[overridden_bucket_name]['std_deviation_bounds']['upper'] 133 | elif 'STD_DEVIATION_LOWER_BOUND' == sql_function_name: 134 | _projection_mapped_to_ = (overridden_bucket_name, 'std_deviation_bounds.lower') 135 | selector = lambda bucket: bucket[overridden_bucket_name]['std_deviation_bounds']['lower'] 136 | else: 137 | _projection_mapped_to_ = (overridden_bucket_name, sql_function_name.lower()) 138 | selector = lambda bucket: bucket[overridden_bucket_name][sql_function_name.lower()] 139 | request = {'extended_stats': {'field': params[0].as_field_name()}, '_projection_mapped_to_': _projection_mapped_to_} 140 | return request, selector 141 | -------------------------------------------------------------------------------- /es_sql/executors/translators/sort_translator.py: -------------------------------------------------------------------------------- 1 | from es_sql.sqlparse import sql as stypes 2 | 3 | 4 | def translate_sort(sql_select): 5 | sort = [] 6 | for id in sql_select.order_by or []: 7 | asc_or_desc = 'asc' 8 | if type(id) == stypes.Identifier: 9 | if 'DESC' == id.tokens[-1].value.upper(): 10 | asc_or_desc = 'desc' 11 | field_name = id.tokens[0].as_field_name() 12 | projection = sql_select.projections.get(field_name) 13 | else: 14 | field_name = id.as_field_name() 15 | projection = sql_select.projections.get(field_name) 16 | group_by = sql_select.group_by.get(field_name) 17 | if group_by and group_by.is_field(): 18 | sort.append({'_term': asc_or_desc}) 19 | else: 20 | buckets_path = sql_select.buckets_names.get(field_name) 21 | if field_name in sql_select.projection_mapping: 22 | sort.append({sql_select.projection_mapping[field_name]: asc_or_desc}) 23 | elif buckets_path: 24 | sort.append({buckets_path: asc_or_desc}) 25 | else: 26 | sort.append({field_name: asc_or_desc}) 27 | return sort[0] if len(sort) == 1 else sort 28 | -------------------------------------------------------------------------------- /es_sql/sqlparse/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2008 Andi Albrecht, albrecht.andi@gmail.com 2 | # 3 | # This module is part of python-sqlparse and is released under 4 | # the BSD License: http://www.opensource.org/licenses/bsd-license.php. 5 | 6 | """Parse SQL statements.""" 7 | 8 | 9 | __version__ = '0.2.0.dev0' 10 | 11 | 12 | # Setup namespace 13 | from . import engine 14 | from . import filters 15 | from . import formatter 16 | from .compat import u 17 | 18 | 19 | def parse(sql, encoding=None): 20 | """Parse sql and return a list of statements. 21 | 22 | :param sql: A string containting one or more SQL statements. 23 | :param encoding: The encoding of the statement (optional). 24 | :returns: A tuple of :class:`~sqlparse.sql.Statement` instances. 25 | """ 26 | return tuple(parsestream(sql, encoding)) 27 | 28 | 29 | def parsestream(stream, encoding=None): 30 | """Parses sql statements from file-like object. 31 | 32 | :param stream: A file-like object. 33 | :param encoding: The encoding of the stream contents (optional). 34 | :returns: A generator of :class:`~sqlparse.sql.Statement` instances. 35 | """ 36 | stack = engine.FilterStack() 37 | stack.full_analyze() 38 | return stack.run(stream, encoding) 39 | 40 | 41 | def format(sql, **options): 42 | """Format *sql* according to *options*. 43 | 44 | Available options are documented in :ref:`formatting`. 45 | 46 | In addition to the formatting options this function accepts the 47 | keyword "encoding" which determines the encoding of the statement. 48 | 49 | :returns: The formatted SQL statement as string. 50 | """ 51 | encoding = options.pop('encoding', None) 52 | stack = engine.FilterStack() 53 | options = formatter.validate_options(options) 54 | stack = formatter.build_filter_stack(stack, options) 55 | stack.postprocess.append(filters.SerializerUnicode()) 56 | return ''.join(stack.run(sql, encoding)) 57 | 58 | 59 | def split(sql, encoding=None): 60 | """Split *sql* into single statements. 61 | 62 | :param sql: A string containting one or more SQL statements. 63 | :param encoding: The encoding of the statement (optional). 64 | :returns: A list of strings. 65 | """ 66 | stack = engine.FilterStack() 67 | stack.split_statements = True 68 | return [u(stmt).strip() for stmt in stack.run(sql, encoding)] 69 | 70 | 71 | from es_sql.sqlparse.engine.filter import StatementFilter 72 | 73 | 74 | def split2(stream): 75 | splitter = StatementFilter() 76 | return list(splitter.process(None, stream)) 77 | -------------------------------------------------------------------------------- /es_sql/sqlparse/compat.py: -------------------------------------------------------------------------------- 1 | """Python 2/3 compatibility. 2 | 3 | This module only exists to avoid a dependency on six 4 | for very trivial stuff. We only need to take care of 5 | string types, buffers and metaclasses. 6 | 7 | Parts of the code is copied directly from six: 8 | https://bitbucket.org/gutworth/six 9 | """ 10 | 11 | import sys 12 | 13 | PY2 = sys.version_info[0] == 2 14 | PY3 = sys.version_info[0] == 3 15 | 16 | if PY3: 17 | text_type = str 18 | string_types = (str,) 19 | from io import StringIO 20 | 21 | def u(s): 22 | return str(s) 23 | 24 | elif PY2: 25 | text_type = unicode 26 | string_types = (basestring,) 27 | from StringIO import StringIO # flake8: noqa 28 | 29 | def u(s): 30 | return unicode(s) 31 | 32 | 33 | # Directly copied from six: 34 | def with_metaclass(meta, *bases): 35 | """Create a base class with a metaclass.""" 36 | # This requires a bit of explanation: the basic idea is to make a dummy 37 | # metaclass for one level of class instantiation that replaces itself with 38 | # the actual metaclass. 39 | class metaclass(meta): 40 | def __new__(cls, name, this_bases, d): 41 | return meta(name, bases, d) 42 | return type.__new__(metaclass, 'temporary_class', (), {}) 43 | -------------------------------------------------------------------------------- /es_sql/sqlparse/datetime_evaluator.py: -------------------------------------------------------------------------------- 1 | import re 2 | import datetime 3 | 4 | NOW = None 5 | 6 | 7 | def datetime_functions(): 8 | functions = {'now': eval_now, 'today': eval_today, 'eval_datetime': eval_datetime, 'interval': eval_interval, 'timestamp': eval_timestamp} 9 | for k, v in functions.items(): 10 | functions[k.upper()] = v 11 | return functions 12 | 13 | 14 | def eval_now(): 15 | return NOW or datetime.datetime.now() 16 | 17 | def eval_today(): 18 | now = eval_now() 19 | return datetime.datetime(now.year, now.month, now.day, tzinfo=now.tzinfo) 20 | 21 | def eval_interval(datetime_value): 22 | return eval_datetime('INTERVAL', datetime_value) 23 | 24 | 25 | def eval_timestamp(datetime_value): 26 | return eval_datetime('TIMESTAMP', datetime_value) 27 | 28 | 29 | def eval_datetime(datetime_type, datetime_value): 30 | if 'INTERVAL' == datetime_type.upper(): 31 | try: 32 | return eval_interval(datetime_value) 33 | except: 34 | LOGGER.debug('failed to parse: %s' % datetime_value, exc_info=1) 35 | raise 36 | elif 'TIMESTAMP' == datetime_type.upper(): 37 | return datetime.datetime.strptime(datetime_value, '%Y-%m-%d %H:%M:%S') 38 | else: 39 | raise Exception('unsupported datetime type: %s' % datetime_type) 40 | 41 | 42 | PATTERN_INTERVAL = re.compile( 43 | r'((\d+)\s+(DAYS?|HOURS?|MINUTES?|SECONDS?))?\s*' 44 | r'((\d+)\s+(HOURS?|MINUTES?|SECONDS?))?\s*' 45 | r'((\d+)\s+(MINUTES?|SECONDS?))?\s*' 46 | r'((\d+)\s+(SECONDS?))?', re.IGNORECASE) 47 | 48 | 49 | def eval_interval(interval): 50 | interval = interval.strip() 51 | match = PATTERN_INTERVAL.match(interval) 52 | if not match or match.end() != len(interval): 53 | raise Exception('%s is invalid' % interval) 54 | timedelta = datetime.timedelta() 55 | last_pos = 0 56 | _, q1, u1, _, q2, u2, _, q3, u3, _, q4, u4 = match.groups() 57 | for quantity, unit in [(q1, u1), (q2, u2), (q3, u3), (q4, u4)]: 58 | if not quantity: 59 | continue 60 | unit = unit.upper() 61 | if unit in ('DAY', 'DAYS'): 62 | timedelta += datetime.timedelta(days=int(quantity)) 63 | elif unit in ('HOUR', 'HOURS'): 64 | timedelta += datetime.timedelta(hours=int(quantity)) 65 | elif unit in ('MINUTE', 'MINUTES'): 66 | timedelta += datetime.timedelta(minutes=int(quantity)) 67 | elif unit in ('SECOND', 'SECONDS'): 68 | timedelta += datetime.timedelta(seconds=int(quantity)) 69 | else: 70 | raise Exception('unknown unit: %s' % unit) 71 | return timedelta 72 | -------------------------------------------------------------------------------- /es_sql/sqlparse/datetime_evaluator_test.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | 3 | import datetime 4 | 5 | from es_sql.sqlparse import datetime_evaluator 6 | 7 | 8 | class TestEvalInterval(unittest.TestCase): 9 | def test_day(self): 10 | self.assertEqual(datetime.timedelta(days=1), datetime_evaluator.eval_interval('1 DAY')) 11 | 12 | def test_days(self): 13 | self.assertEqual(datetime.timedelta(days=2), datetime_evaluator.eval_interval('2 DAYS')) 14 | 15 | def test_hour(self): 16 | self.assertEqual(datetime.timedelta(hours=1), datetime_evaluator.eval_interval('1 hour')) 17 | 18 | def test_hours(self): 19 | self.assertEqual(datetime.timedelta(hours=2), datetime_evaluator.eval_interval('2 hours')) 20 | 21 | def test_minute(self): 22 | self.assertEqual(datetime.timedelta(minutes=1), datetime_evaluator.eval_interval('1 minute')) 23 | 24 | def test_minutes(self): 25 | self.assertEqual(datetime.timedelta(minutes=2), datetime_evaluator.eval_interval('2 minutes')) 26 | 27 | def test_second(self): 28 | self.assertEqual(datetime.timedelta(seconds=1), datetime_evaluator.eval_interval('1 second')) 29 | 30 | def test_seconds(self): 31 | self.assertEqual(datetime.timedelta(seconds=2), datetime_evaluator.eval_interval('2 seconds')) 32 | 33 | def test_1_day_2_hour_3_minute_4_second(self): 34 | self.assertEqual( 35 | datetime.timedelta(days=1, hours=2, minutes=3, seconds=4), 36 | datetime_evaluator.eval_interval('1 DAY 2 HOURS 3 MINUTES 4 SECONDS')) 37 | 38 | def test_invalid_character(self): 39 | try: 40 | datetime_evaluator.eval_interval('1 DAY 2 HOURD 3 MINUTE') 41 | except: 42 | return 43 | self.fail('should fail') 44 | -------------------------------------------------------------------------------- /es_sql/sqlparse/engine/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2008 Andi Albrecht, albrecht.andi@gmail.com 2 | # 3 | # This module is part of python-sqlparse and is released under 4 | # the BSD License: http://www.opensource.org/licenses/bsd-license.php. 5 | 6 | """filter""" 7 | 8 | from es_sql.sqlparse import lexer 9 | from . import grouping 10 | from .filter import StatementFilter 11 | 12 | # XXX remove this when cleanup is complete 13 | Filter = object 14 | 15 | 16 | class FilterStack(object): 17 | def __init__(self): 18 | self.preprocess = [] 19 | self.stmtprocess = [] 20 | self.postprocess = [] 21 | self.split_statements = False 22 | self._grouping = False 23 | 24 | def _flatten(self, stream): 25 | for token in stream: 26 | if token.is_group(): 27 | for t in self._flatten(token.tokens): 28 | yield t 29 | else: 30 | yield token 31 | 32 | def enable_grouping(self): 33 | self._grouping = True 34 | 35 | def full_analyze(self): 36 | self.enable_grouping() 37 | 38 | def run(self, sql, encoding=None): 39 | stream = lexer.tokenize(sql, encoding) 40 | # Process token stream 41 | if self.preprocess: 42 | for filter_ in self.preprocess: 43 | stream = filter_.process(self, stream) 44 | 45 | if self.stmtprocess or self.postprocess or self.split_statements \ 46 | or self._grouping: 47 | splitter = StatementFilter() 48 | stream = splitter.process(self, stream) 49 | 50 | if self._grouping: 51 | 52 | def _group(stream): 53 | for stmt in stream: 54 | grouping.group(stmt) 55 | yield stmt 56 | 57 | stream = _group(stream) 58 | 59 | if self.stmtprocess: 60 | 61 | def _run1(stream): 62 | ret = [] 63 | for stmt in stream: 64 | for filter_ in self.stmtprocess: 65 | filter_.process(self, stmt) 66 | ret.append(stmt) 67 | return ret 68 | 69 | stream = _run1(stream) 70 | 71 | if self.postprocess: 72 | 73 | def _run2(stream): 74 | for stmt in stream: 75 | stmt.tokens = list(self._flatten(stmt.tokens)) 76 | for filter_ in self.postprocess: 77 | stmt = filter_.process(self, stmt) 78 | yield stmt 79 | 80 | stream = _run2(stream) 81 | 82 | return stream 83 | -------------------------------------------------------------------------------- /es_sql/sqlparse/engine/filter.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | from es_sql.sqlparse.sql import Statement, Token 4 | from es_sql.sqlparse import tokens as T 5 | 6 | 7 | class StatementFilter: 8 | "Filter that split stream at individual statements" 9 | 10 | def __init__(self): 11 | self._in_declare = False 12 | self._in_dbldollar = False 13 | self._is_create = False 14 | self._begin_depth = 0 15 | 16 | def _reset(self): 17 | "Set the filter attributes to its default values" 18 | self._in_declare = False 19 | self._in_dbldollar = False 20 | self._is_create = False 21 | self._begin_depth = 0 22 | 23 | def _change_splitlevel(self, ttype, value): 24 | "Get the new split level (increase, decrease or remain equal)" 25 | # PostgreSQL 26 | if ttype == T.Name.Builtin \ 27 | and value.startswith('$') and value.endswith('$'): 28 | if self._in_dbldollar: 29 | self._in_dbldollar = False 30 | return -1 31 | else: 32 | self._in_dbldollar = True 33 | return 1 34 | elif self._in_dbldollar: 35 | return 0 36 | 37 | # ANSI 38 | if ttype not in T.Keyword: 39 | return 0 40 | 41 | unified = value.upper() 42 | 43 | if unified == 'DECLARE' and self._is_create and self._begin_depth == 0: 44 | self._in_declare = True 45 | return 1 46 | 47 | if unified == 'BEGIN': 48 | self._begin_depth += 1 49 | if self._in_declare or self._is_create: 50 | # FIXME(andi): This makes no sense. 51 | return 1 52 | return 0 53 | 54 | if unified in ('END IF', 'END FOR', 'END WHILE'): 55 | return -1 56 | 57 | if unified == 'END': 58 | # Should this respect a preceeding BEGIN? 59 | # In CASE ... WHEN ... END this results in a split level -1. 60 | self._begin_depth = max(0, self._begin_depth - 1) 61 | return -1 62 | 63 | if ttype is T.Keyword.DDL and unified.startswith('CREATE'): 64 | self._is_create = True 65 | return 0 66 | 67 | if unified in ('IF', 'FOR', 'WHILE') \ 68 | and self._is_create and self._begin_depth > 0: 69 | return 1 70 | 71 | # Default 72 | return 0 73 | 74 | def process(self, stack, stream): 75 | "Process the stream" 76 | consume_ws = False 77 | splitlevel = 0 78 | stmt = None 79 | stmt_tokens = [] 80 | 81 | # Run over all stream tokens 82 | for ttype, value in stream: 83 | # Yield token if we finished a statement and there's no whitespaces 84 | if consume_ws and ttype not in (T.Whitespace, T.Comment.Single): 85 | stmt.tokens = stmt_tokens 86 | yield stmt 87 | 88 | # Reset filter and prepare to process next statement 89 | self._reset() 90 | consume_ws = False 91 | splitlevel = 0 92 | stmt = None 93 | 94 | # Create a new statement if we are not currently in one of them 95 | if stmt is None: 96 | stmt = Statement() 97 | stmt_tokens = [] 98 | 99 | # Change current split level (increase, decrease or remain equal) 100 | splitlevel += self._change_splitlevel(ttype, value) 101 | 102 | # Append the token to the current statement 103 | stmt_tokens.append(Token(ttype, value)) 104 | 105 | # Check if we get the end of a statement 106 | if splitlevel <= 0 and ttype is T.Punctuation and value == ';': 107 | consume_ws = True 108 | 109 | # Yield pending statement (if any) 110 | if stmt is not None: 111 | stmt.tokens = stmt_tokens 112 | yield stmt -------------------------------------------------------------------------------- /es_sql/sqlparse/exceptions.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2012 Andi Albrecht, albrecht.andi@gmail.com 2 | # 3 | # This module is part of python-sqlparse and is released under 4 | # the BSD License: http://www.opensource.org/licenses/bsd-license.php. 5 | 6 | """Exceptions used in this package.""" 7 | 8 | 9 | class SQLParseError(Exception): 10 | """Base class for exceptions in this module.""" 11 | -------------------------------------------------------------------------------- /es_sql/sqlparse/formatter.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2008 Andi Albrecht, albrecht.andi@gmail.com 2 | # 3 | # This module is part of python-sqlparse and is released under 4 | # the BSD License: http://www.opensource.org/licenses/bsd-license.php. 5 | 6 | """SQL formatter""" 7 | 8 | from . import filters 9 | from .exceptions import SQLParseError 10 | 11 | 12 | def validate_options(options): 13 | """Validates options.""" 14 | kwcase = options.get('keyword_case', None) 15 | if kwcase not in [None, 'upper', 'lower', 'capitalize']: 16 | raise SQLParseError('Invalid value for keyword_case: %r' % kwcase) 17 | 18 | idcase = options.get('identifier_case', None) 19 | if idcase not in [None, 'upper', 'lower', 'capitalize']: 20 | raise SQLParseError('Invalid value for identifier_case: %r' % idcase) 21 | 22 | ofrmt = options.get('output_format', None) 23 | if ofrmt not in [None, 'sql', 'python', 'php']: 24 | raise SQLParseError('Unknown output format: %r' % ofrmt) 25 | 26 | strip_comments = options.get('strip_comments', False) 27 | if strip_comments not in [True, False]: 28 | raise SQLParseError('Invalid value for strip_comments: %r' 29 | % strip_comments) 30 | 31 | strip_ws = options.get('strip_whitespace', False) 32 | if strip_ws not in [True, False]: 33 | raise SQLParseError('Invalid value for strip_whitespace: %r' 34 | % strip_ws) 35 | 36 | truncate_strings = options.get('truncate_strings', None) 37 | if truncate_strings is not None: 38 | try: 39 | truncate_strings = int(truncate_strings) 40 | except (ValueError, TypeError): 41 | raise SQLParseError('Invalid value for truncate_strings: %r' 42 | % truncate_strings) 43 | if truncate_strings <= 1: 44 | raise SQLParseError('Invalid value for truncate_strings: %r' 45 | % truncate_strings) 46 | options['truncate_strings'] = truncate_strings 47 | options['truncate_char'] = options.get('truncate_char', '[...]') 48 | 49 | reindent = options.get('reindent', False) 50 | if reindent not in [True, False]: 51 | raise SQLParseError('Invalid value for reindent: %r' 52 | % reindent) 53 | elif reindent: 54 | options['strip_whitespace'] = True 55 | indent_tabs = options.get('indent_tabs', False) 56 | if indent_tabs not in [True, False]: 57 | raise SQLParseError('Invalid value for indent_tabs: %r' % indent_tabs) 58 | elif indent_tabs: 59 | options['indent_char'] = '\t' 60 | else: 61 | options['indent_char'] = ' ' 62 | indent_width = options.get('indent_width', 2) 63 | try: 64 | indent_width = int(indent_width) 65 | except (TypeError, ValueError): 66 | raise SQLParseError('indent_width requires an integer') 67 | if indent_width < 1: 68 | raise SQLParseError('indent_width requires an positive integer') 69 | options['indent_width'] = indent_width 70 | 71 | right_margin = options.get('right_margin', None) 72 | if right_margin is not None: 73 | try: 74 | right_margin = int(right_margin) 75 | except (TypeError, ValueError): 76 | raise SQLParseError('right_margin requires an integer') 77 | if right_margin < 10: 78 | raise SQLParseError('right_margin requires an integer > 10') 79 | options['right_margin'] = right_margin 80 | 81 | return options 82 | 83 | 84 | def build_filter_stack(stack, options): 85 | """Setup and return a filter stack. 86 | 87 | Args: 88 | stack: :class:`~sqlparse.filters.FilterStack` instance 89 | options: Dictionary with options validated by validate_options. 90 | """ 91 | # Token filter 92 | if options.get('keyword_case', None): 93 | stack.preprocess.append( 94 | filters.KeywordCaseFilter(options['keyword_case'])) 95 | 96 | if options.get('identifier_case', None): 97 | stack.preprocess.append( 98 | filters.IdentifierCaseFilter(options['identifier_case'])) 99 | 100 | if options.get('truncate_strings', None) is not None: 101 | stack.preprocess.append(filters.TruncateStringFilter( 102 | width=options['truncate_strings'], char=options['truncate_char'])) 103 | 104 | # After grouping 105 | if options.get('strip_comments', False): 106 | stack.enable_grouping() 107 | stack.stmtprocess.append(filters.StripCommentsFilter()) 108 | 109 | if options.get('strip_whitespace', False) \ 110 | or options.get('reindent', False): 111 | stack.enable_grouping() 112 | stack.stmtprocess.append(filters.StripWhitespaceFilter()) 113 | 114 | if options.get('reindent', False): 115 | stack.enable_grouping() 116 | stack.stmtprocess.append( 117 | filters.ReindentFilter(char=options['indent_char'], 118 | width=options['indent_width'])) 119 | 120 | if options.get('right_margin', False): 121 | stack.enable_grouping() 122 | stack.stmtprocess.append( 123 | filters.RightMarginFilter(width=options['right_margin'])) 124 | 125 | # Serializer 126 | if options.get('output_format'): 127 | frmt = options['output_format'] 128 | if frmt.lower() == 'php': 129 | fltr = filters.OutputPHPFilter() 130 | elif frmt.lower() == 'python': 131 | fltr = filters.OutputPythonFilter() 132 | else: 133 | fltr = None 134 | if fltr is not None: 135 | stack.postprocess.append(fltr) 136 | 137 | return stack 138 | -------------------------------------------------------------------------------- /es_sql/sqlparse/functions.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Created on 17/05/2012 3 | 4 | @author: piranna 5 | 6 | Several utility functions to extract info from the SQL sentences 7 | ''' 8 | 9 | from .filters import ColumnsSelect, Limit 10 | from .pipeline import Pipeline 11 | from .tokens import Keyword, Whitespace 12 | 13 | 14 | def getlimit(stream): 15 | """Function that return the LIMIT of a input SQL """ 16 | pipe = Pipeline() 17 | 18 | pipe.append(Limit()) 19 | 20 | result = pipe(stream) 21 | try: 22 | return int(result) 23 | except ValueError: 24 | return result 25 | 26 | 27 | def getcolumns(stream): 28 | """Function that return the colums of a SELECT query""" 29 | pipe = Pipeline() 30 | 31 | pipe.append(ColumnsSelect()) 32 | 33 | return pipe(stream) 34 | 35 | 36 | class IsType(object): 37 | """Functor that return is the statement is of a specific type""" 38 | def __init__(self, type): 39 | self.type = type 40 | 41 | def __call__(self, stream): 42 | for token_type, value in stream: 43 | if token_type not in Whitespace: 44 | return token_type in Keyword and value == self.type 45 | -------------------------------------------------------------------------------- /es_sql/sqlparse/ordereddict.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2009 Raymond Hettinger 2 | # 3 | # Permission is hereby granted, free of charge, to any person 4 | # obtaining a copy of this software and associated documentation files 5 | # (the "Software"), to deal in the Software without restriction, 6 | # including without limitation the rights to use, copy, modify, merge, 7 | # publish, distribute, sublicense, and/or sell copies of the Software, 8 | # and to permit persons to whom the Software is furnished to do so, 9 | # subject to the following conditions: 10 | # 11 | # The above copyright notice and this permission notice shall be 12 | # included in all copies or substantial portions of the Software. 13 | # 14 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 15 | # EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES 16 | # OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 17 | # NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT 18 | # HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, 19 | # WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 20 | # FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 21 | # OTHER DEALINGS IN THE SOFTWARE. 22 | 23 | from UserDict import DictMixin 24 | 25 | class OrderedDict(dict, DictMixin): 26 | 27 | def __init__(self, *args, **kwds): 28 | if len(args) > 1: 29 | raise TypeError('expected at most 1 arguments, got %d' % len(args)) 30 | try: 31 | self.__end 32 | except AttributeError: 33 | self.clear() 34 | self.update(*args, **kwds) 35 | 36 | def clear(self): 37 | self.__end = end = [] 38 | end += [None, end, end] # sentinel node for doubly linked list 39 | self.__map = {} # key --> [key, prev, next] 40 | dict.clear(self) 41 | 42 | def __setitem__(self, key, value): 43 | if key not in self: 44 | end = self.__end 45 | curr = end[1] 46 | curr[2] = end[1] = self.__map[key] = [key, curr, end] 47 | dict.__setitem__(self, key, value) 48 | 49 | def __delitem__(self, key): 50 | dict.__delitem__(self, key) 51 | key, prev, next = self.__map.pop(key) 52 | prev[2] = next 53 | next[1] = prev 54 | 55 | def __iter__(self): 56 | end = self.__end 57 | curr = end[2] 58 | while curr is not end: 59 | yield curr[0] 60 | curr = curr[2] 61 | 62 | def __reversed__(self): 63 | end = self.__end 64 | curr = end[1] 65 | while curr is not end: 66 | yield curr[0] 67 | curr = curr[1] 68 | 69 | def popitem(self, last=True): 70 | if not self: 71 | raise KeyError('dictionary is empty') 72 | if last: 73 | key = reversed(self).next() 74 | else: 75 | key = iter(self).next() 76 | value = self.pop(key) 77 | return key, value 78 | 79 | def __reduce__(self): 80 | items = [[k, self[k]] for k in self] 81 | tmp = self.__map, self.__end 82 | del self.__map, self.__end 83 | inst_dict = vars(self).copy() 84 | self.__map, self.__end = tmp 85 | if inst_dict: 86 | return (self.__class__, (items,), inst_dict) 87 | return self.__class__, (items,) 88 | 89 | def keys(self): 90 | return list(self) 91 | 92 | setdefault = DictMixin.setdefault 93 | update = DictMixin.update 94 | pop = DictMixin.pop 95 | values = DictMixin.values 96 | items = DictMixin.items 97 | iterkeys = DictMixin.iterkeys 98 | itervalues = DictMixin.itervalues 99 | iteritems = DictMixin.iteritems 100 | 101 | def __repr__(self): 102 | if not self: 103 | return '%s()' % (self.__class__.__name__,) 104 | return '%s(%r)' % (self.__class__.__name__, self.items()) 105 | 106 | def copy(self): 107 | return self.__class__(self) 108 | 109 | @classmethod 110 | def fromkeys(cls, iterable, value=None): 111 | d = cls() 112 | for key in iterable: 113 | d[key] = value 114 | return d 115 | 116 | def __eq__(self, other): 117 | if isinstance(other, OrderedDict): 118 | if len(self) != len(other): 119 | return False 120 | for p, q in zip(self.items(), other.items()): 121 | if p != q: 122 | return False 123 | return True 124 | return dict.__eq__(self, other) 125 | 126 | def __ne__(self, other): 127 | return not self == other 128 | 129 | def prepend(self, key, value): 130 | if key not in self: 131 | end = self.__end 132 | curr = end[1] 133 | curr[2] = end[1] = self.__map[key] = [key, curr, end] 134 | dict.__setitem__(self, key, value) -------------------------------------------------------------------------------- /es_sql/sqlparse/pipeline.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2011 Jesus Leganes "piranna", piranna@gmail.com 2 | # 3 | # This module is part of python-sqlparse and is released under 4 | # the BSD License: http://www.opensource.org/licenses/bsd-license.php. 5 | 6 | from types import GeneratorType 7 | 8 | 9 | class Pipeline(list): 10 | """Pipeline to process filters sequentially""" 11 | 12 | def __call__(self, stream): 13 | """Run the pipeline 14 | 15 | Return a static (non generator) version of the result 16 | """ 17 | 18 | # Run the stream over all the filters on the pipeline 19 | for filter in self: 20 | # Functions and callable objects (objects with '__call__' method) 21 | if callable(filter): 22 | stream = filter(stream) 23 | 24 | # Normal filters (objects with 'process' method) 25 | else: 26 | stream = filter.process(None, stream) 27 | 28 | # If last filter return a generator, staticalize it inside a list 29 | if isinstance(stream, GeneratorType): 30 | return list(stream) 31 | return stream 32 | -------------------------------------------------------------------------------- /es_sql/sqlparse/tokens.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2008 Andi Albrecht, albrecht.andi@gmail.com 2 | # 3 | # This module is part of python-sqlparse and is released under 4 | # the BSD License: http://www.opensource.org/licenses/bsd-license.php. 5 | 6 | # The Token implementation is based on pygment's token system written 7 | # by Georg Brandl. 8 | # http://pygments.org/ 9 | 10 | """Tokens""" 11 | 12 | 13 | class _TokenType(tuple): 14 | parent = None 15 | 16 | def split(self): 17 | buf = [] 18 | node = self 19 | while node is not None: 20 | buf.append(node) 21 | node = node.parent 22 | buf.reverse() 23 | return buf 24 | 25 | def __contains__(self, val): 26 | return val is not None and (self is val or val[:len(self)] == self) 27 | 28 | def __getattr__(self, val): 29 | if not val or not val[0].isupper(): 30 | return tuple.__getattribute__(self, val) 31 | new = _TokenType(self + (val,)) 32 | setattr(self, val, new) 33 | new.parent = self 34 | return new 35 | 36 | def __hash__(self): 37 | return hash(tuple(self)) 38 | 39 | def __repr__(self): 40 | return 'Token' + (self and '.' or '') + '.'.join(self) 41 | 42 | 43 | Token = _TokenType() 44 | 45 | # Special token types 46 | Text = Token.Text 47 | Whitespace = Text.Whitespace 48 | Newline = Whitespace.Newline 49 | Error = Token.Error 50 | # Text that doesn't belong to this lexer (e.g. HTML in PHP) 51 | Other = Token.Other 52 | 53 | # Common token types for source code 54 | Keyword = Token.Keyword 55 | Name = Token.Name 56 | Literal = Token.Literal 57 | String = Literal.String 58 | Number = Literal.Number 59 | Punctuation = Token.Punctuation 60 | Operator = Token.Operator 61 | Comparison = Operator.Comparison 62 | Wildcard = Token.Wildcard 63 | Comment = Token.Comment 64 | Assignment = Token.Assignement 65 | 66 | # Generic types for non-source code 67 | Generic = Token.Generic 68 | 69 | # String and some others are not direct childs of Token. 70 | # alias them: 71 | Token.Token = Token 72 | Token.String = String 73 | Token.Number = Number 74 | 75 | # SQL specific tokens 76 | DML = Keyword.DML 77 | DDL = Keyword.DDL 78 | Command = Keyword.Command 79 | 80 | Group = Token.Group 81 | Group.Parenthesis = Token.Group.Parenthesis 82 | Group.Comment = Token.Group.Comment 83 | Group.Where = Token.Group.Where 84 | -------------------------------------------------------------------------------- /es_sql/sqlparse/utils.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Created on 17/05/2012 3 | 4 | @author: piranna 5 | ''' 6 | 7 | import re 8 | from ordereddict import OrderedDict 9 | 10 | 11 | class Cache(OrderedDict): 12 | """Cache with LRU algorithm using an OrderedDict as basis 13 | """ 14 | def __init__(self, maxsize=100): 15 | OrderedDict.__init__(self) 16 | 17 | self._maxsize = maxsize 18 | 19 | def __getitem__(self, key, *args, **kwargs): 20 | # Get the key and remove it from the cache, or raise KeyError 21 | value = OrderedDict.__getitem__(self, key) 22 | del self[key] 23 | 24 | # Insert the (key, value) pair on the front of the cache 25 | OrderedDict.__setitem__(self, key, value) 26 | 27 | # Return the value from the cache 28 | return value 29 | 30 | def __setitem__(self, key, value, *args, **kwargs): 31 | # Key was inserted before, remove it so we put it at front later 32 | if key in self: 33 | del self[key] 34 | 35 | # Too much items on the cache, remove the least recent used 36 | elif len(self) >= self._maxsize: 37 | self.popitem(False) 38 | 39 | # Insert the (key, value) pair on the front of the cache 40 | OrderedDict.__setitem__(self, key, value, *args, **kwargs) 41 | 42 | 43 | def memoize_generator(func): 44 | """Memoize decorator for generators 45 | 46 | Store `func` results in a cache according to their arguments as 'memoize' 47 | does but instead this works on decorators instead of regular functions. 48 | Obviusly, this is only useful if the generator will always return the same 49 | values for each specific parameters... 50 | """ 51 | cache = Cache() 52 | 53 | def wrapped_func(*args, **kwargs): 54 | params = (args, tuple(sorted(kwargs.items()))) 55 | 56 | # Look if cached 57 | try: 58 | cached = cache[params] 59 | 60 | # Not cached, exec and store it 61 | except KeyError: 62 | cached = [] 63 | 64 | for item in func(*args, **kwargs): 65 | cached.append(item) 66 | yield item 67 | 68 | cache[params] = cached 69 | 70 | # Cached, yield its items 71 | else: 72 | for item in cached: 73 | yield item 74 | 75 | return wrapped_func 76 | 77 | 78 | # This regular expression replaces the home-cooked parser that was here before. 79 | # It is much faster, but requires an extra post-processing step to get the 80 | # desired results (that are compatible with what you would expect from the 81 | # str.splitlines() method). 82 | # 83 | # It matches groups of characters: newlines, quoted strings, or unquoted text, 84 | # and splits on that basis. The post-processing step puts those back together 85 | # into the actual lines of SQL. 86 | SPLIT_REGEX = re.compile(r""" 87 | ( 88 | (?: # Start of non-capturing group 89 | (?:\r\n|\r|\n) | # Match any single newline, or 90 | [^\r\n'"]+ | # Match any character series without quotes or 91 | # newlines, or 92 | "(?:[^"\\]|\\.)*" | # Match double-quoted strings, or 93 | '(?:[^'\\]|\\.)*' # Match single quoted strings 94 | ) 95 | ) 96 | """, re.VERBOSE) 97 | 98 | LINE_MATCH = re.compile(r'(\r\n|\r|\n)') 99 | 100 | 101 | def split_unquoted_newlines(text): 102 | """Split a string on all unquoted newlines. 103 | 104 | Unlike str.splitlines(), this will ignore CR/LF/CR+LF if the requisite 105 | character is inside of a string.""" 106 | lines = SPLIT_REGEX.split(text) 107 | outputlines = [''] 108 | for line in lines: 109 | if not line: 110 | continue 111 | elif LINE_MATCH.match(line): 112 | outputlines.append('') 113 | else: 114 | outputlines[-1] += line 115 | return outputlines 116 | -------------------------------------------------------------------------------- /es_sql/tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/taowen/es-monitor/c4deceb4964857f495d13bfaf2d92f36734c9e1c/es_sql/tests/__init__.py -------------------------------------------------------------------------------- /es_sql/tests/execute_sql_test.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | 3 | from es_sql import es_query 4 | 5 | 6 | class TestExecuteSQL(unittest.TestCase): 7 | def setUp(self): 8 | super(TestExecuteSQL, self).setUp() 9 | self.old_create_executor = es_query.create_executor 10 | es_query.create_executor = self.create_executor 11 | 12 | def tearDown(self): 13 | es_query.create_executor = self.old_create_executor 14 | super(TestExecuteSQL, self).tearDown() 15 | 16 | def create_executor(self, sql_selects, joinable_results): 17 | return self 18 | 19 | def execute(self, es_url, arguments): 20 | return [{'some_key': 'some_value'}] 21 | 22 | def test_no_save_as(self): 23 | result_map = es_query.execute_sql(None, """ 24 | SELECT * FROM abc 25 | """) 26 | self.assertEqual(['result'], result_map.keys()) 27 | 28 | def test_save_as(self): 29 | result_map = es_query.execute_sql(None, """ 30 | SELECT * FROM abc; 31 | SAVE RESULT AS result1; 32 | """) 33 | self.assertEqual(['result1'], result_map.keys()) 34 | 35 | def test_remove(self): 36 | result_map = es_query.execute_sql(None, """ 37 | SELECT * FROM abc; 38 | SAVE RESULT AS result1; 39 | REMOVE RESULT result1; 40 | """) 41 | self.assertEqual([], result_map.keys()) 42 | 43 | def test_python_code(self): 44 | result_map = es_query.execute_sql(None, """ 45 | result_map['result1'] = [] 46 | """) 47 | self.assertEqual(['result1'], result_map.keys()) 48 | 49 | def test_complex_python_code(self): 50 | result_map = es_query.execute_sql(None, """ 51 | result_map['result1'] = [] 52 | for i in range(100): 53 | result_map['result1'].append(i) 54 | """) 55 | self.assertEqual(['result1'], result_map.keys()) 56 | -------------------------------------------------------------------------------- /es_sql/tests/join/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/taowen/es-monitor/c4deceb4964857f495d13bfaf2d92f36734c9e1c/es_sql/tests/join/__init__.py -------------------------------------------------------------------------------- /es_sql/tests/join/client_size_join_test.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | from es_sql import es_query 3 | from es_sql.sqlparse.sql_select import SqlSelect 4 | 5 | 6 | class TestClientSideJoin(unittest.TestCase): 7 | def test_join_on_one_field(self): 8 | executor = es_query.create_executor( 9 | 'SELECT * FROM quote JOIN matched_symbols ON quote.symbol = matched_symbols.symbol', { 10 | 'matched_symbols': [ 11 | {'symbol': '1'}, 12 | {'symbol': '2'} 13 | ] 14 | }) 15 | self.assertEqual( 16 | {'query': {'bool': {'filter': [{}, {'terms': {u'symbol': ['1', '2']}}]}}}, 17 | executor.request) 18 | 19 | def test_join_on_two_fields(self): 20 | executor = es_query.create_executor( 21 | 'SELECT * FROM quote JOIN ' 22 | 'matched_symbols ON quote.symbol = matched_symbols.symbol ' 23 | 'AND quote.date = matched_symbols.date', { 24 | 'matched_symbols': [ 25 | {'symbol': '1', 'date': '1998'}, 26 | {'symbol': '2', 'date': '1998'} 27 | ] 28 | } 29 | ) 30 | self.assertEqual( 31 | {'query': {'bool': {'filter': {}, 'should': [ 32 | {'bool': {'filter': [{'term': {u'symbol': '1'}}, {'term': {u'date': '1998'}}]}}, 33 | {'bool': {'filter': [{'term': {u'symbol': '2'}}, {'term': {u'date': '1998'}}]}}]}}}, 34 | executor.request) 35 | 36 | def test_select_inside_join(self): 37 | executor = es_query.create_executor( 38 | 'SELECT COUNT(*) FROM quote JOIN ' 39 | 'matched_symbols ON quote.symbol = matched_symbols.symbol ' 40 | 'AND quote.date = matched_symbols.date', { 41 | 'matched_symbols': [ 42 | {'symbol': '1', 'date': '1998'}, 43 | {'symbol': '2', 'date': '1998'} 44 | ] 45 | } 46 | ) 47 | self.assertEqual( 48 | {'query': {'bool': {'filter': {}, 'should': [ 49 | {'bool': {'filter': [{'term': {u'symbol': '1'}}, {'term': {u'date': '1998'}}]}}, 50 | {'bool': {'filter': [{'term': {u'symbol': '2'}}, {'term': {u'date': '1998'}}]}}]}}, 'aggs': {}, 51 | 'size': 0}, 52 | executor.request) 53 | -------------------------------------------------------------------------------- /es_sql/tests/join/server_side_join_test.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | from es_sql import es_query 3 | from es_sql.sqlparse.sql_select import SqlSelect 4 | 5 | 6 | class TestClientSideJoin(unittest.TestCase): 7 | def test_join_on_one_field(self): 8 | executor = es_query.create_executor([ 9 | "WITH finance_symbols AS (SELECT * FROM symbol WHERE sector='Finance')", 10 | 'SELECT * FROM quote JOIN finance_symbols ON quote.symbol = finance_symbols.symbol' 11 | ]) 12 | self.assertEqual( 13 | {'query': {'bool': {'filter': [ 14 | {}, {'filterjoin': { 15 | u'symbol': {'indices': u'symbol*', 'path': u'symbol', 16 | 'query': {'term': {u'sector': 'Finance'}}}}}] 17 | }}}, 18 | executor.request) 19 | -------------------------------------------------------------------------------- /es_sql/tests/select_from_leaf/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/taowen/es-monitor/c4deceb4964857f495d13bfaf2d92f36734c9e1c/es_sql/tests/select_from_leaf/__init__.py -------------------------------------------------------------------------------- /es_sql/tests/select_from_leaf/order_by_test.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | 3 | from es_sql import es_query 4 | 5 | 6 | class TestSelectFromLeafOrderBy(unittest.TestCase): 7 | def test_order_by(self): 8 | executor = es_query.create_executor("SELECT * FROM symbol ORDER BY name") 9 | self.assertEqual({'sort': {'name': 'asc'}}, executor.request) 10 | 11 | def test_order_by_asc(self): 12 | executor = es_query.create_executor("SELECT * FROM symbol ORDER BY name asc") 13 | self.assertEqual({'sort': {'name': 'asc'}}, executor.request) 14 | 15 | def test_order_by_desc(self): 16 | executor = es_query.create_executor("SELECT * FROM symbol ORDER BY name DESC") 17 | self.assertEqual({'sort': {'name': 'desc'}}, executor.request) 18 | -------------------------------------------------------------------------------- /es_sql/tests/select_from_leaf/projections_test.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | 3 | from es_sql import es_query 4 | 5 | 6 | class TestSelectFromLeafProjections(unittest.TestCase): 7 | def test_select_all(self): 8 | executor = es_query.create_executor('SELECT * FROM symbol') 9 | self.assertIsNotNone(executor) 10 | self.assertEqual({}, executor.request) 11 | rows = executor.select_response({ 12 | "hits": { 13 | "hits": [ 14 | { 15 | "_score": 1.0, 16 | "_type": "symbol", 17 | "_id": "AVLgXwu88_EnCX8dV9PN", 18 | "_source": { 19 | "exchange": "nasdaq" 20 | }, 21 | "_index": "symbol" 22 | } 23 | ], 24 | "total": 6714, 25 | "max_score": 1.0 26 | }, 27 | "_shards": { 28 | "successful": 3, 29 | "failed": 0, 30 | "total": 3 31 | }, 32 | "took": 32, 33 | "timed_out": False 34 | }) 35 | self.assertEqual([{ 36 | '_id': 'AVLgXwu88_EnCX8dV9PN', '_type': 'symbol', 37 | '_index': 'symbol', 'exchange': 'nasdaq'}], 38 | rows) 39 | 40 | def test_select_one_field(self): 41 | executor = es_query.create_executor('SELECT exchange FROM symbol') 42 | self.assertEqual({}, executor.request) 43 | rows = executor.select_response({ 44 | "hits": { 45 | "hits": [ 46 | { 47 | "_score": 1.0, 48 | "_type": "symbol", 49 | "_id": "AVLgXwu88_EnCX8dV9PN", 50 | "_source": { 51 | "exchange": "nasdaq" 52 | }, 53 | "_index": "symbol" 54 | } 55 | ], 56 | "total": 6714, 57 | "max_score": 1.0 58 | }, 59 | "_shards": { 60 | "successful": 3, 61 | "failed": 0, 62 | "total": 3 63 | }, 64 | "took": 32, 65 | "timed_out": False 66 | }) 67 | self.assertEqual([{'exchange': 'nasdaq'}], rows) 68 | 69 | def test_select_system_field(self): 70 | executor = es_query.create_executor('SELECT _id, _type, _index FROM symbol') 71 | self.assertEqual({}, executor.request) 72 | rows = executor.select_response({ 73 | "hits": { 74 | "hits": [ 75 | { 76 | "_score": 1.0, 77 | "_type": "symbol", 78 | "_id": "AVLgXwu88_EnCX8dV9PN", 79 | "_source": { 80 | "exchange": "nasdaq" 81 | }, 82 | "_index": "symbol" 83 | } 84 | ], 85 | "total": 6714, 86 | "max_score": 1.0 87 | }, 88 | "_shards": { 89 | "successful": 3, 90 | "failed": 0, 91 | "total": 3 92 | }, 93 | "took": 32, 94 | "timed_out": False 95 | }) 96 | self.assertEqual([{ 97 | '_id': 'AVLgXwu88_EnCX8dV9PN', '_type': 'symbol', 98 | '_index': 'symbol'}], 99 | rows) 100 | 101 | def test_select_nested_field(self): 102 | executor = es_query.create_executor('SELECT "a.exchange" FROM symbol') 103 | self.assertEqual({}, executor.request) 104 | rows = executor.select_response({ 105 | "hits": { 106 | "hits": [ 107 | { 108 | "_score": 1.0, 109 | "_type": "symbol", 110 | "_id": "AVLgXwu88_EnCX8dV9PN", 111 | "_source": { 112 | "a": { 113 | "exchange": "nasdaq" 114 | } 115 | }, 116 | "_index": "symbol" 117 | } 118 | ], 119 | "total": 6714, 120 | "max_score": 1.0 121 | }, 122 | "_shards": { 123 | "successful": 3, 124 | "failed": 0, 125 | "total": 3 126 | }, 127 | "took": 32, 128 | "timed_out": False 129 | }) 130 | self.assertEqual([{ 131 | 'a.exchange': 'nasdaq'}], 132 | rows) 133 | 134 | def test_select_nested_field_via_dot(self): 135 | executor = es_query.create_executor('SELECT a.exchange FROM symbol') 136 | self.assertEqual({}, executor.request) 137 | rows = executor.select_response({ 138 | "hits": { 139 | "hits": [ 140 | { 141 | "_score": 1.0, 142 | "_type": "symbol", 143 | "_id": "AVLgXwu88_EnCX8dV9PN", 144 | "_source": { 145 | "a": { 146 | "exchange": "nasdaq" 147 | } 148 | }, 149 | "_index": "symbol" 150 | } 151 | ], 152 | "total": 6714, 153 | "max_score": 1.0 154 | }, 155 | "_shards": { 156 | "successful": 3, 157 | "failed": 0, 158 | "total": 3 159 | }, 160 | "took": 32, 161 | "timed_out": False 162 | }) 163 | self.assertEqual([{ 164 | 'a.exchange': 'nasdaq'}], 165 | rows) 166 | 167 | 168 | def test_select_expression(self): 169 | executor = es_query.create_executor('SELECT "a.price"/2 FROM symbol') 170 | self.assertEqual({}, executor.request) 171 | rows = executor.select_response({ 172 | "hits": { 173 | "hits": [ 174 | { 175 | "_score": 1.0, 176 | "_type": "symbol", 177 | "_id": "AVLgXwu88_EnCX8dV9PN", 178 | "_source": { 179 | "a": { 180 | "price": 100 181 | } 182 | }, 183 | "_index": "symbol" 184 | } 185 | ], 186 | "total": 6714, 187 | "max_score": 1.0 188 | }, 189 | "_shards": { 190 | "successful": 3, 191 | "failed": 0, 192 | "total": 3 193 | }, 194 | "took": 32, 195 | "timed_out": False 196 | }) 197 | self.assertEqual([{ 198 | '"a.price"/2': 50}], 199 | rows) 200 | -------------------------------------------------------------------------------- /es_sql/tests/select_from_leaf/where_test.py: -------------------------------------------------------------------------------- 1 | import datetime 2 | import unittest 3 | 4 | from es_sql import es_query 5 | from es_sql.sqlparse import datetime_evaluator 6 | 7 | 8 | class TestSelectFromLeafWhere(unittest.TestCase): 9 | def test_field_eq_string(self): 10 | executor = es_query.create_executor("SELECT * FROM symbol WHERE exchange='nyse'") 11 | self.assertEqual({'query': {'term': {'exchange': 'nyse'}}}, executor.request) 12 | executor = es_query.create_executor("SELECT * FROM symbol WHERE exchange=%(exchange)s") 13 | self.assertEqual({ 14 | 'query': {'term': {u'exchange': '%(exchange)s'}}, 15 | '_parameters_': {u'exchange': { 16 | 'path': ['query', 'term', u'exchange'], 17 | 'field_hint': 'exchange' 18 | }}} 19 | , executor.request) 20 | 21 | def test_and(self): 22 | executor = es_query.create_executor("SELECT * FROM symbol WHERE exchange='nyse' AND sector='Technology'") 23 | self.assertEqual( 24 | {'query': {'bool': {'filter': [{'term': {'exchange': 'nyse'}}, {'term': {'sector': 'Technology'}}]}}}, 25 | executor.request) 26 | 27 | def test_and_not(self): 28 | executor = es_query.create_executor("SELECT * FROM symbol WHERE exchange='nyse' AND NOT sector='Technology'") 29 | self.assertEqual( 30 | {'query': {'bool': { 31 | 'filter': [{'term': {'exchange': 'nyse'}}], 32 | 'must_not': [{'term': {'sector': 'Technology'}}]}}}, 33 | executor.request) 34 | 35 | def test_not_and_not(self): 36 | executor = es_query.create_executor( 37 | "SELECT * FROM symbol WHERE NOT exchange='nyse' AND NOT sector='Technology'") 38 | self.assertEqual( 39 | {'query': {'bool': { 40 | 'must_not': [{'term': {'exchange': 'nyse'}}, {'term': {'sector': 'Technology'}}]}}}, 41 | executor.request) 42 | 43 | def test_and_and(self): 44 | executor = es_query.create_executor( 45 | "SELECT * FROM symbol WHERE exchange='nyse' AND sector='Technology' AND ipo_year=1998") 46 | self.assertEqual( 47 | {'query': {'bool': {'filter': [ 48 | {'term': {'exchange': 'nyse'}}, 49 | {'term': {'sector': 'Technology'}}, 50 | {'term': {'ipo_year': 1998}}, 51 | ]}}}, 52 | executor.request) 53 | 54 | def test_or(self): 55 | executor = es_query.create_executor("SELECT * FROM symbol WHERE exchange='nyse' OR sector='Technology'") 56 | self.assertEqual( 57 | {'query': {'bool': {'should': [{'term': {'exchange': 'nyse'}}, {'term': {'sector': 'Technology'}}]}}}, 58 | executor.request) 59 | 60 | def test_or_not(self): 61 | executor = es_query.create_executor("SELECT * FROM symbol WHERE exchange='nyse' OR NOT sector='Technology'") 62 | self.assertEqual( 63 | {'query': {'bool': {'should': [ 64 | {'term': {'exchange': 'nyse'}}, 65 | {'bool': {'must_not': [{'term': {'sector': 'Technology'}}]}}]}}}, 66 | executor.request) 67 | 68 | def test_and_or_must_use_parentheses(self): 69 | try: 70 | executor = es_query.create_executor( 71 | "SELECT * FROM symbol WHERE exchange='nyse' AND sector='Technology' OR ipo_year > 1998") 72 | except: 73 | return 74 | self.fail('should fail') 75 | 76 | def test_and_or_used_parentheses(self): 77 | executor = es_query.create_executor( 78 | "SELECT * FROM symbol WHERE exchange='nyse' AND (sector='Technology' OR ipo_year > 1998)") 79 | self.assertEqual( 80 | {'query': {'bool': {'filter': [ 81 | {'term': {'exchange': 'nyse'}}, 82 | {'bool': {'should': [ 83 | {'term': {'sector': 'Technology'}}, 84 | {'range': {'ipo_year': {'gt': 1998.0}}}]}} 85 | ]}}}, 86 | executor.request) 87 | 88 | def test_field_gt_numeric(self): 89 | executor = es_query.create_executor("SELECT * FROM symbol WHERE last_sale > 1000") 90 | self.assertEqual( 91 | {'query': {'range': {'last_sale': {'gt': 1000.0}}}}, 92 | executor.request) 93 | executor = es_query.create_executor("SELECT * FROM symbol WHERE last_sale > %(param1)s") 94 | self.assertEqual( 95 | {'query': {'range': {u'last_sale': {'gt': '%(param1)s'}}}, 96 | '_parameters_': {u'param1': { 97 | 'path': ['query', 'range', u'last_sale', 'gt'], 98 | 'field_hint': 'last_sale' 99 | }}}, 100 | executor.request) 101 | 102 | def test_field_gte_numeric(self): 103 | executor = es_query.create_executor("SELECT * FROM symbol WHERE last_sale >= 1000") 104 | self.assertEqual( 105 | {'query': {'range': {'last_sale': {'gte': 1000.0}}}}, 106 | executor.request) 107 | 108 | def test_field_lt_numeric(self): 109 | executor = es_query.create_executor("SELECT * FROM symbol WHERE last_sale < 1000") 110 | self.assertEqual( 111 | {'query': {'range': {'last_sale': {'lt': 1000.0}}}}, 112 | executor.request) 113 | 114 | def test_field_lte_numeric(self): 115 | executor = es_query.create_executor("SELECT * FROM symbol WHERE last_sale <= 1000") 116 | self.assertEqual( 117 | {'query': {'range': {'last_sale': {'lte': 1000.0}}}}, 118 | executor.request) 119 | 120 | def test_field_not_eq_numeric(self): 121 | executor = es_query.create_executor("SELECT * FROM symbol WHERE last_sale != 1000") 122 | self.assertEqual( 123 | {'query': {'bool': {'must_not': {'term': {'last_sale': 1000}}}}}, 124 | executor.request) 125 | executor = es_query.create_executor("SELECT * FROM symbol WHERE 1000 != last_sale") 126 | self.assertEqual( 127 | {'query': {'bool': {'must_not': {'term': {'last_sale': 1000}}}}}, 128 | executor.request) 129 | executor = es_query.create_executor("SELECT * FROM symbol WHERE last_sale != %(param1)s") 130 | self.assertEqual( 131 | {'query': {'bool': {'must_not': {'term': {u'last_sale': '%(param1)s'}}}}, '_parameters_': { 132 | u'param1': {'path': ['query', 'bool', 'must_not', 'term', u'last_sale'], 'field_hint': u'last_sale'}}}, 133 | executor.request) 134 | 135 | def test_field_in_range(self): 136 | executor = es_query.create_executor("SELECT * FROM symbol WHERE last_sale > 500 AND last_sale < 600") 137 | self.assertEqual( 138 | {'query': {'range': {'last_sale': {'lt': 600.0, 'gt': 500.0}}}}, 139 | executor.request) 140 | 141 | def test_field_in_range_not_merged(self): 142 | executor = es_query.create_executor("SELECT * FROM symbol WHERE last_sale > 500 AND last_sale > 600") 143 | self.assertEqual( 144 | {'query': {'bool': {'filter': [ 145 | {'range': {'last_sale': {'gt': 500.0}}}, 146 | {'range': {'last_sale': {'gt': 600.0}}}] 147 | }}}, 148 | executor.request) 149 | 150 | def test_is_null(self): 151 | executor = es_query.create_executor("SELECT * FROM symbol WHERE last_sale IS NULL") 152 | self.assertEqual( 153 | {'query': {'bool': {'must_not': {'exists': {'field': 'last_sale'}}}}}, 154 | executor.request) 155 | 156 | def test_is_not_null(self): 157 | executor = es_query.create_executor("SELECT * FROM symbol WHERE last_sale IS NOT NULL") 158 | self.assertEqual( 159 | {'query': {'exists': {'field': 'last_sale'}}}, 160 | executor.request) 161 | 162 | def test_field_can_be_right_operand(self): 163 | executor = es_query.create_executor("SELECT * FROM symbol WHERE 'nyse'=exchange") 164 | self.assertEqual({'query': {'term': {'exchange': 'nyse'}}}, executor.request) 165 | executor = es_query.create_executor("SELECT * FROM symbol WHERE 1998 now()") 175 | self.assertEqual({'query': {'range': {'ts': {'gt': 1470585600000L}}}}, executor.request) 176 | 177 | def test_now_expression(self): 178 | datetime_evaluator.NOW = datetime.datetime(2016, 8, 8) 179 | executor = es_query.create_executor("SELECT * FROM symbol WHERE ts > now() - INTERVAL '1 DAY'") 180 | self.assertEqual({'query': {'range': {'ts': {'gt': 1470585600000L - 24 * 60 * 60 * 1000}}}}, executor.request) 181 | 182 | def test_today_expression(self): 183 | datetime_evaluator.NOW = datetime.datetime(2016, 8, 8) 184 | executor = es_query.create_executor("SELECT * FROM symbol WHERE ts > today() - interval('1 day')") 185 | self.assertEqual({'query': {'range': {'ts': {'gt': 1470585600000L - 24 * 60 * 60 * 1000}}}}, executor.request) 186 | 187 | def test_timestamp(self): 188 | executor = es_query.create_executor("SELECT * FROM symbol WHERE ts > TIMESTAMP '2016-08-08 00:00:00'") 189 | self.assertEqual({'query': {'range': {'ts': {'gt': 1470585600000L}}}}, executor.request) 190 | 191 | def test_in(self): 192 | executor = es_query.create_executor("SELECT * FROM symbol WHERE symbol IN ('AAPL', 'GOOG')") 193 | self.assertEqual({'query': {'terms': {u'symbol': ['AAPL', 'GOOG']}}}, executor.request) 194 | executor = es_query.create_executor("SELECT * FROM symbol WHERE symbol IN %(param1)s") 195 | self.assertEqual( 196 | {'query': {'terms': {u'symbol': '%(param1)s'}}, 197 | '_parameters_': {u'param1': {'path': ['query', 'terms', u'symbol'], 'field_hint': u'symbol'}}}, 198 | executor.request) 199 | 200 | def test_type_eq(self): 201 | executor = es_query.create_executor("SELECT * FROM symbol WHERE _type = 'symbol'") 202 | self.assertEqual({'query': {'type': {'value': 'symbol'}}}, executor.request) 203 | 204 | def test_id_eq(self): 205 | executor = es_query.create_executor("SELECT * FROM symbol WHERE _id = '1'") 206 | self.assertEqual({'query': {'ids': {'value': ['1']}}}, executor.request) 207 | 208 | def test_id_in(self): 209 | executor = es_query.create_executor("SELECT * FROM symbol WHERE _id IN ('1', '2')") 210 | self.assertEqual({'query': {'ids': {'value': ['1', '2']}}}, executor.request) 211 | 212 | def test_type_merged_with_ids(self): 213 | executor = es_query.create_executor("SELECT * FROM symbol WHERE _type = 'symbol' AND _id = '1'") 214 | self.assertEqual({'query': {'ids': {'value': ['1'], 'type': 'symbol'}}}, executor.request) 215 | executor = es_query.create_executor("SELECT * FROM symbol WHERE _type = 'symbol' AND _id IN ('1', '2')") 216 | self.assertEqual({'query': {'ids': {'value': ['1', '2'], 'type': 'symbol'}}}, executor.request) 217 | executor = es_query.create_executor("SELECT * FROM symbol WHERE _id IN ('1', '2') AND _type = 'symbol'") 218 | self.assertEqual({'query': {'ids': {'value': ['1', '2'], 'type': 'symbol'}}}, executor.request) 219 | executor = es_query.create_executor( 220 | "SELECT * FROM symbol WHERE _id IN ('1', '2') AND _type = 'symbol' AND _type='abc'") 221 | self.assertEqual({'query': { 222 | 'bool': {'filter': [{'ids': {'type': 'symbol', 'value': ['1', '2']}}, {'type': {'value': 'abc'}}]}}}, 223 | executor.request) 224 | 225 | def test_like(self): 226 | executor = es_query.create_executor("SELECT * FROM symbol WHERE symbol LIKE 'AAP%'") 227 | self.assertEqual({'query': {'wildcard': {u'symbol': 'AAP*'}}}, executor.request) 228 | executor = es_query.create_executor("SELECT * FROM symbol WHERE symbol LIKE %(param1)s") 229 | self.assertEqual( 230 | {'query': {'wildcard': {u'symbol': '%(param1)s'}}, '_parameters_': { 231 | u'param1': {'path': ['query', 'wildcard', u'symbol'], 'field_hint': u'symbol'}}}, 232 | executor.request) 233 | -------------------------------------------------------------------------------- /es_sql/tests/select_inside_branch/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/taowen/es-monitor/c4deceb4964857f495d13bfaf2d92f36734c9e1c/es_sql/tests/select_inside_branch/__init__.py -------------------------------------------------------------------------------- /es_sql/tests/select_inside_branch/group_by_test.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | 3 | from es_sql import es_query 4 | 5 | 6 | class SelectInsideBranchGroupByTest(unittest.TestCase): 7 | def test_drill_down_one_direction(self): 8 | executor = es_query.create_executor([ 9 | "WITH all_symbols AS (SELECT MAX(market_cap) AS max_all_times FROM symbol)", 10 | "SELECT ipo_year, MAX(market_cap) AS max_this_year FROM all_symbols GROUP BY ipo_year LIMIT 5"]) 11 | self.assertEqual( 12 | {'aggs': {'max_all_times': {'max': {'field': 'market_cap'}}, 13 | 'ipo_year': {'terms': {'field': 'ipo_year', 'size': 5}, 14 | 'aggs': {'max_this_year': {'max': {'field': 'market_cap'}}}}}, 'size': 0}, 15 | executor.request) 16 | 17 | def test_drill_down_two_directions(self): 18 | executor = es_query.create_executor([ 19 | "WITH all_symbols AS (SELECT MAX(market_cap) AS max_all_times FROM symbol)", 20 | "SELECT ipo_year, MAX(market_cap) AS max_this_year FROM all_symbols GROUP BY ipo_year LIMIT 1", 21 | "SELECT sector, MAX(market_cap) AS max_this_sector FROM all_symbols GROUP BY sector LIMIT 1"]) 22 | self.assertEqual( 23 | {'aggs': {u'sector': {'terms': {'field': u'sector', 'size': 1}, 24 | 'aggs': {u'max_this_sector': {u'max': {'field': u'market_cap'}}}}, 25 | u'max_all_times': {u'max': {'field': u'market_cap'}}, 26 | u'ipo_year': {'terms': {'field': u'ipo_year', 'size': 1}, 27 | 'aggs': {u'max_this_year': {u'max': {'field': u'market_cap'}}}}}, 'size': 0}, 28 | executor.request) 29 | -------------------------------------------------------------------------------- /es_sql/tests/select_inside_branch/having_test.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | 3 | from es_sql import es_query 4 | 5 | 6 | class SelectInsideBranchHavingTest(unittest.TestCase): 7 | def test_having_can_reference_child_buckets(self): 8 | executor = es_query.create_executor( 9 | ["WITH per_year AS (SELECT ipo_year, COUNT(*) AS ipo_count FROM symbol \n" 10 | "GROUP BY ipo_year HAVING max_in_finance > 200)", 11 | "SELECT MAX(market_cap) AS max_in_finance FROM per_year WHERE sector='Finance'"]) 12 | self.assertEqual( 13 | {'aggs': {u'ipo_year': {'terms': {'field': u'ipo_year', 'size': 0}, 'aggs': { 14 | 'level2': {'filter': {'term': {u'sector': 'Finance'}}, 15 | 'aggs': {u'max_in_finance': {u'max': {'field': u'market_cap'}}}}, 'having': { 16 | 'bucket_selector': {'buckets_path': {u'max_in_finance': u'level2.max_in_finance'}, 17 | 'script': {'lang': 'expression', 'inline': u' max_in_finance > 200'}}}}}}, 'size': 0}, 18 | executor.request) 19 | -------------------------------------------------------------------------------- /es_sql/tests/select_inside_branch/order_by_test.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | 3 | from es_sql import es_query 4 | 5 | 6 | class SelectInsideBranchOrderByTest(unittest.TestCase): 7 | def test_order_by_can_reference_child_buckets(self): 8 | executor = es_query.create_executor( 9 | ["WITH per_year AS (SELECT ipo_year, COUNT(*) AS ipo_count FROM symbol \n" 10 | "GROUP BY ipo_year ORDER BY max_in_finance LIMIT 2)", 11 | "SELECT MAX(market_cap) AS max_in_finance FROM per_year WHERE sector='Finance'"]) 12 | self.assertEqual( 13 | {'aggs': { 14 | u'ipo_year': {'terms': {'field': u'ipo_year', 'order': {u'level2.max_in_finance': 'asc'}, 'size': 2}, 15 | 'aggs': {'level2': {'filter': {'term': {u'sector': 'Finance'}}, 16 | 'aggs': {u'max_in_finance': {u'max': {'field': u'market_cap'}}}}}}}, 17 | 'size': 0}, 18 | executor.request) 19 | -------------------------------------------------------------------------------- /es_sql/tests/select_inside_branch/projection_test.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | 3 | from es_sql import es_query 4 | 5 | 6 | class SelectInsideProjectionTest(unittest.TestCase): 7 | def test_one_level(self): 8 | executor = es_query.create_executor([ 9 | "WITH all_symbols AS (SELECT MAX(sum_this_year) AS max_all_times FROM symbol)", 10 | "SELECT ipo_year, SUM(market_cap) AS sum_this_year FROM all_symbols GROUP BY ipo_year LIMIT 5"]) 11 | self.assertEqual( 12 | {'aggs': {u'max_all_times': {u'max_bucket': {'buckets_path': u'ipo_year.sum_this_year'}}, 13 | u'ipo_year': {'terms': {'field': u'ipo_year', 'size': 5}, 14 | 'aggs': {u'sum_this_year': {u'sum': {'field': u'market_cap'}}}}}, 'size': 0}, 15 | executor.request) 16 | 17 | def test_two_level(self): 18 | executor = es_query.create_executor([ 19 | "WITH all_symbols AS (SELECT MAX(sum_this_year) AS max_all_times FROM symbol)", 20 | "WITH finance_symbols AS (SELECT * FROM all_symbols WHERE sector='Finance')", 21 | "SELECT ipo_year, SUM(market_cap) AS sum_this_year FROM finance_symbols GROUP BY ipo_year LIMIT 5"]) 22 | self.assertEqual( 23 | {'aggs': { 24 | u'max_all_times': {u'max_bucket': {'buckets_path': u'finance_symbols>ipo_year.sum_this_year'}}, 25 | 'finance_symbols': {'filter': {'term': {u'sector': 'Finance'}}, 'aggs': { 26 | u'ipo_year': {'terms': {'field': u'ipo_year', 'size': 5}, 27 | 'aggs': {u'sum_this_year': {u'sum': {'field': u'market_cap'}}}}}}}, 'size': 0}, 28 | executor.request) 29 | 30 | def test_csum(self): 31 | executor = es_query.create_executor([ 32 | "SELECT year, MAX(adj_close) AS max_adj_close, CSUM(max_adj_close) FROM quote " 33 | "WHERE symbol='AAPL' GROUP BY date_trunc('year', \"date\") AS year"]) 34 | self.assertEqual( 35 | {'query': {'term': {u'symbol': 'AAPL'}}, 'aggs': { 36 | u'year': {'date_histogram': {'field': u'date', 'interval': 'year', 'time_zone': '+08:00'}, 37 | 'aggs': {u'max_adj_close': {u'max': {'field': u'adj_close'}}, 38 | 'CSUM(max_adj_close)': {'cumulative_sum': {'buckets_path': u'max_adj_close'}}}}}, 39 | 'size': 0}, 40 | executor.request) 41 | 42 | def test_moving_average(self): 43 | executor = es_query.create_executor([ 44 | "SELECT year, MAX(adj_close) AS max_adj_close, MOVING_Avg(max_adj_close) FROM quote " 45 | "WHERE symbol='AAPL' GROUP BY date_trunc('year', \"date\") AS year"]) 46 | self.assertEqual( 47 | {'query': {'term': {u'symbol': 'AAPL'}}, 'aggs': { 48 | u'year': {'date_histogram': {'field': u'date', 'interval': 'year', 'time_zone': '+08:00'}, 49 | 'aggs': {u'max_adj_close': {u'max': {'field': u'adj_close'}}, 50 | 'MOVING_Avg(max_adj_close)': { 51 | 'moving_avg': {'buckets_path': u'max_adj_close'}}}}}, 52 | 'size': 0}, 53 | executor.request) 54 | 55 | def test_moving_average_with_params(self): 56 | executor = es_query.create_executor([ 57 | "SELECT year, MAX(adj_close) AS max_adj_close, MOVING_Avg(max_adj_close, '{\"window\":5}') AS ma FROM quote " 58 | "WHERE symbol='AAPL' GROUP BY date_trunc('year', \"date\") AS year"]) 59 | self.assertEqual( 60 | {'query': {'term': {u'symbol': 'AAPL'}}, 'aggs': { 61 | u'year': {'date_histogram': {'field': u'date', 'interval': 'year', 'time_zone': '+08:00'}, 62 | 'aggs': {u'max_adj_close': {u'max': {'field': u'adj_close'}}, 63 | 'ma': { 64 | 'moving_avg': {'buckets_path': u'max_adj_close', 'window': 5}}}}}, 65 | 'size': 0}, 66 | executor.request) 67 | 68 | def test_moving_average_with_named_params(self): 69 | executor = es_query.create_executor([ 70 | "SELECT year, MAX(adj_close) AS max_adj_close, MOVING_Avg(max_adj_close, window=5, settings='{\"alpha\":0.8}') AS ma FROM quote " 71 | "WHERE symbol='AAPL' GROUP BY date_trunc('year', \"date\") AS year"]) 72 | self.assertEqual( 73 | {'query': {'term': {u'symbol': 'AAPL'}}, 'aggs': { 74 | u'year': {'date_histogram': {'field': u'date', 'interval': 'year', 'time_zone': '+08:00'}, 75 | 'aggs': {u'max_adj_close': {u'max': {'field': u'adj_close'}}, 76 | 'ma': { 77 | 'moving_avg': {'buckets_path': u'max_adj_close', 'window': 5, 78 | 'settings': {'alpha': 0.8}}}}}}, 'size': 0}, 79 | executor.request) 80 | 81 | def test_serial_diff(self): 82 | executor = es_query.create_executor([ 83 | "SELECT year, MAX(adj_close) AS max_adj_close, SERIAL_DIFF(max_adj_close, lag=7) AS ma FROM quote " 84 | "WHERE symbol='AAPL' GROUP BY date_trunc('year', \"date\") AS year"]) 85 | self.assertEqual( 86 | {'query': {'term': {u'symbol': 'AAPL'}}, 'aggs': { 87 | u'year': {'date_histogram': {'field': u'date', 'interval': 'year', 'time_zone': '+08:00'}, 88 | 'aggs': {u'max_adj_close': {u'max': {'field': u'adj_close'}}, 89 | u'ma': {u'serial_diff': {'buckets_path': u'max_adj_close', u'lag': 7}}}}}, 90 | 'size': 0}, 91 | executor.request) 92 | 93 | def test_drivative(self): 94 | executor = es_query.create_executor([ 95 | "SELECT year, MAX(adj_close) AS max_adj_close, DERIVATIVE(max_adj_close) FROM quote " 96 | "WHERE symbol='AAPL' GROUP BY date_trunc('year', \"date\") AS year"]) 97 | self.assertEqual( 98 | {'query': {'term': {u'symbol': 'AAPL'}}, 'aggs': { 99 | u'year': {'date_histogram': {'field': u'date', 'interval': 'year', 'time_zone': '+08:00'}, 100 | 'aggs': {u'max_adj_close': {u'max': {'field': u'adj_close'}}, 101 | 'DERIVATIVE(max_adj_close)': { 102 | 'derivative': {'buckets_path': u'max_adj_close'}}}}}, 103 | 'size': 0}, 104 | executor.request) 105 | 106 | def test_bucket_script(self): 107 | executor = es_query.create_executor([ 108 | "WITH all_estimate AS (SELECT err_count/total_count AS err_rate, COUNT(*) AS total_count " 109 | "FROM gs_plutus_debug GROUP BY req.district)", 110 | "WITH err AS (SELECT COUNT(*) AS err_count FROM all_estimate WHERE errno>0)"]) 111 | self.assertEqual( 112 | {'aggs': {'req.district': { 113 | 'terms': {'field': 'req.district', 'size': 0}, 114 | 'aggs': {'err': {'filter': {'range': {u'errno': {'gt': 0}}}, 'aggs': {}}, 115 | u'err_rate': {'bucket_script': { 116 | 'buckets_path': {u'total_count': '_count', 117 | u'err_count': 'err._count'}, 118 | 'script': {'lang': 'expression', 119 | 'inline': u'err_count/total_count'}}}}}}, 'size': 0}, 120 | executor.request) 121 | -------------------------------------------------------------------------------- /es_sql/tests/select_inside_branch/response_test.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | 3 | from es_sql import es_query 4 | 5 | 6 | class SelectInsideBranchResponseTest(unittest.TestCase): 7 | def test_one_child(self): 8 | executor = es_query.create_executor([ 9 | "WITH all_symbols AS (SELECT MAX(market_cap) AS max_all_times FROM symbol)", 10 | "SELECT ipo_year, MAX(market_cap) AS max_this_year FROM all_symbols GROUP BY ipo_year LIMIT 5"]) 11 | rows = executor.select_response({ 12 | "hits": { 13 | "hits": [], 14 | "total": 6714, 15 | "max_score": 0.0 16 | }, 17 | "_shards": { 18 | "successful": 1, 19 | "failed": 0, 20 | "total": 1 21 | }, 22 | "took": 2, 23 | "aggregations": { 24 | "max_all_times": { 25 | "value": 522690000000.0 26 | }, 27 | "ipo_year": { 28 | "buckets": [ 29 | { 30 | "max_this_year": { 31 | "value": 54171930444.0 32 | }, 33 | "key": 2014, 34 | "doc_count": 390 35 | }, 36 | { 37 | "max_this_year": { 38 | "value": 5416144671.0 39 | }, 40 | "key": 2015, 41 | "doc_count": 334 42 | }, 43 | { 44 | "max_this_year": { 45 | "value": 10264219758.0 46 | }, 47 | "key": 2013, 48 | "doc_count": 253 49 | }, 50 | { 51 | "max_this_year": { 52 | "value": 287470000000.0 53 | }, 54 | "key": 2012, 55 | "doc_count": 147 56 | }, 57 | { 58 | "max_this_year": { 59 | "value": 7436036210.0 60 | }, 61 | "key": 2011, 62 | "doc_count": 144 63 | } 64 | ], 65 | "sum_other_doc_count": 1630, 66 | "doc_count_error_upper_bound": 0 67 | } 68 | }, 69 | "timed_out": False 70 | }) 71 | self.assertEqual( 72 | [{'max_this_year': 54171930444.0, u'ipo_year': 2014, 'max_all_times': 522690000000.0, '_bucket_path': ['level2']}, 73 | {'max_this_year': 5416144671.0, u'ipo_year': 2015, 'max_all_times': 522690000000.0, '_bucket_path': ['level2']}, 74 | {'max_this_year': 10264219758.0, u'ipo_year': 2013, 'max_all_times': 522690000000.0, '_bucket_path': ['level2']}, 75 | {'max_this_year': 287470000000.0, u'ipo_year': 2012, 'max_all_times': 522690000000.0, '_bucket_path': ['level2']}, 76 | {'max_this_year': 7436036210.0, u'ipo_year': 2011, 'max_all_times': 522690000000.0, '_bucket_path': ['level2']}], 77 | rows) 78 | 79 | def test_two_children(self): 80 | executor = es_query.create_executor([ 81 | "WITH all_symbols AS (SELECT MAX(market_cap) AS max_all_times FROM symbol)", 82 | "SELECT ipo_year, MAX(market_cap) AS max_this_year FROM all_symbols GROUP BY ipo_year LIMIT 1", 83 | "SELECT sector, MAX(market_cap) AS max_this_sector FROM all_symbols GROUP BY sector LIMIT 1"]) 84 | rows = executor.select_response({ 85 | "hits": { 86 | "hits": [], 87 | "total": 6714, 88 | "max_score": 0.0 89 | }, 90 | "_shards": { 91 | "successful": 1, 92 | "failed": 0, 93 | "total": 1 94 | }, 95 | "took": 2, 96 | "aggregations": { 97 | "sector": { 98 | "buckets": [ 99 | { 100 | "max_this_sector": { 101 | "value": 34620000000.0 102 | }, 103 | "key": "n/a", 104 | "doc_count": 1373 105 | } 106 | ], 107 | "sum_other_doc_count": 5341, 108 | "doc_count_error_upper_bound": 0 109 | }, 110 | "max_all_times": { 111 | "value": 522690000000.0 112 | }, 113 | "ipo_year": { 114 | "buckets": [ 115 | { 116 | "max_this_year": { 117 | "value": 54171930444.0 118 | }, 119 | "key": 2014, 120 | "doc_count": 390 121 | } 122 | ], 123 | "sum_other_doc_count": 2508, 124 | "doc_count_error_upper_bound": 0 125 | } 126 | }, 127 | "timed_out": False 128 | }) 129 | self.assertEqual( 130 | [{'max_this_year': 54171930444.0, 'ipo_year': 2014, 'max_all_times': 522690000000.0, '_bucket_path': ['level2']}, 131 | {'sector': 'n/a', 'max_all_times': 522690000000.0, 'max_this_sector': 34620000000.0, '_bucket_path': ['level3']}], 132 | rows) 133 | 134 | def test_filter_only_will_not_create_new_row(self): 135 | executor = es_query.create_executor([ 136 | "WITH all_symbols AS (SELECT MAX(market_cap) AS max_all_times FROM symbol)", 137 | "SELECT MAX(market_cap) AS max_at_2000 FROM all_symbols WHERE ipo_year=2000", 138 | "SELECT MAX(market_cap) AS max_at_2001 FROM all_symbols WHERE ipo_year=2001"]) 139 | rows = executor.select_response({ 140 | "hits": { 141 | "hits": [], 142 | "total": 6714, 143 | "max_score": 0.0 144 | }, 145 | "_shards": { 146 | "successful": 1, 147 | "failed": 0, 148 | "total": 1 149 | }, 150 | "took": 4, 151 | "aggregations": { 152 | "level2": { 153 | "max_at_2000": { 154 | "value": 20310000000.0 155 | }, 156 | "doc_count": 58 157 | }, 158 | "level3": { 159 | "max_at_2001": { 160 | "value": 8762940000.0 161 | }, 162 | "doc_count": 38 163 | }, 164 | "max_all_times": { 165 | "value": 522690000000.0 166 | } 167 | }, 168 | "timed_out": False 169 | }) 170 | self.assertEqual( 171 | [{'max_at_2000': 20310000000.0, 'max_at_2001': 8762940000.0, 'max_all_times': 522690000000.0}], 172 | rows) 173 | 174 | def test_filter_upon_filter(self): 175 | executor = es_query.create_executor([ 176 | "WITH all_symbols AS (SELECT MAX(market_cap) AS max_all_times FROM symbol)", 177 | "WITH year_2001 AS (SELECT MAX(market_cap) AS max_at_2000 FROM all_symbols WHERE ipo_year=2000)", 178 | "SELECT MAX(market_cap) AS max_at_2001_finance FROM year_2001 WHERE sector='Finance'"]) 179 | rows = executor.select_response({ 180 | "hits": { 181 | "hits": [], 182 | "total": 6714, 183 | "max_score": 0.0 184 | }, 185 | "_shards": { 186 | "successful": 1, 187 | "failed": 0, 188 | "total": 1 189 | }, 190 | "took": 3, 191 | "aggregations": { 192 | "year_2001": { 193 | "max_at_2000": { 194 | "value": 20310000000.0 195 | }, 196 | "level3": { 197 | "max_at_2001_finance": { 198 | "value": 985668354.0 199 | }, 200 | "doc_count": 2 201 | }, 202 | "doc_count": 58 203 | }, 204 | "max_all_times": { 205 | "value": 522690000000.0 206 | } 207 | }, 208 | "timed_out": False 209 | }) 210 | self.assertEqual( 211 | [{'max_at_2000': 20310000000.0, 'max_all_times': 522690000000.0, 'max_at_2001_finance': 985668354.0}], 212 | rows) 213 | 214 | def test_filter_then_group_by(self): 215 | executor = es_query.create_executor([ 216 | "WITH all_symbols AS (SELECT MAX(market_cap) AS max_all_times FROM symbol)", 217 | "WITH year_2000 AS (SELECT MAX(market_cap) AS max_at_2000 FROM all_symbols WHERE ipo_year=2000)", 218 | "SELECT sector, MAX(market_cap) AS max_per_sector FROM year_2000 GROUP BY sector LIMIT 2"]) 219 | rows = executor.select_response({ 220 | "hits": { 221 | "hits": [], 222 | "total": 6714, 223 | "max_score": 0.0 224 | }, 225 | "_shards": { 226 | "successful": 1, 227 | "failed": 0, 228 | "total": 1 229 | }, 230 | "took": 5, 231 | "aggregations": { 232 | "year_2000": { 233 | "max_at_2000": { 234 | "value": 20310000000.0 235 | }, 236 | "sector": { 237 | "buckets": [ 238 | { 239 | "max_per_sector": { 240 | "value": 19600000000.0 241 | }, 242 | "key": "Health Care", 243 | "doc_count": 18 244 | }, 245 | { 246 | "max_per_sector": { 247 | "value": 4440000000.0 248 | }, 249 | "key": "Technology", 250 | "doc_count": 16 251 | } 252 | ], 253 | "sum_other_doc_count": 24, 254 | "doc_count_error_upper_bound": 0 255 | }, 256 | "doc_count": 58 257 | }, 258 | "max_all_times": { 259 | "value": 522690000000.0 260 | } 261 | }, 262 | "timed_out": False 263 | }) 264 | self.assertEqual( 265 | [{"sector": "Health Care", "max_all_times": 522690000000.0, "max_at_2000": 20310000000.0, 266 | "max_per_sector": 19600000000.0, "_bucket_path": ["year_2000", "level3"]}, 267 | {"sector": "Technology", "max_all_times": 522690000000.0, "max_at_2000": 20310000000.0, 268 | "max_per_sector": 4440000000.0, "_bucket_path": ["year_2000", "level3"]}], 269 | rows) 270 | -------------------------------------------------------------------------------- /es_sql/tests/select_inside_branch/where_test.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | 3 | from es_sql import es_query 4 | 5 | 6 | class SelectInsideBranchWhereTest(unittest.TestCase): 7 | def test_filter_without_group_by(self): 8 | executor = es_query.create_executor([ 9 | "WITH all_symbols AS (SELECT MAX(market_cap) AS max_all_times FROM symbol)", 10 | "SELECT MAX(market_cap) AS max_at_2000 FROM all_symbols WHERE ipo_year=2000"]) 11 | self.assertEqual( 12 | {'aggs': {'level2': {'filter': {'term': {u'ipo_year': 2000}}, 13 | 'aggs': {u'max_at_2000': {u'max': {'field': u'market_cap'}}}}, 14 | u'max_all_times': {u'max': {'field': u'market_cap'}}}, 'size': 0}, 15 | executor.request) 16 | 17 | def test_two_filters(self): 18 | executor = es_query.create_executor([ 19 | "WITH all_symbols AS (SELECT MAX(market_cap) AS max_all_times FROM symbol)", 20 | "SELECT MAX(market_cap) AS max_at_2000 FROM all_symbols WHERE ipo_year=2000", 21 | "SELECT MAX(market_cap) AS max_at_2001 FROM all_symbols WHERE ipo_year=2001"]) 22 | self.assertEqual( 23 | {'aggs': {'level2': {'filter': {'term': {u'ipo_year': 2000}}, 24 | 'aggs': {u'max_at_2000': {u'max': {'field': u'market_cap'}}}}, 25 | 'level3': {'filter': {'term': {u'ipo_year': 2001}}, 26 | 'aggs': {u'max_at_2001': {u'max': {'field': u'market_cap'}}}}, 27 | u'max_all_times': {u'max': {'field': u'market_cap'}}}, 'size': 0}, 28 | executor.request) 29 | 30 | def test_filter_upon_filter(self): 31 | executor = es_query.create_executor([ 32 | "WITH all_symbols AS (SELECT MAX(market_cap) AS max_all_times FROM symbol)", 33 | "WITH year_2001 AS (SELECT MAX(market_cap) AS max_at_2000 FROM all_symbols WHERE ipo_year=2000)", 34 | "SELECT MAX(market_cap) AS max_at_2001_finance FROM year_2001 WHERE sector='Finance'"]) 35 | self.assertEqual( 36 | {'aggs': {'year_2001': {'filter': {'term': {u'ipo_year': 2000}}, 37 | 'aggs': {u'max_at_2000': {u'max': {'field': u'market_cap'}}, 38 | 'level3': {'filter': {'term': {u'sector': 'Finance'}}, 'aggs': { 39 | u'max_at_2001_finance': {u'max': {'field': u'market_cap'}}}}}}, 40 | u'max_all_times': {u'max': {'field': u'market_cap'}}}, 'size': 0}, 41 | executor.request) 42 | 43 | def test_filter_then_group_by(self): 44 | executor = es_query.create_executor([ 45 | "WITH all_symbols AS (SELECT MAX(market_cap) AS max_all_times FROM symbol)", 46 | "WITH year_2000 AS (SELECT MAX(market_cap) AS max_at_2000 FROM all_symbols WHERE ipo_year=2000)", 47 | "SELECT sector, MAX(market_cap) AS max_per_sector FROM year_2000 GROUP BY sector LIMIT 2"]) 48 | self.assertEqual( 49 | { 50 | "aggs": { 51 | "year_2000": { 52 | "filter": { 53 | "term": { 54 | "ipo_year": 2000 55 | } 56 | }, 57 | "aggs": { 58 | "max_at_2000": { 59 | "max": { 60 | "field": "market_cap" 61 | } 62 | }, 63 | "sector": { 64 | "terms": { 65 | "field": "sector", 66 | "size": 2 67 | }, 68 | "aggs": { 69 | "max_per_sector": { 70 | "max": { 71 | "field": "market_cap" 72 | } 73 | } 74 | } 75 | } 76 | } 77 | }, 78 | "max_all_times": { 79 | "max": { 80 | "field": "market_cap" 81 | } 82 | } 83 | }, 84 | "size": 0 85 | }, 86 | executor.request) 87 | -------------------------------------------------------------------------------- /es_sql/tests/select_inside_leaf/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/taowen/es-monitor/c4deceb4964857f495d13bfaf2d92f36734c9e1c/es_sql/tests/select_inside_leaf/__init__.py -------------------------------------------------------------------------------- /es_sql/tests/select_inside_leaf/group_by_test.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | 3 | from es_sql import es_query 4 | 5 | 6 | class SelectInsideLeafGroupByTest(unittest.TestCase): 7 | def test_group_by_one(self): 8 | executor = es_query.create_executor("SELECT ipo_year, COUNT(*) FROM symbol GROUP BY ipo_year") 9 | self.assertEqual( 10 | {'aggs': {'ipo_year': {'terms': {'field': 'ipo_year', 'size': 0}, 'aggs': {}}}, 'size': 0}, 11 | executor.request) 12 | 13 | def test_group_by_field_as_alias(self): 14 | executor = es_query.create_executor("SELECT year, COUNT(*) FROM symbol GROUP BY ipo_year AS year") 15 | self.assertEqual( 16 | {'aggs': {'year': {'terms': {'field': 'ipo_year', 'size': 0}, 'aggs': {}}}, 'size': 0}, 17 | executor.request) 18 | 19 | def test_group_by_can_be_put_in_select(self): 20 | executor = es_query.create_executor("SELECT ipo_year AS year, COUNT(*) FROM symbol GROUP BY year") 21 | self.assertEqual( 22 | {'aggs': {'year': {'terms': {'field': 'ipo_year', 'size': 0}, 'aggs': {}}}, 'size': 0}, 23 | executor.request) 24 | 25 | def test_group_by_two(self): 26 | executor = es_query.create_executor("SELECT ipo_year, COUNT(*) FROM symbol GROUP BY ipo_year, abc") 27 | self.assertEqual( 28 | {'aggs': { 29 | 'ipo_year': {'terms': {'field': 'ipo_year', 'size': 0}, 'aggs': { 30 | 'abc': {'terms': {'field': 'abc', 'size': 0}, 'aggs': {}}}}}, 31 | 'size': 0}, 32 | executor.request) 33 | 34 | def test_group_by_date_trunc(self): 35 | executor = es_query.create_executor( 36 | "SELECT year, MAX(adj_close) FROM quote WHERE symbol='AAPL' " 37 | "GROUP BY date_trunc('year',\"date\") AS year") 38 | self.assertEqual( 39 | {'query': {'term': {'symbol': 'AAPL'}}, 'aggs': { 40 | 'year': {'date_histogram': {'field': 'date', 'interval': 'year', 'time_zone': '+08:00'}, 41 | 'aggs': {'MAX(adj_close)': {'max': {'field': 'adj_close'}}}}}, 'size': 0}, 42 | executor.request) 43 | 44 | def test_group_by_date_trunc(self): 45 | executor = es_query.create_executor( 46 | "SELECT year, MAX(adj_close) FROM quote WHERE symbol='AAPL' " 47 | "GROUP BY TO_CHAR(date_trunc('year',\"date\"), '%Y-%m-%d') AS year") 48 | self.assertEqual( 49 | {'query': {'term': {'symbol': 'AAPL'}}, 'aggs': { 50 | 'year': {'date_histogram': {'field': 'date', 'interval': 'year', 51 | 'time_zone': '+08:00', 'format': 'yyyy-MM-dd'}, 52 | 'aggs': {'MAX(adj_close)': {'max': {'field': 'adj_close'}}}}}, 'size': 0}, 53 | executor.request) 54 | 55 | def test_group_by_histogram(self): 56 | executor = es_query.create_executor( 57 | "SELECT ipo_year_range, COUNT(*) FROM symbol " 58 | "GROUP BY histogram(ipo_year, 5) AS ipo_year_range") 59 | self.assertEqual( 60 | {'aggs': {'ipo_year_range': {'aggs': {}, 'histogram': {'field': 'ipo_year', 'interval': 5}}}, 'size': 0}, 61 | executor.request) 62 | 63 | def test_group_by_numeric_range(self): 64 | executor = es_query.create_executor( 65 | "SELECT ipo_year_range, COUNT(*) FROM symbol " 66 | "GROUP BY CASE " 67 | " WHEN ipo_year_range >= 2000 THEN 'post_2000' " 68 | " WHEN ipo_year_range < 2000 THEN 'pre_2000' END AS ipo_year_range") 69 | self.assertEqual( 70 | {'aggs': {'ipo_year_range': { 71 | 'range': {'ranges': [{'from': 2000.0, 'key': 'post_2000'}, {'to': 2000.0, 'key': 'pre_2000'}], 72 | 'field': 'ipo_year_range'}, 'aggs': {}}}, 'size': 0}, 73 | executor.request) 74 | 75 | def test_group_by_filters(self): 76 | executor = es_query.create_executor( 77 | "SELECT ipo_year_range, COUNT(*) FROM symbol " 78 | "GROUP BY CASE " 79 | " WHEN ipo_year_range > 2000 THEN 'post_2000' " 80 | " WHEN ipo_year_range < 2000 THEN 'pre_2000'" 81 | " ELSE '2000' END AS ipo_year_range") 82 | self.assertEqual( 83 | {'aggs': {'ipo_year_range': {'filters': { 84 | 'filters': {'pre_2000': {'range': {'ipo_year_range': {'lt': 2000}}}, 85 | 'post_2000': {'range': {'ipo_year_range': {'gt': 2000}}}}, 86 | 'other_bucket_key': '2000'}, 87 | 'aggs': {}}}, 'size': 0}, 88 | executor.request) 89 | 90 | def test_group_by_single_value_script(self): 91 | executor = es_query.create_executor( 92 | "SELECT ipo_year_range, COUNT(*) FROM symbol GROUP BY ipo_year / 6 AS ipo_year_range") 93 | self.assertEqual( 94 | {'aggs': {'ipo_year_range': { 95 | 'terms': {'field': 'ipo_year', 'size': 0, 'script': {'lang': 'expression', 'inline': '_value / 6'}}, 96 | 'aggs': {}}}, 'size': 0}, 97 | executor.request) 98 | 99 | def test_group_by_multiple_values_script(self): 100 | executor = es_query.create_executor( 101 | "SELECT shares_count, COUNT(*) FROM symbol GROUP BY market_cap / last_sale AS shares_count") 102 | self.assertEqual( 103 | {'aggs': {'shares_count': { 104 | 'terms': {'size': 0, 'script': { 105 | 'lang': 'expression', 106 | 'inline': "doc['market_cap'].value / doc['last_sale'].value"}}, 107 | 'aggs': {}}}, 'size': 0}, 108 | executor.request) 109 | 110 | def test_group_by_function(self): 111 | executor = es_query.create_executor( 112 | "SELECT shares_count, COUNT(*) FROM symbol GROUP BY floor(market_cap / last_sale) AS shares_count") 113 | self.assertEqual( 114 | {'aggs': {'shares_count': { 115 | 'terms': {'size': 0, 'script': { 116 | 'lang': 'expression', 117 | 'inline': "floor(doc['market_cap'].value / doc['last_sale'].value)"}}, 118 | 'aggs': {}}}, 'size': 0}, 119 | executor.request) 120 | -------------------------------------------------------------------------------- /es_sql/tests/select_inside_leaf/having_test.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | 3 | from es_sql import es_query 4 | 5 | 6 | class SelectInsideLeafHavingTest(unittest.TestCase): 7 | def test_having_by_count(self): 8 | executor = es_query.create_executor( 9 | "SELECT ipo_year, COUNT(*) AS ipo_count FROM symbol GROUP BY ipo_year HAVING ipo_count > 100") 10 | self.assertEqual( 11 | {'aggs': {u'ipo_year': {'terms': {'field': u'ipo_year', 'size': 0}, 'aggs': {'having': { 12 | 'bucket_selector': {'buckets_path': {u'ipo_count': '_count'}, 13 | 'script': {'lang': 'expression', 'inline': u' ipo_count > 100'}}}}}}, 'size': 0}, 14 | executor.request) 15 | 16 | def test_having_by_key(self): 17 | executor = es_query.create_executor( 18 | "SELECT ipo_year, COUNT(*) AS ipo_count FROM symbol GROUP BY ipo_year HAVING ipo_year > 100") 19 | {'aggs': {u'ipo_year': {'terms': {'field': u'ipo_year', 'size': 0}, 'aggs': {'having': { 20 | 'bucket_selector': {'buckets_path': {u'ipo_year': '_key'}, 21 | 'script': {'lang': 'expression', 'inline': u' _key > 100'}}}}}}, 'size': 0} 22 | 23 | self.assertEqual( 24 | {'aggs': {'ipo_year': {'terms': {'field': 'ipo_year', 'size': 0}, 'aggs': {'having': { 25 | 'bucket_selector': {'buckets_path': {'ipo_year': '_key'}, 26 | 'script': {'lang': 'expression', 'inline': ' ipo_year > 100'}}}}}}, 'size': 0}, 27 | executor.request) 28 | 29 | def test_having_by_metric(self): 30 | executor = es_query.create_executor( 31 | "SELECT ipo_year, MAX(market_cap) AS max_market_cap FROM symbol GROUP BY ipo_year HAVING max_market_cap > 100") 32 | self.assertEqual( 33 | {'aggs': {'ipo_year': {'terms': {'field': 'ipo_year', 'size': 0}, 'aggs': {'having': { 34 | 'bucket_selector': { 35 | 'buckets_path': {'max_market_cap': 'max_market_cap'}, 36 | 'script': { 37 | 'lang': 'expression', 'inline': ' max_market_cap > 100'}}}, 38 | 'max_market_cap': {'max': { 39 | 'field': 'market_cap'}}}}}, 40 | 'size': 0}, 41 | executor.request) 42 | -------------------------------------------------------------------------------- /es_sql/tests/select_inside_leaf/metric_test.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | 3 | from es_sql import es_query 4 | 5 | 6 | class SelectInsideLeafMetricTest(unittest.TestCase): 7 | def test_count_star(self): 8 | executor = es_query.create_executor("SELECT COUNT(*) FROM symbol") 9 | self.assertEqual({'aggs': {}, 'size': 0}, executor.request) 10 | 11 | def test_count_field(self): 12 | executor = es_query.create_executor("SELECT COUNT(ipo_year) FROM symbol") 13 | self.assertEqual( 14 | {'aggs': {'COUNT(ipo_year)': {'value_count': {'field': 'ipo_year'}}}, 'size': 0}, 15 | executor.request) 16 | 17 | def test_count_distinct(self): 18 | executor = es_query.create_executor("SELECT COUNT(DISTINCT ipo_year) FROM symbol") 19 | self.assertEqual( 20 | {'aggs': {'COUNT(DISTINCT ipo_year)': {'cardinality': {'field': 'ipo_year'}}}, 'size': 0}, 21 | executor.request) 22 | 23 | def test_max(self): 24 | executor = es_query.create_executor("SELECT MAX(ipo_year) FROM symbol") 25 | self.assertEqual( 26 | {'aggs': {'MAX(ipo_year)': {'max': {'field': 'ipo_year'}}}, 'size': 0}, 27 | executor.request) 28 | 29 | def test_min(self): 30 | executor = es_query.create_executor("SELECT MIN(ipo_year) FROM symbol") 31 | self.assertEqual( 32 | {'aggs': {'MIN(ipo_year)': {'min': {'field': 'ipo_year'}}}, 'size': 0}, 33 | executor.request) 34 | 35 | def test_avg(self): 36 | executor = es_query.create_executor("SELECT AVG(ipo_year) FROM symbol") 37 | self.assertEqual( 38 | {'aggs': {'AVG(ipo_year)': {'avg': {'field': 'ipo_year'}}}, 'size': 0}, 39 | executor.request) 40 | 41 | def test_sum(self): 42 | executor = es_query.create_executor("SELECT SUM(market_cap) FROM symbol") 43 | self.assertEqual( 44 | {'aggs': {'SUM(market_cap)': {'sum': {'field': 'market_cap'}}}, 'size': 0}, 45 | executor.request) 46 | 47 | def test_count_dot(self): 48 | executor = es_query.create_executor("SELECT COUNT(a.b) FROM symbol") 49 | self.assertEqual( 50 | {'aggs': {'COUNT(a.b)': {'value_count': {'field': u'a.b'}}}, 'size': 0}, 51 | executor.request) 52 | 53 | def test_count_distinct_dot(self): 54 | executor = es_query.create_executor("SELECT COUNT(DISTINCT a.b) FROM symbol") 55 | self.assertEqual( 56 | {'aggs': {'COUNT(DISTINCT a.b)': {'cardinality': {'field': u'a.b'}}}, 'size': 0}, 57 | executor.request) 58 | 59 | def test_sum_of_squares(self): 60 | executor = es_query.create_executor("SELECT sum_of_squares(a) FROM symbol") 61 | self.assertEqual( 62 | {'aggs': {u'a_extended_stats': {'extended_stats': {'field': u'a'}}}, 'size': 0}, 63 | executor.request) 64 | 65 | def test_sum_of_squares_with_std_deviation(self): 66 | executor = es_query.create_executor("SELECT sum_of_squares(a), std_deviation(a) FROM symbol") 67 | self.assertEqual( 68 | {'aggs': {u'a_extended_stats': {'extended_stats': {'field': u'a'}}}, 'size': 0}, 69 | executor.request) 70 | -------------------------------------------------------------------------------- /es_sql/tests/select_inside_leaf/order_by_test.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | 3 | from es_sql import es_query 4 | 5 | 6 | class SelectInsideLeafOrderByTest(unittest.TestCase): 7 | def test_order_by_term(self): 8 | executor = es_query.create_executor( 9 | "SELECT ipo_year, COUNT(*) FROM symbol GROUP BY ipo_year ORDER BY ipo_year") 10 | self.assertEqual( 11 | {'aggs': { 12 | 'ipo_year': {'terms': {'field': 'ipo_year', 'order': {'_term': 'asc'}, 'size': 0}, 'aggs': {}}}, 13 | 'size': 0}, 14 | executor.request) 15 | 16 | def test_order_by_count(self): 17 | executor = es_query.create_executor( 18 | "SELECT ipo_year, COUNT(*) AS c FROM symbol GROUP BY ipo_year ORDER BY c") 19 | self.assertEqual( 20 | {'aggs': { 21 | 'ipo_year': {'terms': {'field': 'ipo_year', 'order': {'_count': 'asc'}, 'size': 0}, 'aggs': {}}}, 22 | 'size': 0}, 23 | executor.request) 24 | 25 | def test_order_by_metric(self): 26 | executor = es_query.create_executor( 27 | "SELECT ipo_year, MAX(market_cap) AS c FROM symbol GROUP BY ipo_year ORDER BY c") 28 | self.assertEqual( 29 | {'aggs': {'ipo_year': {'terms': {'field': 'ipo_year', 'order': {'c': 'asc'}, 'size': 0}, 30 | 'aggs': {'c': {'max': {'field': 'market_cap'}}}}}, 'size': 0}, 31 | executor.request) 32 | 33 | def test_order_by_histogram(self): 34 | executor = es_query.create_executor( 35 | "SELECT ipo_year_range, MAX(market_cap) AS max_market_cap FROM symbol " 36 | "GROUP BY histogram(ipo_year, 3) AS ipo_year_range ORDER BY ipo_year_range LIMIT 2") 37 | self.assertEqual( 38 | {'aggs': {'ipo_year_range': {'aggs': {'max_market_cap': {'max': {'field': 'market_cap'}}}, 39 | 'histogram': {'field': 'ipo_year', 'interval': 3, 'order': {'_key': 'asc'}, 40 | 'size': 2}}}, 'size': 0}, 41 | executor.request) 42 | 43 | def test_order_by_extended_stats(self): 44 | executor = es_query.create_executor( 45 | "SELECT ipo_year, STD_DEVIATION(market_cap) AS s FROM symbol GROUP BY ipo_year ORDER BY s") 46 | self.assertEqual( 47 | {'aggs': {u'ipo_year': { 48 | 'terms': {'field': u'ipo_year', 'order': {'market_cap_extended_stats.std_deviation': 'asc'}, 49 | 'size': 0}, 'aggs': { 50 | u'market_cap_extended_stats': {'extended_stats': {'field': u'market_cap'}}}}}, 'size': 0}, 51 | executor.request) 52 | -------------------------------------------------------------------------------- /es_sql/tests/select_inside_leaf/response_test.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | 3 | from es_sql import es_query 4 | 5 | 6 | class SelectInsideLeafResponseTest(unittest.TestCase): 7 | def test_no_group_by(self): 8 | executor = es_query.create_executor("select count(*) from quote") 9 | rows = executor.select_response({ 10 | "hits": { 11 | "hits": [], 12 | "total": 20994400, 13 | "max_score": 0.0 14 | }, 15 | "_shards": { 16 | "successful": 1, 17 | "failed": 0, 18 | "total": 1 19 | }, 20 | "took": 26, 21 | "timed_out": False 22 | }) 23 | self.assertEqual([{'count(*)': 20994400}], rows) 24 | 25 | def test_single_group_by(self): 26 | executor = es_query.create_executor("select exchange, count(*) from symbol group by exchange") 27 | rows = executor.select_response({ 28 | "hits": { 29 | "hits": [], 30 | "total": 6714, 31 | "max_score": 0.0 32 | }, 33 | "_shards": { 34 | "successful": 1, 35 | "failed": 0, 36 | "total": 1 37 | }, 38 | "took": 23, 39 | "aggregations": { 40 | "exchange": { 41 | "buckets": [ 42 | { 43 | "key": "nyse", 44 | "doc_count": 3240 45 | }, 46 | { 47 | "key": "nasdaq", 48 | "doc_count": 3089 49 | }, 50 | { 51 | "key": "nyse mkt", 52 | "doc_count": 385 53 | } 54 | ], 55 | "sum_other_doc_count": 0, 56 | "doc_count_error_upper_bound": 0 57 | } 58 | }, 59 | "timed_out": False 60 | }) 61 | self.assertEqual( 62 | [{'count(*)': 3240, u'exchange': 'nyse'}, 63 | {'count(*)': 3089, u'exchange': 'nasdaq'}, 64 | {'count(*)': 385, u'exchange': 'nyse mkt'}], 65 | rows) 66 | 67 | def test_multiple_group_by(self): 68 | executor = es_query.create_executor( 69 | "select exchange, sector, max(market_cap) from symbol group by exchange, sector") 70 | rows = executor.select_response({ 71 | "hits": { 72 | "hits": [], 73 | "total": 6714, 74 | "max_score": 0.0 75 | }, 76 | "_shards": { 77 | "successful": 1, 78 | "failed": 0, 79 | "total": 1 80 | }, 81 | "took": 11, 82 | "aggregations": { 83 | "exchange": { 84 | "buckets": [ 85 | { 86 | "sector": { 87 | "buckets": [ 88 | { 89 | "max(market_cap)": { 90 | "value": 1409695805.0 91 | }, 92 | "key": "n/a", 93 | "doc_count": 963 94 | } 95 | ], 96 | "sum_other_doc_count": 0, 97 | "doc_count_error_upper_bound": 0 98 | }, 99 | "key": "nyse", 100 | "doc_count": 3240 101 | }, 102 | { 103 | "sector": { 104 | "buckets": [ 105 | { 106 | "max(market_cap)": { 107 | "value": 30620000000.0 108 | }, 109 | "key": "Finance", 110 | "doc_count": 637 111 | }, 112 | { 113 | "max(market_cap)": { 114 | "value": 126540000000.0 115 | }, 116 | "key": "Health Care", 117 | "doc_count": 621 118 | } 119 | ], 120 | "sum_other_doc_count": 0, 121 | "doc_count_error_upper_bound": 0 122 | }, 123 | "key": "nasdaq", 124 | "doc_count": 3089 125 | }, 126 | { 127 | "sector": { 128 | "buckets": [ 129 | { 130 | "max(market_cap)": { 131 | "value": 971774087.0 132 | }, 133 | "key": "n/a", 134 | "doc_count": 123 135 | }, 136 | { 137 | "max(market_cap)": { 138 | "value": 424184478.0 139 | }, 140 | "key": "Basic Industries", 141 | "doc_count": 52 142 | } 143 | ], 144 | "sum_other_doc_count": 0, 145 | "doc_count_error_upper_bound": 0 146 | }, 147 | "key": "nyse mkt", 148 | "doc_count": 385 149 | } 150 | ], 151 | "sum_other_doc_count": 0, 152 | "doc_count_error_upper_bound": 0 153 | } 154 | }, 155 | "timed_out": False 156 | }) 157 | self.assertEqual( 158 | [{u'sector': 'n/a', 'max(market_cap)': 1409695805.0, u'exchange': 'nyse'}, 159 | {u'sector': 'Finance', 'max(market_cap)': 30620000000.0, u'exchange': 'nasdaq'}, 160 | {u'sector': 'Health Care', 'max(market_cap)': 126540000000.0, u'exchange': 'nasdaq'}, 161 | {u'sector': 'n/a', 'max(market_cap)': 971774087.0, u'exchange': 'nyse mkt'}, 162 | {u'sector': 'Basic Industries', 'max(market_cap)': 424184478.0, u'exchange': 'nyse mkt'}], 163 | rows) 164 | 165 | def test_sum_of_squares(self): 166 | executor = es_query.create_executor("SELECT sum_of_squares(last_sale), std_deviation(last_sale) FROM symbol") 167 | rows = executor.select_response({ 168 | "hits": { 169 | "hits": [], 170 | "total": 6714, 171 | "max_score": 0.0 172 | }, 173 | "_shards": { 174 | "successful": 3, 175 | "failed": 0, 176 | "total": 3 177 | }, 178 | "took": 5, 179 | "aggregations": { 180 | "last_sale_extended_stats": { 181 | "count": 6634, 182 | "min": 0.0, 183 | "sum_of_squares": 320576400178.0, 184 | "max": 269500.0, 185 | "sum": 17407390.0, 186 | "std_deviation": 6437.239059099383, 187 | "std_deviation_bounds": { 188 | "upper": 15498.444051270819, 189 | "lower": -10250.512185126712 190 | }, 191 | "variance": 41438046.703994706, 192 | "avg": 2623.965933072053 193 | } 194 | }, 195 | "timed_out": False 196 | }) 197 | self.assertEqual([ 198 | {'sum_of_squares(last_sale)': 320576400178.0, 'std_deviation(last_sale)': 6437.239059099383}], 199 | rows) 200 | -------------------------------------------------------------------------------- /explorer-ui/.babelrc: -------------------------------------------------------------------------------- 1 | { 2 | "presets": ["es2015", "stage-2"], 3 | "plugins": ["transform-runtime"], 4 | "comments": false 5 | } 6 | -------------------------------------------------------------------------------- /explorer-ui/.eslintrc.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | root: true, 3 | // https://github.com/feross/standard/blob/master/RULES.md#javascript-standard-style 4 | extends: 'standard', 5 | // required to lint *.vue files 6 | plugins: [ 7 | 'html' 8 | ], 9 | // add your custom rules here 10 | 'rules': { 11 | // allow paren-less arrow functions 12 | 'arrow-parens': 0, 13 | // allow debugger during development 14 | 'no-debugger': process.env.NODE_ENV === 'production' ? 2 : 0 15 | } 16 | } 17 | -------------------------------------------------------------------------------- /explorer-ui/.gitignore: -------------------------------------------------------------------------------- 1 | .DS_Store 2 | node_modules/ 3 | dist/ 4 | npm-debug.log 5 | -------------------------------------------------------------------------------- /explorer-ui/README.md: -------------------------------------------------------------------------------- 1 | # explorer-ui 2 | 3 | > A Vue.js project 4 | 5 | ## Build Setup 6 | 7 | ``` bash 8 | # install dependencies 9 | npm install 10 | 11 | # serve with hot reload at localhost:8080 12 | npm run dev 13 | 14 | # build for production with minification 15 | npm run build 16 | 17 | # run unit tests 18 | npm test 19 | ``` 20 | 21 | For detailed explanation on how things work, consult the [docs for vue-loader](http://vuejs.github.io/vue-loader). 22 | -------------------------------------------------------------------------------- /explorer-ui/build/dev-client.js: -------------------------------------------------------------------------------- 1 | var hotClient = require('webpack-hot-middleware/client?noInfo=true&reload=true') 2 | 3 | hotClient.subscribe(function (event) { 4 | if (event.action === 'reload') { 5 | window.location.reload() 6 | } 7 | }) 8 | -------------------------------------------------------------------------------- /explorer-ui/build/dev-server.js: -------------------------------------------------------------------------------- 1 | var express = require('express') 2 | var proxy = require('express-http-proxy') 3 | var webpack = require('webpack') 4 | var config = require('./webpack.dev.conf') 5 | 6 | var app = express() 7 | var compiler = webpack(config) 8 | 9 | var devMiddleware = require('webpack-dev-middleware')(compiler, { 10 | publicPath: config.output.publicPath, 11 | stats: { 12 | colors: true, 13 | chunks: false 14 | } 15 | }) 16 | 17 | var hotMiddleware = require('webpack-hot-middleware')(compiler) 18 | // force page reload when html-webpack-plugin template changes 19 | compiler.plugin('compilation', function (compilation) { 20 | compilation.plugin('html-webpack-plugin-after-emit', function (data, cb) { 21 | hotMiddleware.publish({ action: 'reload' }) 22 | cb() 23 | }) 24 | }) 25 | 26 | // handle fallback for HTML5 history API 27 | app.use(require('connect-history-api-fallback')()) 28 | // serve webpack bundle output 29 | app.use(devMiddleware) 30 | // enable hot-reload and state-preserving 31 | // compilation error display 32 | app.use(hotMiddleware) 33 | 34 | app.use('/translate', proxy('127.0.0.1:8000', { 35 | forwardPath: function(req, res) { 36 | return '/translate'; 37 | } 38 | })); 39 | 40 | app.listen(8080, function (err) { 41 | if (err) { 42 | console.log(err) 43 | return 44 | } 45 | console.log('Listening at http://localhost:8080') 46 | }) 47 | -------------------------------------------------------------------------------- /explorer-ui/build/karma.conf.js: -------------------------------------------------------------------------------- 1 | var webpackConf = require('./webpack.base.conf') 2 | delete webpackConf.entry 3 | 4 | module.exports = function (config) { 5 | config.set({ 6 | browsers: ['PhantomJS'], 7 | frameworks: ['jasmine'], 8 | reporters: ['spec'], 9 | files: ['../test/unit/index.js'], 10 | preprocessors: { 11 | '../test/unit/index.js': ['webpack'] 12 | }, 13 | webpack: webpackConf, 14 | webpackMiddleware: { 15 | noInfo: true 16 | } 17 | }) 18 | } 19 | -------------------------------------------------------------------------------- /explorer-ui/build/webpack.base.conf.js: -------------------------------------------------------------------------------- 1 | var path = require('path') 2 | 3 | module.exports = { 4 | entry: { 5 | app: './src/main.js' 6 | }, 7 | output: { 8 | path: path.resolve(__dirname, '../dist/static'), 9 | publicPath: '/static/', 10 | filename: '[name].js' 11 | }, 12 | resolve: { 13 | extensions: ['', '.js', '.vue'], 14 | alias: { 15 | 'src': path.resolve(__dirname, '../src') 16 | } 17 | }, 18 | resolveLoader: { 19 | root: path.join(__dirname, 'node_modules') 20 | }, 21 | module: { 22 | preLoaders: [ 23 | { 24 | test: /\.vue$/, 25 | loader: 'eslint', 26 | exclude: /node_modules/ 27 | }, 28 | { 29 | test: /\.js$/, 30 | loader: 'eslint', 31 | exclude: /node_modules/ 32 | } 33 | ], 34 | loaders: [ 35 | { 36 | test: /\.vue$/, 37 | loader: 'vue' 38 | }, 39 | { 40 | test: /\.js$/, 41 | loader: 'babel', 42 | exclude: /node_modules/ 43 | }, 44 | { 45 | test: /\.json$/, 46 | loader: 'json' 47 | }, 48 | { 49 | test: /\.(png|jpg|gif|svg)$/, 50 | loader: 'url', 51 | query: { 52 | limit: 10000, 53 | name: '[name].[ext]?[hash:7]' 54 | } 55 | } 56 | ] 57 | }, 58 | eslint: { 59 | formatter: require('eslint-friendly-formatter') 60 | } 61 | } 62 | -------------------------------------------------------------------------------- /explorer-ui/build/webpack.dev.conf.js: -------------------------------------------------------------------------------- 1 | var webpack = require('webpack') 2 | var config = require('./webpack.base.conf') 3 | var HtmlWebpackPlugin = require('html-webpack-plugin') 4 | 5 | // eval-source-map is faster for development 6 | config.devtool = 'eval-source-map' 7 | 8 | // add hot-reload related code to entry chunks 9 | var polyfill = 'eventsource-polyfill' 10 | var devClient = './build/dev-client' 11 | Object.keys(config.entry).forEach(function (name, i) { 12 | var extras = i === 0 ? [polyfill, devClient] : [devClient] 13 | config.entry[name] = extras.concat(config.entry[name]) 14 | }) 15 | 16 | // necessary for the html plugin to work properly 17 | // when serving the html from in-memory 18 | config.output.publicPath = '/' 19 | 20 | config.plugins = (config.plugins || []).concat([ 21 | // https://github.com/glenjamin/webpack-hot-middleware#installation--usage 22 | new webpack.optimize.OccurenceOrderPlugin(), 23 | new webpack.HotModuleReplacementPlugin(), 24 | new webpack.NoErrorsPlugin(), 25 | // https://github.com/ampedandwired/html-webpack-plugin 26 | new HtmlWebpackPlugin({ 27 | filename: 'index.html', 28 | template: 'src/index.html', 29 | inject: true 30 | }) 31 | ]) 32 | 33 | module.exports = config 34 | -------------------------------------------------------------------------------- /explorer-ui/build/webpack.prod.conf.js: -------------------------------------------------------------------------------- 1 | var webpack = require('webpack') 2 | var config = require('./webpack.base.conf') 3 | var ExtractTextPlugin = require('extract-text-webpack-plugin') 4 | var HtmlWebpackPlugin = require('html-webpack-plugin') 5 | 6 | // naming output files with hashes for better caching. 7 | // dist/index.html will be auto-generated with correct URLs. 8 | config.output.filename = '[name].[chunkhash].js' 9 | config.output.chunkFilename = '[id].[chunkhash].js' 10 | 11 | // whether to generate source map for production files. 12 | // disabling this can speed up the build. 13 | var SOURCE_MAP = true 14 | 15 | config.devtool = SOURCE_MAP ? 'source-map' : false 16 | 17 | // generate loader string to be used with extract text plugin 18 | function generateExtractLoaders (loaders) { 19 | return loaders.map(function (loader) { 20 | return loader + '-loader' + (SOURCE_MAP ? '?sourceMap' : '') 21 | }).join('!') 22 | } 23 | 24 | // http://vuejs.github.io/vue-loader/configurations/extract-css.html 25 | var cssExtractLoaders = { 26 | css: ExtractTextPlugin.extract('vue-style-loader', generateExtractLoaders(['css'])), 27 | less: ExtractTextPlugin.extract('vue-style-loader', generateExtractLoaders(['css', 'less'])), 28 | sass: ExtractTextPlugin.extract('vue-style-loader', generateExtractLoaders(['css', 'sass'])), 29 | stylus: ExtractTextPlugin.extract('vue-style-loader', generateExtractLoaders(['css', 'stylus'])) 30 | } 31 | 32 | config.vue = config.vue || {} 33 | config.vue.loaders = config.vue.loaders || {} 34 | Object.keys(cssExtractLoaders).forEach(function (key) { 35 | config.vue.loaders[key] = cssExtractLoaders[key] 36 | }) 37 | 38 | config.plugins = (config.plugins || []).concat([ 39 | // http://vuejs.github.io/vue-loader/workflow/production.html 40 | new webpack.DefinePlugin({ 41 | 'process.env': { 42 | NODE_ENV: '"production"' 43 | } 44 | }), 45 | new webpack.optimize.UglifyJsPlugin({ 46 | compress: { 47 | warnings: false 48 | } 49 | }), 50 | new webpack.optimize.OccurenceOrderPlugin(), 51 | // extract css into its own file 52 | new ExtractTextPlugin('[name].[contenthash].css'), 53 | // generate dist index.html with correct asset hash for caching. 54 | // you can customize output by editing /src/index.html 55 | // see https://github.com/ampedandwired/html-webpack-plugin 56 | new HtmlWebpackPlugin({ 57 | filename: '../index.html', 58 | template: 'src/index.html', 59 | inject: true, 60 | minify: { 61 | removeComments: true, 62 | collapseWhitespace: true, 63 | removeAttributeQuotes: true 64 | // more options: 65 | // https://github.com/kangax/html-minifier#options-quick-reference 66 | } 67 | }) 68 | ]) 69 | 70 | module.exports = config 71 | -------------------------------------------------------------------------------- /explorer-ui/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "explorer-ui", 3 | "description": "A Vue.js project", 4 | "author": "taowen", 5 | "private": true, 6 | "scripts": { 7 | "dev": "node build/dev-server.js", 8 | "build": "rimraf dist && cross-env NODE_ENV=production webpack --progress --hide-modules --config build/webpack.prod.conf.js", 9 | "test": "karma start build/karma.conf.js --single-run" 10 | }, 11 | "dependencies": { 12 | "vue": "^1.0.16", 13 | "vue-resource": "^0.7.0" 14 | }, 15 | "devDependencies": { 16 | "babel-core": "^6.0.0", 17 | "babel-loader": "^6.0.0", 18 | "babel-plugin-transform-runtime": "^6.0.0", 19 | "babel-preset-es2015": "^6.0.0", 20 | "babel-preset-stage-2": "^6.0.0", 21 | "babel-runtime": "^5.8.0", 22 | "connect-history-api-fallback": "^1.1.0", 23 | "cross-env": "^1.0.7", 24 | "css-loader": "^0.23.0", 25 | "eslint": "^2.0.0", 26 | "eslint-config-standard": "^5.1.0", 27 | "eslint-friendly-formatter": "^1.2.2", 28 | "eslint-loader": "^1.3.0", 29 | "eslint-plugin-html": "^1.3.0", 30 | "eslint-plugin-promise": "^1.0.8", 31 | "eslint-plugin-standard": "^1.3.2", 32 | "eventsource-polyfill": "^0.9.6", 33 | "express": "^4.13.3", 34 | "express-http-proxy": "^0.6.0", 35 | "extract-text-webpack-plugin": "^1.0.1", 36 | "file-loader": "^0.8.4", 37 | "function-bind": "^1.0.2", 38 | "html-webpack-plugin": "^2.8.1", 39 | "inject-loader": "^2.0.1", 40 | "jasmine-core": "^2.4.1", 41 | "json-loader": "^0.5.4", 42 | "karma": "^0.13.15", 43 | "karma-jasmine": "^0.3.6", 44 | "karma-phantomjs-launcher": "^1.0.0", 45 | "karma-spec-reporter": "0.0.24", 46 | "karma-webpack": "^1.7.0", 47 | "phantomjs-prebuilt": "^2.1.3", 48 | "rimraf": "^2.5.0", 49 | "url-loader": "^0.5.7", 50 | "vue-hot-reload-api": "^1.2.0", 51 | "vue-html-loader": "^1.0.0", 52 | "vue-loader": "^8.1.3", 53 | "vue-style-loader": "^1.0.0", 54 | "webpack": "^1.12.2", 55 | "webpack-dev-middleware": "^1.4.0", 56 | "webpack-hot-middleware": "^2.6.0" 57 | } 58 | } 59 | -------------------------------------------------------------------------------- /explorer-ui/src/App.vue: -------------------------------------------------------------------------------- 1 | 6 | 7 | 18 | 19 | 25 | -------------------------------------------------------------------------------- /explorer-ui/src/assets/logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/taowen/es-monitor/c4deceb4964857f495d13bfaf2d92f36734c9e1c/explorer-ui/src/assets/logo.png -------------------------------------------------------------------------------- /explorer-ui/src/components/Query.vue: -------------------------------------------------------------------------------- 1 | 19 | 20 | 40 | 41 | -------------------------------------------------------------------------------- /explorer-ui/src/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | Data Explorer 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | -------------------------------------------------------------------------------- /explorer-ui/src/main.js: -------------------------------------------------------------------------------- 1 | import Vue from 'vue' 2 | import App from './App' 3 | 4 | /* eslint-disable no-new */ 5 | new Vue({ 6 | el: 'body', 7 | components: { App } 8 | }) 9 | -------------------------------------------------------------------------------- /explorer-ui/test/unit/Hello.spec.js: -------------------------------------------------------------------------------- 1 | /* global describe, it, expect */ 2 | 3 | import Vue from 'vue' 4 | import Hello from 'src/components/Hello' 5 | 6 | describe('Hello.vue', () => { 7 | it('should render correct contents', () => { 8 | const vm = new Vue({ 9 | template: '
', 10 | components: { Hello } 11 | }).$mount() 12 | expect(vm.$el.querySelector('.hello h1').textContent).toBe('Hello World!') 13 | }) 14 | }) 15 | 16 | // also see example testing a component with mocks at 17 | // https://github.com/vuejs/vue-loader-example/blob/master/test/unit/a.spec.js#L24-L49 18 | -------------------------------------------------------------------------------- /explorer-ui/test/unit/index.js: -------------------------------------------------------------------------------- 1 | // Polyfill fn.bind() for PhantomJS 2 | /* eslint-disable no-extend-native */ 3 | Function.prototype.bind = require('function-bind') 4 | 5 | // require all test files (files that ends with .spec.js) 6 | var testsContext = require.context('.', true, /\.spec$/) 7 | testsContext.keys().forEach(testsContext) 8 | -------------------------------------------------------------------------------- /explorer/__init__.py: -------------------------------------------------------------------------------- 1 | from app import app -------------------------------------------------------------------------------- /explorer/__main__.py: -------------------------------------------------------------------------------- 1 | import os 2 | import distutils.spawn 3 | import sys 4 | 5 | BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) 6 | try: 7 | os.mkdir(os.path.join('log')) 8 | except: 9 | pass 10 | os.chdir(BASE_DIR) 11 | os.execv(distutils.spawn.find_executable('gunicorn'), [ 12 | 'gunicorn', 'explorer:app', '-k', 'gevent', 13 | '--access-logfile', 'log/access.log', 14 | '--error-logfile', 'log/error.log'] + sys.argv[1:]) 15 | -------------------------------------------------------------------------------- /explorer/app.py: -------------------------------------------------------------------------------- 1 | import json 2 | import logging 3 | from logging.handlers import RotatingFileHandler 4 | import os 5 | import sys 6 | import traceback 7 | 8 | import flask 9 | 10 | from es_sql import es_query 11 | 12 | app_dir = os.path.dirname(__file__) 13 | app = flask.Flask(__name__, template_folder=app_dir) 14 | 15 | app.logger.addHandler(logging.StreamHandler()) 16 | app.logger.setLevel(logging.INFO) 17 | 18 | @app.route('/') 19 | def explorer(): 20 | return flask.render_template('index.html') 21 | 22 | 23 | @app.route('/static/') 24 | def send_res(path): 25 | return flask.send_from_directory(os.path.join(app_dir, 'static'), path) 26 | 27 | 28 | @app.route('/translate', methods=['GET', 'POST']) 29 | def translate(): 30 | if flask.request.method == 'GET': 31 | sql = flask.request.args.get('q') 32 | else: 33 | sql = flask.request.get_data(parse_form_data=False) 34 | try: 35 | executor = es_query.create_executor(sql.split(';')) 36 | es_req = executor.request 37 | es_req['indices'] = executor.sql_select.from_indices 38 | resp = { 39 | 'error': None, 40 | 'data': es_req 41 | } 42 | return json.dumps(resp, indent=2) 43 | except: 44 | etype, value, tb = sys.exc_info() 45 | resp = { 46 | 'request_args': flask.request.args, 47 | 'request_body': flask.request.get_data(parse_form_data=False), 48 | 'traceback': traceback.format_exception(etype, value, tb), 49 | 'error': str(value), 50 | 'data': None 51 | } 52 | return json.dumps(resp, indent=2) 53 | 54 | 55 | @app.route('/search', methods=['GET', 'POST']) 56 | def search(): 57 | if flask.request.method == 'GET': 58 | sql = flask.request.args.get('q') 59 | else: 60 | sql = flask.request.get_data(parse_form_data=False) 61 | es_hosts = flask.request.args.get('elasticsearch') 62 | try: 63 | resp = { 64 | 'error': None, 65 | 'data': es_query.execute_sql(es_hosts, sql) 66 | } 67 | return json.dumps(resp, indent=2) 68 | except: 69 | etype, value, tb = sys.exc_info() 70 | resp = { 71 | 'request_args': flask.request.args, 72 | 'request_body': flask.request.get_data(parse_form_data=False), 73 | 'traceback': traceback.format_exception(etype, value, tb), 74 | 'error': str(value), 75 | 'data': None 76 | } 77 | return json.dumps(resp, indent=2) 78 | 79 | 80 | @app.route('/search_with_arguments', methods=['POST']) 81 | def search_with_arguments(): 82 | req = json.loads(flask.request.get_data(parse_form_data=False)) 83 | try: 84 | resp = { 85 | 'error': None, 86 | 'data': es_query.execute_sql(req['elasticsearch'], req['sql'], req['arguments']) 87 | } 88 | return json.dumps(resp, indent=2) 89 | except: 90 | etype, value, tb = sys.exc_info() 91 | resp = { 92 | 'request_args': flask.request.args, 93 | 'request_body': flask.request.get_data(parse_form_data=False), 94 | 'traceback': traceback.format_exception(etype, value, tb), 95 | 'error': str(value), 96 | 'data': None 97 | } 98 | return json.dumps(resp, indent=2) 99 | 100 | 101 | if __name__ == '__main__': 102 | app.run() 103 | -------------------------------------------------------------------------------- /explorer/index.html: -------------------------------------------------------------------------------- 1 | ../explorer-ui/dist/index.html -------------------------------------------------------------------------------- /explorer/requirement.txt: -------------------------------------------------------------------------------- 1 | gevent 2 | flask 3 | gunicorn -------------------------------------------------------------------------------- /explorer/static: -------------------------------------------------------------------------------- 1 | ../explorer-ui/dist/static -------------------------------------------------------------------------------- /plugin.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | cd $(dirname $0 ) 3 | python es_monitor.py "$@" 4 | -------------------------------------------------------------------------------- /sample/import-githubarchive.py: -------------------------------------------------------------------------------- 1 | import gzip 2 | import urllib2 3 | import json 4 | 5 | 6 | def main(): 7 | create_index_template() 8 | for j in range(1, 32): 9 | for i in range(24): 10 | filename = 'githubarchive/2015-01/2015-01-%02d-%s.json.gz' % (j, i) 11 | print('import %s' % filename) 12 | gzipFile = gzip.GzipFile(filename) 13 | events = gzipFile.readlines() 14 | bulk_import_lines = [] 15 | index_name = 'githubarchive-%s' % '2015-01-%02d' % j 16 | for event in events: 17 | eventObj = json.loads(event) 18 | bulk_import_lines.append(json.dumps( 19 | {'index': {'_index': index_name, '_type': eventObj['type'], '_id': eventObj['id']}})) 20 | bulk_import_lines.append(event) 21 | request = urllib2.Request('http://localhost:9200/_bulk', data='\n'.join(bulk_import_lines)) 22 | urllib2.urlopen(request).read() 23 | 24 | 25 | def create_index_template(): 26 | mappings = {} 27 | for type in ['PushEvent', 'CreateEvent', 'DeleteEvent', 'ForkEvent', 'GollumEvent', 'IssueCommentEvent', 28 | 'IssuesEvent', 'MemberEvent', 'PullRequestEvent', 'WatchEvent', 'CommitCommentEvent', 'PublicEvent', 29 | 'PullRequestReviewCommentEvent', 'ReleaseEvent']: 30 | mappings[type] = { 31 | '_all': {'enabled': False}, 32 | '_source': {'enabled': False}, 33 | 'dynamic': False, 34 | 'properties': { 35 | 'created_at': {'type': 'date', 'index': 'not_analyzed'}, 36 | 'type': {'type': 'string', 'index': 'not_analyzed'}, 37 | 'repo': { 38 | 'properties': { 39 | 'name': {'type': 'string', 'index': 'not_analyzed'} 40 | } 41 | }, 42 | 'actor': { 43 | 'properties': { 44 | 'login': {'type': 'string', 'index': 'not_analyzed'} 45 | } 46 | }, 47 | 'org': { 48 | 'properties': { 49 | 'login': {'type': 'string', 'index': 'not_analyzed'} 50 | } 51 | } 52 | } 53 | } 54 | request = urllib2.Request('http://localhost:9200/_template/githubarchive', data=json.dumps({ 55 | 'template': 'githubarchive-*', 56 | 'settings': { 57 | 'number_of_shards': 1, 58 | 'number_of_replicas': 0, 59 | 'index.codec': 'best_compression' 60 | }, 61 | 'mappings': mappings 62 | })) 63 | request.get_method = lambda: 'PUT' 64 | response = urllib2.urlopen(request).read() 65 | print(response) 66 | 67 | 68 | main() 69 | -------------------------------------------------------------------------------- /sample/import-quote.py: -------------------------------------------------------------------------------- 1 | import urllib2 2 | import json 3 | import sys 4 | import os 5 | import zipfile 6 | import csv 7 | import StringIO 8 | import contextlib 9 | 10 | 11 | def main(): 12 | create_index_template() 13 | quote_csvs = read_quote_csvs() 14 | delete_index('quote') 15 | create_index('quote') 16 | for symbol, quote_csv in quote_csvs: 17 | quotes = read_quotes(quote_csv, symbol) 18 | lines = list(bluk_import_lines('quote', quotes)) 19 | to_import_count = len(lines) / 2 20 | imported_count = read_index_documents_count(symbol) 21 | if imported_count >= to_import_count - 5: 22 | print('skip %s' % symbol) 23 | continue 24 | print('import %s' % symbol) 25 | request = urllib2.Request('http://localhost:9200/_bulk', data='\n'.join(lines)) 26 | response = urllib2.urlopen(request).read() 27 | 28 | 29 | def read_index_documents_count(symbol): 30 | try: 31 | return json.loads(urllib2.urlopen('http://127.0.0.1:9200/quote/_count?q=symbol:%s' % symbol).read())['count'] 32 | except: 33 | return 0 34 | 35 | 36 | def create_index_template(): 37 | request = urllib2.Request('http://localhost:9200/_template/quote', data=json.dumps({ 38 | 'template': 'quote', 39 | 'settings': { 40 | 'number_of_shards': 1, 41 | 'number_of_replicas': 0, 42 | 'index.codec': 'best_compression' 43 | }, 44 | 'mappings': { 45 | 'quote': { 46 | '_all': {'enabled': False}, 47 | '_source': {'enabled': True}, 48 | 'properties': { 49 | 'symbol': {'type': 'string', 'index': 'not_analyzed'}, 50 | 'adj_close': {'type': 'long', 'index': 'not_analyzed'}, 51 | 'close': {'type': 'long', 'index': 'not_analyzed'}, 52 | 'open': {'type': 'long', 'index': 'not_analyzed'}, 53 | 'high': {'type': 'long', 'index': 'not_analyzed'}, 54 | 'low': {'type': 'long', 'index': 'not_analyzed'}, 55 | 'volume': {'type': 'long', 'index': 'not_analyzed'}, 56 | 'date': {'type': 'date', 'index': 'not_analyzed'} 57 | } 58 | } 59 | } 60 | })) 61 | request.get_method = lambda: 'PUT' 62 | response = urllib2.urlopen(request).read() 63 | print(response) 64 | 65 | 66 | def read_quote_csvs(): 67 | with zipfile.ZipFile('quote.zip') as quote_zip: 68 | for file in quote_zip.namelist(): 69 | if file.endswith('.csv'): 70 | symbol = os.path.basename(file).replace('.csv', '') 71 | yield symbol, quote_zip.read(file) 72 | 73 | 74 | def read_quotes(quote_csv, symbol): 75 | with contextlib.closing(StringIO.StringIO(quote_csv)) as f: 76 | quotes = csv.DictReader(f, fieldnames=['date', 'open', 'high', 'low', 'close', 'volume', 'adj_close']) 77 | quotes.next() 78 | for quote in quotes: 79 | try: 80 | quote['symbol'] = symbol 81 | quote['open'] = long(float(quote['open']) * 100) 82 | quote['high'] = long(float(quote['high']) * 100) 83 | quote['low'] = long(float(quote['low']) * 100) 84 | quote['close'] = long(float(quote['close']) * 100) 85 | quote['adj_close'] = long(float(quote['adj_close']) * 100) 86 | quote['volume'] = long(quote['volume']) 87 | yield quote 88 | except: 89 | pass 90 | 91 | 92 | def bluk_import_lines(index_name, quotes): 93 | for quote in quotes: 94 | yield json.dumps( 95 | {'index': {'_index': index_name, '_type': 'quote', '_id': '%s-%s' % (quote['symbol'], quote['date'])}}) 96 | yield json.dumps(quote) 97 | 98 | 99 | def delete_index(index_name): 100 | try: 101 | request = urllib2.Request('http://localhost:9200/%s/' % index_name) 102 | request.get_method = lambda: 'DELETE' 103 | response = urllib2.urlopen(request).read() 104 | except: 105 | pass 106 | 107 | 108 | def create_index(index_name): 109 | request = urllib2.Request('http://localhost:9200/%s/' % index_name) 110 | request.get_method = lambda: 'PUT' 111 | try: 112 | response = urllib2.urlopen(request).read() 113 | except urllib2.HTTPError as e: 114 | print(e.read()) 115 | sys.exit(1) 116 | 117 | 118 | main() 119 | -------------------------------------------------------------------------------- /sample/import-symbol.py: -------------------------------------------------------------------------------- 1 | import urllib2 2 | import json 3 | import sys 4 | import csv 5 | import os 6 | import zipfile 7 | import StringIO 8 | 9 | 10 | def main(): 11 | create_index_template() 12 | delete_index() 13 | create_index() 14 | symbols = list(read_symbols()) 15 | request = urllib2.Request('http://localhost:9200/_bulk', data='\n'.join(bluk_import_lines(symbols))) 16 | response = urllib2.urlopen(request).read() 17 | print(response) 18 | # download_quote(symbols) 19 | 20 | 21 | def download_quote(symbols): 22 | for symbol in symbols: 23 | symbol = symbol['symbol'] 24 | if os.path.exists('quote/%s.csv' % symbol): 25 | continue 26 | print('download quotes of %s ...' % symbol) 27 | try: 28 | response = urllib2.urlopen( 29 | 'http://ichart.finance.yahoo.com/table.csv?s=%s&a=03&b=07&c=1981&d=09&e=04&f=2016' % symbol).read() 30 | with open('quote/%s.csv' % symbol, 'w') as f: 31 | f.write(response) 32 | except urllib2.HTTPError as e: 33 | print(e.code, e.read()) 34 | 35 | 36 | def bluk_import_lines(symbols): 37 | for symbol in symbols: 38 | yield json.dumps({'index': {'_index': 'symbol', '_type': 'symbol'}}) 39 | yield json.dumps(symbol) 40 | 41 | 42 | def read_symbols(): 43 | for exchange in ['nasdaq', 'nyse', 'nyse mkt']: 44 | with open('symbol/%s.csv' % exchange) as f: 45 | f.readline() 46 | if 'nasdaq' == exchange: 47 | field_names = ['symbol', 'name', 'last_sale', 'market_cap', 'ipo_year', 'sector', 'industry'] 48 | else: 49 | field_names = ['symbol', 'name', 'last_sale', 'market_cap', None, 'ipo_year', 'sector', 'industry'] 50 | for symbol in csv.DictReader(f, fieldnames=field_names): 51 | symbol.pop(None, None) 52 | symbol['exchange'] = exchange 53 | if 'n/a' == symbol['ipo_year']: 54 | symbol['ipo_year'] = None 55 | else: 56 | symbol['ipo_year'] = int(symbol['ipo_year']) 57 | if 'n/a' == symbol['last_sale']: 58 | symbol['last_sale'] = None 59 | else: 60 | symbol['last_sale'] = int(float(symbol['last_sale']) * 100) 61 | if 'n/a' == symbol['market_cap']: 62 | symbol['market_cap'] = None 63 | elif '0' == symbol['market_cap']: 64 | symbol['market_cap'] = 0 65 | else: 66 | try: 67 | symbol['market_cap'] = long(float(symbol['market_cap'][1:])) 68 | except: 69 | unit = symbol['market_cap'][-1] 70 | if 'M' == unit: 71 | symbol['market_cap'] = long(float(symbol['market_cap'][1:-1]) * 1000L * 1000L) 72 | elif 'B' == unit: 73 | symbol['market_cap'] = long(float(symbol['market_cap'][1:-1]) * 1000L * 1000L * 1000L) 74 | else: 75 | raise Exception('unexpected unit: %s' % symbol['market_cap']) 76 | yield symbol 77 | 78 | 79 | def delete_index(): 80 | try: 81 | request = urllib2.Request('http://localhost:9200/symbol/') 82 | request.get_method = lambda: 'DELETE' 83 | response = urllib2.urlopen(request).read() 84 | print(response) 85 | except: 86 | pass 87 | 88 | 89 | def create_index(): 90 | request = urllib2.Request('http://localhost:9200/symbol/') 91 | request.get_method = lambda: 'PUT' 92 | try: 93 | response = urllib2.urlopen(request).read() 94 | print(response) 95 | except urllib2.HTTPError as e: 96 | print(e.read()) 97 | sys.exit(1) 98 | 99 | 100 | def create_index_template(): 101 | request = urllib2.Request('http://localhost:9200/_template/symbol', data=json.dumps({ 102 | 'template': 'symbol', 103 | 'settings': { 104 | 'number_of_shards': 3, 105 | 'number_of_replicas': 0 106 | }, 107 | 'mappings': { 108 | 'symbol': { 109 | '_source': {'enabled': True}, 110 | 'properties': { 111 | 'symbol': {'type': 'string', 'index': 'not_analyzed'}, 112 | 'name': {'type': 'string', 'index': 'analyzed'}, 113 | 'last_sale': {'type': 'long', 'index': 'not_analyzed'}, 114 | 'market_cap': {'type': 'long', 'index': 'not_analyzed'}, 115 | 'ipo_year': {'type': 'integer', 'index': 'not_analyzed'}, 116 | 'sector': {'type': 'string', 'index': 'not_analyzed'}, 117 | 'industry': {'type': 'string', 'index': 'not_analyzed'}, 118 | 'exchange': {'type': 'string', 'index': 'not_analyzed'} 119 | } 120 | } 121 | } 122 | })) 123 | request.get_method = lambda: 'PUT' 124 | response = urllib2.urlopen(request).read() 125 | print(response) 126 | 127 | 128 | main() 129 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | from os.path import join, dirname 3 | from setuptools import setup, find_packages 4 | import sys 5 | import os 6 | 7 | VERSION = (2, 0, 0) 8 | __version__ = VERSION 9 | __versionstr__ = '.'.join(map(str, VERSION)) 10 | 11 | f = open(join(dirname(__file__), 'es_sql', 'README.md')) 12 | long_description = f.read().strip() 13 | f.close() 14 | 15 | install_requires = [ 16 | ] 17 | tests_require = [ 18 | ] 19 | 20 | # use external unittest for 2.6 21 | if sys.version_info[:2] == (2, 6): 22 | install_requires.append('unittest2') 23 | 24 | setup( 25 | name = 'es-sql', 26 | description = "Use sql to query from Elasticsearch", 27 | license="Apache License, Version 2.0", 28 | url = "https://github.com/taowen/es-monitor", 29 | long_description = long_description, 30 | version = __versionstr__, 31 | author = "Tao Wen", 32 | author_email = "taowen@gmail.com", 33 | packages=find_packages( 34 | where='.', 35 | include=('es_sql*', ) 36 | ), 37 | keywords="sql elasticsearch es", 38 | classifiers = [ 39 | "Development Status :: 4 - Beta", 40 | "License :: OSI Approved :: Apache Software License", 41 | "Intended Audience :: Developers", 42 | "Operating System :: OS Independent", 43 | "Programming Language :: Python", 44 | "Programming Language :: Python :: 2", 45 | "Programming Language :: Python :: 2.6", 46 | "Programming Language :: Python :: 2.7", 47 | "Programming Language :: Python :: Implementation :: CPython", 48 | "Programming Language :: Python :: Implementation :: PyPy", 49 | ], 50 | install_requires=install_requires, 51 | entry_points={ 52 | 'console_scripts': [ 53 | 'es-sql = es_sql.__main__:main' 54 | ] 55 | }, 56 | test_suite='es_sql.run_tests.run_all', 57 | tests_require=tests_require, 58 | ) --------------------------------------------------------------------------------