├── .gitignore ├── README.md ├── elseql ├── __init__.py ├── elseql.py ├── parser.py ├── search.py └── version.py ├── setup.py └── win32 ├── README.md ├── elseql.spec └── elseql_run.py /.gitignore: -------------------------------------------------------------------------------- 1 | *.py[co] 2 | 3 | # Packages 4 | *.egg 5 | *.egg-info 6 | dist 7 | build 8 | eggs 9 | parts 10 | bin 11 | var 12 | sdist 13 | develop-eggs 14 | .installed.cfg 15 | 16 | # Installer logs 17 | pip-log.txt 18 | 19 | # Unit test / coverage reports 20 | .coverage 21 | .tox 22 | 23 | #Translations 24 | *.mo 25 | 26 | #Mr Developer 27 | .mr.developer.cfg 28 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | elseql 2 | ====== 3 | You know, for Query 4 | ------------------- 5 | A SQL-like command line / REPL client for ElasticSearch 6 | 7 | ### USAGE 8 | 9 | elseql [--debug] [--port=host:port] 10 | 11 | ### COMMANDS 12 | 13 | * select - see SEARCH SYNTAX 14 | * describe [index] 15 | * set options [on|off] 16 | * help 17 | 18 | ### SEARCH SYNTAX 19 | 20 | SELECT {fields} 21 | [FACETS facet-fields] 22 | [SCRIPT script-field = 'script'] 23 | FROM index 24 | [WHERE where-condition] 25 | [FILTER filter-condition] 26 | [ORDERY BY order-fields] 27 | [LIMIT [start,] count] 28 | 29 | where: 30 | fields: '*' or comma-separated list of field names to be returned 31 | 32 | facet-fields: comma-separated list of fields to execute a facet query on 33 | 34 | script-field: name of script field, to be used in select clause 35 | script: ElasticSearch script 36 | 37 | index: index to query 38 | 39 | where-condition: 40 | {field-name} [ = != > >= < <= ] {value} 41 | {field-name} LIKE {value} 42 | {field-name} IN (value1, value2, ...) 43 | {field-name} BETWEEN {min-value} AND {max-value} 44 | NOT {where-condition} 45 | {where-condition} AND {where-condition} 46 | {where-condition} OR {where-condition} 47 | 48 | or where-condition: 49 | 'query in Lucene syntax' 50 | 51 | filter-condition: 52 | QUERY {where-condition} - query filter, same syntax as where condition 53 | EXIST {field-name} - exists field filter 54 | MISSING {field.name} - missing field filter 55 | 56 | order-fields: comma-separated list of {field-name} [ASC | DESC] 57 | 58 | start: start index for pagination 59 | count: maximum number of returned results 60 | 61 | A special case for LIMIT start,count allows to do a "scroll" query (i.e. results will be returned in batches): 62 | 63 | start: -1 - enable "scroll" query 64 | count: batch size - the query will return {count} results (actually {count} per shard) and will be repeated until all results are returned. 65 | 66 | This is very useful when you are expecting large result sets (or you are doing a full table scan). Note that in 67 | "scroll" mode sort and facets are disabled. 68 | 69 | ### INSTALLATION 70 | 71 | From pypi: 72 | 73 | sudo easy_install elseql 74 | or: 75 | 76 | sudo pip install elseql 77 | 78 | With python and setuptools installed: 79 | 80 | sudo python setup.py install 81 | 82 | You can also run the command without installing as: 83 | 84 | python -m elseql.elseql 85 | 86 | To do this you will need the pyparsing, rawes and cmd2 packages installed, that are automatically installed in the previous step. 87 | 88 | sudo easy_install pyparsing 89 | sudo easy_install rawes 90 | sudo easy_install cmd2 91 | 92 | The cmd2 package add a few extra features "command-line" related features. The most useful is redirection: 93 | 94 | elsesql> select id,field1,field2 from index where condition > result.csv 95 | 96 | Note that because '>' is used for redirection you'll need to use GT in the where clause insted (also available LT, GTE, LTE) 97 | 98 | ### SEE ALSO 99 | 100 | - http://elasticsearch.org/, You know, for Search 101 | - https://github.com/raff/elseql-go, a Go implementation 102 | -------------------------------------------------------------------------------- /elseql/__init__.py: -------------------------------------------------------------------------------- 1 | from version import __version__ # noqa: F401 2 | -------------------------------------------------------------------------------- /elseql/elseql.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # Copyright (c) 2012 Raffaele Sena https://github.com/raff 3 | # 4 | # Permission is hereby granted, free of charge, to any person obtaining a 5 | # copy of this software and associated documentation files (the 6 | # "Software"), to deal in the Software without restriction, including 7 | # without limitation the rights to use, copy, modify, merge, publish, dis- 8 | # tribute, sublicense, and/or sell copies of the Software, and to permit 9 | # persons to whom the Software is furnished to do so, subject to the fol- 10 | # lowing conditions: 11 | # 12 | # The above copyright notice and this permission notice shall be included 13 | # in all copies or substantial portions of the Software. 14 | # 15 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS 16 | # OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- 17 | # ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT 18 | # SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, 19 | # WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 21 | 22 | # 23 | # A SQL-like command line tool to query ElasticSearch 24 | # 25 | from __future__ import print_function 26 | 27 | import sys 28 | 29 | try: 30 | import readline 31 | assert readline 32 | except ImportError: 33 | try: 34 | import pyreadline as readline 35 | assert readline 36 | except ImportError: 37 | readline = None 38 | else: 39 | import rlcompleter 40 | assert rlcompleter 41 | 42 | if(sys.platform == 'darwin') and 'libedit' in readline.__doc__: 43 | readline.parse_and_bind("bind ^I rl_complete") 44 | else: 45 | readline.parse_and_bind("tab: complete") 46 | 47 | # readline.parse_and_bind("tab: complete") 48 | 49 | import os 50 | import os.path 51 | import shlex 52 | import traceback 53 | 54 | if False: 55 | from pprint import pprint 56 | else: 57 | import json 58 | 59 | def pprint(obj): 60 | print(json.dumps(obj, indent=2)) 61 | 62 | from cmd2 import Cmd 63 | from search import ElseSearch, DEFAULT_PORT 64 | from version import __version__ 65 | 66 | HISTORY_FILE = ".elseql_history" 67 | 68 | 69 | class DebugPrinter: 70 | def write(self, s): 71 | print(s) 72 | 73 | 74 | class ElseShell(Cmd): 75 | 76 | prompt = "elseql> " 77 | port = DEFAULT_PORT 78 | creds = None 79 | debug = False 80 | query = False 81 | 82 | def __init__(self, port, debug): 83 | Cmd.__init__(self) 84 | 85 | self.settable.update({ 86 | "prompt": "Set command prompt", 87 | "port": "Set service [host:]port", 88 | "creds": "Set credentials (user:password)", 89 | "debug": "Set debug mode", 90 | "query": "Display query before results" 91 | }) 92 | 93 | if readline: 94 | path = os.path.join(os.environ.get('HOME', ''), HISTORY_FILE) 95 | self.history_file = os.path.abspath(path) 96 | else: 97 | self.history_file = None 98 | 99 | self.debug = debug 100 | self.port = port 101 | self.init_search() 102 | 103 | def init_search(self): 104 | self.search = ElseSearch(self.port, self.debug) 105 | 106 | if self.search.host: 107 | print("connected to", self.search.host) 108 | else: 109 | print("not connected") 110 | 111 | def _onchange_port(self, old=None, new=None): 112 | self.port = new 113 | self.init_search() 114 | 115 | def _onchange_creds(self, old=None, new=None): 116 | print("change creds to", new) 117 | 118 | self.creds = new.split(":", 1) 119 | print("creds:", self.creds) 120 | self.init_search() 121 | 122 | def _onchange_debug(self, old=None, new=None): 123 | self.debug = new 124 | self.search.debug = self.debug 125 | 126 | def _onchange_query(self, old=None, new=None): 127 | self.query = new 128 | self.search.print_query = self.query 129 | 130 | def getargs(self, line): 131 | return shlex.split(str(line.decode('string-escape'))) 132 | 133 | def get_boolean(self, arg): 134 | return arg and [v for v in ['t', 'y', 'on', '1'] if arg.startswith(v)] != [] 135 | 136 | def do_version(self, line): 137 | print() 138 | print("elseql %s - you know, for query" % __version__) 139 | print("es version:", self.search.get_version()) 140 | print() 141 | 142 | def do_keywords(self, line): 143 | print(self.search.get_keywords()) 144 | 145 | def do_mapping(self, line): 146 | "mapping [index-name]" 147 | mapping = self.search.get_mapping() 148 | 149 | if line == "--list": 150 | for k in mapping: 151 | print(k) 152 | elif line: 153 | pprint(mapping[line]) 154 | else: 155 | pprint(mapping) 156 | 157 | def do_select(self, line): 158 | self.search.search('select ' + line) 159 | 160 | def do_explain(self, line): 161 | self.search.search(line, explain=True) 162 | 163 | def do_validate(self, line): 164 | self.search.search(line, validate=True) 165 | 166 | def do_EOF(self, line): 167 | "Exit shell" 168 | return True 169 | 170 | def do_shell(self, line): 171 | "Shell" 172 | os.system(line) 173 | 174 | # 175 | # aliases 176 | # 177 | do_describe = do_mapping 178 | 179 | # 180 | # override cmd 181 | # 182 | 183 | def emptyline(self): 184 | pass 185 | 186 | def onecmd(self, s): 187 | try: 188 | return Cmd.onecmd(self, s) 189 | except NotImplementedError as e: 190 | print(e.message) 191 | except: 192 | traceback.print_exc() 193 | return False 194 | 195 | def default(self, line): 196 | line = line.strip() 197 | if line and line[0] in ['#', ';']: 198 | return False 199 | else: 200 | return Cmd.default(self, line) 201 | 202 | def completedefault(self, test, line, beginidx, endidx): 203 | list = [] 204 | 205 | for k in self.search.get_keywords(): 206 | if k.startswith(test): 207 | list.append(k) 208 | 209 | return list 210 | 211 | def preloop(self): 212 | if self.history_file and os.path.exists(self.history_file): 213 | try: 214 | readline.read_history_file(self.history_file) 215 | except: 216 | print("can't read history file") 217 | 218 | def postloop(self): 219 | if self.history_file: 220 | readline.set_history_length(100) 221 | readline.write_history_file(self.history_file) 222 | 223 | print("Goodbye!") 224 | 225 | 226 | def run_command(): 227 | import sys 228 | 229 | args = sys.argv 230 | args.pop(0) # drop progname 231 | debug = False 232 | 233 | port = DEFAULT_PORT 234 | 235 | while args: 236 | if args[0][0] == '-': 237 | arg = args.pop(0) 238 | 239 | if arg.startswith('--port=') or arg.startswith('--host='): 240 | port = arg[7:] 241 | 242 | elif arg == '--debug': 243 | debug = True 244 | 245 | elif arg == '--': 246 | break 247 | 248 | else: 249 | print("invalid argument ", arg) 250 | return 1 251 | else: 252 | break 253 | 254 | ElseShell(port, debug).cmdloop() 255 | 256 | 257 | if __name__ == "__main__": 258 | run_command() 259 | -------------------------------------------------------------------------------- /elseql/parser.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | from __future__ import print_function 4 | from pyparsing import (Optional, Word, delimitedList, Group, quotedString, nums, 5 | Empty, removeQuotes, Combine, Suppress, CaselessLiteral, oneOf, 6 | operatorPrecedence, opAssoc, alphas, alphanums, CaselessKeyword, 7 | Forward, ParseBaseException, ParseException, ParseFatalException) 8 | 9 | 10 | class Operator(object): 11 | name = '' 12 | 13 | def __repr__(self): 14 | return "(%s %s)" % (self.name, self.operands) 15 | 16 | def __init__(self, operands): 17 | self.operands = operands 18 | 19 | def op(self, i): 20 | return self.val(self.operands[i]) 21 | 22 | def val(self, x): 23 | if isinstance(x, basestring): 24 | return x # escape Lucene characters? 25 | elif isinstance(x, bool): 26 | return "true" if x else "false" 27 | else: 28 | return str(x) 29 | 30 | 31 | class BinaryOperator(Operator): 32 | def __init__(self, operands): 33 | self.name = operands[1] 34 | self.operands = [operands[0], operands[2]] 35 | 36 | def __str__(self): 37 | if self.name == '=': 38 | return "%s:%s" % (self.operands[0], self.op(1)) 39 | elif self.name == '!=': 40 | return "NOT (%s:%s)" % (self.operands[0], self.op(1)) 41 | elif self.name in ['<=', 'LTE', 'LE']: 42 | return "%s:[* TO %s]" % (self.operands[0], self.op(1)) 43 | elif self.name in ['>=', 'GTE', 'GE']: 44 | return "%s:[%s TO *]" % (self.operands[0], self.op(1)) 45 | elif self.name in ['<', 'LT']: 46 | return "%s:{* TO %s}" % (self.operands[0], self.op(1)) 47 | elif self.name in ['>', 'GT']: 48 | return "%s:{%s TO *}" % (self.operands[0], self.op(1)) 49 | else: 50 | return "%s %s %s" % (self.operands[0], self.name, self.op(1)) 51 | 52 | 53 | class LikeOperator(Operator): 54 | name = 'LIKE' 55 | 56 | def __str__(self): 57 | return "%s:%s" % (self.operands[0], self.operands[1].replace('*', '\*').replace('%', '*')) 58 | 59 | 60 | class BetweenOperator(Operator): 61 | name = 'BETWEEN' 62 | 63 | def __str__(self): 64 | return "%s:[%s TO %s]" % (self.operands[0], self.op(1), self.op(2)) 65 | 66 | 67 | class InOperator(Operator): 68 | name = 'IN' 69 | 70 | def __init__(self, operands): 71 | self.operands = [operands[0], operands[1:]] 72 | 73 | def __str__(self): 74 | return "%s:(%s)" % (self.operands[0], ' OR '.join([self.val(x) for x in self.operands[1]])) 75 | 76 | 77 | class AndOperator(Operator): 78 | def __init__(self, operands=None): 79 | self.name = 'AND' 80 | self.operands = [x for x in operands[0] if not isinstance(x, basestring)] 81 | 82 | def __str__(self): 83 | return ' AND '.join([self.val(x) for x in self.operands]) 84 | 85 | 86 | class OrOperator(Operator): 87 | def __init__(self, operands=None): 88 | self.name = 'OR' 89 | self.operands = [x for x in operands[0] if not isinstance(x, basestring)] 90 | 91 | def __str__(self): 92 | return ' OR '.join([self.val(x) for x in self.operands]) 93 | 94 | 95 | class NotOperator(Operator): 96 | def __init__(self, operands=None): 97 | self.name = 'NOT' 98 | self.operands = [operands[0][1]] 99 | 100 | def __str__(self): 101 | return "NOT %s" % self.operands[0] 102 | 103 | 104 | class QueryFilter(Operator): 105 | def __init__(self, operands=None): 106 | self.name = "query" 107 | self.operands = [operands[0]] 108 | 109 | def __str__(self): 110 | return self.operands[0] 111 | 112 | 113 | class ExistFilter(Operator): 114 | def __init__(self, operands=None): 115 | self.name = "exists" 116 | self.operands = [operands[0]] 117 | 118 | def __str__(self): 119 | return self.operands[0] 120 | 121 | 122 | class MissingFilter(Operator): 123 | def __init__(self, operands=None): 124 | self.name = "missing" 125 | self.operands = [operands[0]] 126 | 127 | def __str__(self): 128 | return self.operands[0] 129 | 130 | 131 | def makeGroupObject(cls): 132 | def groupAction(s, loc, tokens): 133 | # print("GROUPACTION %s" % tokens) 134 | return cls(tokens) 135 | return groupAction 136 | 137 | 138 | def invalidSyntax(s, loc, token): 139 | raise ParseFatalException(s, loc, "Invalid Syntax") 140 | 141 | 142 | def intValue(t): 143 | return int(t) 144 | 145 | 146 | def floatValue(t): 147 | return float(t) 148 | 149 | 150 | def boolValue(t): 151 | return t.lower() == 'true' 152 | 153 | 154 | def makeAtomObject(fn): 155 | def atomAction(s, loc, tokens): 156 | try: 157 | return fn(tokens[0]) 158 | except: 159 | return fn(tokens) 160 | return atomAction 161 | 162 | 163 | class ElseParserException(ParseBaseException): 164 | pass 165 | 166 | 167 | class ElseParser(object): 168 | # define SQL tokens 169 | selectStmt = Forward() 170 | selectToken = CaselessKeyword("SELECT") 171 | facetToken = CaselessKeyword("FACETS") 172 | scriptToken = CaselessKeyword("SCRIPT") 173 | fromToken = CaselessKeyword("FROM") 174 | whereToken = CaselessKeyword("WHERE") 175 | orderbyToken = CaselessKeyword("ORDER BY") 176 | limitToken = CaselessKeyword("LIMIT") 177 | between = CaselessKeyword("BETWEEN") 178 | likeop = CaselessKeyword("LIKE") 179 | in_ = CaselessKeyword("IN") 180 | and_ = CaselessKeyword("AND") 181 | or_ = CaselessKeyword("OR") 182 | not_ = CaselessKeyword("NOT") 183 | 184 | filterToken = CaselessKeyword("FILTER") 185 | queryToken = CaselessKeyword("QUERY") 186 | existToken = CaselessKeyword("EXIST") 187 | missingToken = CaselessKeyword("MISSING") 188 | 189 | routingToken = CaselessKeyword("ROUTING") 190 | 191 | ident = Word(alphas + "_", alphanums + "_$").setName("identifier") 192 | columnName = delimitedList(ident, ".", combine=True) 193 | columnNameList = Group(delimitedList(columnName)) 194 | indexName = delimitedList(ident, ".", combine=True) 195 | 196 | # likeExpression for SQL LIKE expressions 197 | likeExpr = quotedString.setParseAction(removeQuotes) 198 | 199 | routingExpr = quotedString.setParseAction(removeQuotes) 200 | 201 | E = CaselessLiteral("E") 202 | binop = oneOf("= >= <= < > <> != LT LTE LE GT GTE GE", caseless=True) 203 | lpar = Suppress("(") 204 | rpar = Suppress(")") 205 | comma = Suppress(",") 206 | 207 | arithSign = Word("+-", exact=1) 208 | 209 | realNum = Combine( 210 | Optional(arithSign) + 211 | (Word(nums) + "." + Optional(Word(nums)) | ("." + Word(nums))) + 212 | Optional(E + Optional(arithSign) + Word(nums))).setParseAction(makeAtomObject(floatValue)) 213 | 214 | intNum = Combine(Optional(arithSign) + Word(nums) + 215 | Optional(E + Optional("+") + Word(nums))).setParseAction(makeAtomObject(intValue)) 216 | 217 | boolean = oneOf("true false", caseless=True).setParseAction(makeAtomObject(boolValue)) 218 | 219 | columnRval = realNum | intNum | boolean | quotedString.setParseAction(removeQuotes) 220 | 221 | whereCondition = (columnName + binop + columnRval).setParseAction(makeGroupObject(BinaryOperator)) \ 222 | | (columnName + in_.suppress() + lpar + delimitedList(columnRval) + rpar).setParseAction(makeGroupObject(InOperator)) \ 223 | | (columnName + between.suppress() + columnRval + and_.suppress() + columnRval).setParseAction(makeGroupObject(BetweenOperator)) \ 224 | | (columnName + likeop.suppress() + likeExpr).setParseAction(makeGroupObject(LikeOperator)) \ 225 | | Empty().setParseAction(invalidSyntax) 226 | 227 | boolOperand = whereCondition | boolean 228 | 229 | whereExpression = quotedString.setParseAction(removeQuotes) \ 230 | | operatorPrecedence(boolOperand, [ 231 | (not_, 1, opAssoc.RIGHT, NotOperator), 232 | (or_, 2, opAssoc.LEFT, OrOperator), 233 | (and_, 2, opAssoc.LEFT, AndOperator), 234 | ]) 235 | 236 | filterExpression = (queryToken.suppress() + whereExpression.setResultsName("query")).setParseAction(makeGroupObject(QueryFilter)) \ 237 | | (existToken.suppress() + columnName).setParseAction(makeGroupObject(ExistFilter)) \ 238 | | (missingToken.suppress() + columnName).setParseAction(makeGroupObject(MissingFilter)) 239 | 240 | orderseq = oneOf("asc desc", caseless=True) 241 | orderList = delimitedList( 242 | Group(columnName + Optional(orderseq, default="asc"))) 243 | 244 | limitoffset = intNum 245 | limitcount = intNum 246 | 247 | # selectExpr = ('count(*)' | columnNameList | '*') 248 | selectExpr = (columnNameList | '*') 249 | facetExpr = columnNameList 250 | scriptExpr = columnName + Suppress("=") + quotedString.setParseAction(removeQuotes) 251 | 252 | # define the grammar 253 | selectStmt << (selectToken + 254 | selectExpr.setResultsName("fields") + 255 | Optional(facetToken + facetExpr.setResultsName("facets")) + 256 | Optional(scriptToken + scriptExpr.setResultsName("script")) + 257 | fromToken + indexName.setResultsName("index") + 258 | Optional(whereToken + whereExpression.setResultsName("query")) + 259 | Optional(filterToken + filterExpression.setResultsName("filter")) + 260 | Optional(orderbyToken + orderList.setResultsName("order")) + 261 | Optional(limitToken + Group(Optional(limitoffset + comma) + limitcount).setResultsName("limit")) + 262 | Optional(routingToken + routingExpr.setResultsName("routing"))) 263 | 264 | grammar_parser = selectStmt 265 | 266 | @staticmethod 267 | def parse(stmt, debug=False): 268 | ElseParser.grammar_parser.setDebug(debug) 269 | 270 | try: 271 | return ElseParser.grammar_parser.parseString(stmt, parseAll=True) 272 | except (ParseException, ParseFatalException) as err: 273 | raise ElseParserException(err.pstr, err.loc, err.msg, err.parserElement) 274 | 275 | @staticmethod 276 | def test(stmt): 277 | print("STATEMENT: ", stmt) 278 | print() 279 | 280 | try: 281 | response = ElseParser.parse(stmt) 282 | print("index = ", response.index) 283 | print("fields = ", response.fields) 284 | print("query = ", response.query) 285 | print("script = ", response.script) 286 | print("filter = ", response.filter) 287 | print("order = ", response.order) 288 | print("limit = ", response.limit) 289 | print("facets = ", response.facets) 290 | print("routing = ", response.routing) 291 | 292 | except ElseParserException as err: 293 | print(err.pstr) 294 | print(" "*err.loc + "^\n" + err.msg) 295 | print("ERROR:", err) 296 | 297 | print() 298 | 299 | 300 | if __name__ == '__main__': 301 | import sys 302 | 303 | stmt = " ".join(sys.argv[1:]) 304 | ElseParser.test(stmt) 305 | -------------------------------------------------------------------------------- /elseql/search.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | from __future__ import print_function 4 | 5 | from requests.exceptions import ConnectionError 6 | 7 | from parser import ElseParser, ElseParserException 8 | import rawes 9 | import pprint 10 | 11 | try: # for Python 3 12 | from http.client import HTTPConnection 13 | except ImportError: 14 | from httplib import HTTPConnection 15 | 16 | DEFAULT_PORT = 'localhost:9200' 17 | 18 | 19 | def _csval(v): 20 | if not v: 21 | return '' 22 | 23 | if not isinstance(v, basestring): 24 | return str(v) 25 | 26 | if v.isalnum(): 27 | return v 28 | 29 | return '"%s"' % v.replace('"', '""') 30 | 31 | 32 | def _csvline(l): 33 | try: 34 | return ",".join([_csval(v).encode("utf-8") for v in l]) 35 | except UnicodeDecodeError: 36 | raise Exception("UnicodeDecodeError for %s" % l) 37 | 38 | 39 | class ElseSearch(object): 40 | 41 | def __init__(self, port=None, debug=False): 42 | self.debug = debug 43 | self.print_query = False 44 | 45 | self.es = None 46 | self.mapping = None 47 | self.keywords = None 48 | self.host = None 49 | self.version = None 50 | self.v5 = None 51 | 52 | if port: 53 | try: 54 | self.es = rawes.Elastic(port, headers={'content-type': 'application/json'}) 55 | self.get_mapping() 56 | self.get_version() 57 | except ConnectionError as err: 58 | print("init: cannot connect to", port) 59 | print(err) 60 | 61 | if not self.es: 62 | self.debug = True 63 | 64 | def get_version(self): 65 | if not self.version: 66 | try: 67 | info = self.es.get("") 68 | self.version = info["version"]["number"] 69 | self.v5 = self.version[:2] >= "5." 70 | except ConnectionError as err: 71 | print("mapping: cannot connect to", self.es.url) 72 | print(err) 73 | 74 | return self.version 75 | 76 | def get_mapping(self): 77 | if not self.mapping: 78 | try: 79 | self.mapping = self.es.get("_mapping") 80 | self.keywords = [] 81 | self.host = self.es.url 82 | except ConnectionError as err: 83 | print("mapping: cannot connect to", self.es.url) 84 | print(err) 85 | 86 | return self.mapping 87 | 88 | def get_keywords(self): 89 | if self.keywords: 90 | return self.keywords 91 | 92 | keywords = ['facets', 'filter', 'query', 'exist', 'missing', 'script', 93 | 'from', 'where', 'in', 'between', 'like', 'order by', 'limit', 'and', 'or', 'not'] 94 | 95 | if not self.mapping: 96 | return sorted(keywords) 97 | 98 | def add_properties(plist, doc): 99 | if 'properties' in doc: 100 | props = doc['properties'] 101 | 102 | for p in props: 103 | plist.append(p) # property name 104 | add_properties(plist, props[p]) 105 | 106 | keywords.extend(['_score', '_all']) 107 | 108 | for i in self.mapping: 109 | keywords.append(i) # index name 110 | 111 | index = self.mapping[i] 112 | 113 | for t in index: 114 | keywords.append(t) # document type 115 | 116 | document = index[t] 117 | 118 | if '_source' in document: 119 | source = document['_source'] 120 | if 'enabled' not in source or source['enabled']: 121 | keywords.append('_source') # _source is enabled by default 122 | 123 | add_properties(keywords, document) 124 | 125 | self.keywords = sorted(set(keywords)) 126 | return self.keywords 127 | 128 | def search(self, query, explain=False, validate=False): 129 | try: 130 | request = ElseParser.parse(query) 131 | except ElseParserException as err: 132 | print(err.pstr) 133 | print(" " * err.loc + "^\n") 134 | print("ERROR:", err) 135 | return 1 136 | 137 | params = {} 138 | data_fields = None 139 | 140 | if request.query: 141 | data = {'query': {'query_string': {'query': str(request.query), 'default_operator': 'AND'}}} 142 | else: 143 | data = {'query': {'match_all': {}}} 144 | 145 | if explain: 146 | data['explain'] = True 147 | 148 | if request.filter: 149 | filter = request.filter 150 | 151 | if filter.name == 'query': 152 | data['filter'] = {'query': {'query_string': {'query': str(filter), 'default_operator': 'AND'}}} 153 | else: 154 | data['filter'] = {filter.name: {'field': str(filter)}} 155 | 156 | if request.facets: 157 | # data['facets'] = {f: {"terms": {"field": f}} for f in request.facets} -- not in python 2.6 158 | data['facets'] = dict((f, {"terms": {"field": f}}) for f in request.facets) 159 | 160 | if request.script: 161 | data['script_fields'] = {request.script[0]: {"script": request.script[1]}} 162 | 163 | if request.fields: 164 | fields = request.fields 165 | fields_k = '_source' if self.v5 else 'fields' 166 | if len(fields) == 1: 167 | if fields[0] == '*': 168 | # all fields 169 | pass 170 | elif fields[0] == 'count(*)': 171 | # TODO: only get count 172 | pass 173 | else: 174 | data[fields_k] = [fields[0]] 175 | else: 176 | data[fields_k] = [x for x in fields] 177 | 178 | data_fields = data.get(fields_k) 179 | 180 | if request.order: 181 | data['sort'] = [{x[0]:x[1]} for x in request.order] 182 | 183 | if request.limit: 184 | qfrom = None 185 | qsize = None 186 | 187 | if len(request.limit) > 1: 188 | qfrom = request.limit.pop(0) 189 | 190 | qsize = request.limit[0] 191 | 192 | if qfrom is None: 193 | data['size'] = qsize 194 | 195 | elif qfrom >= 0: 196 | data['from'] = qfrom 197 | data['size'] = qsize 198 | 199 | else: 200 | # 201 | # limit -1, 1000 => scan request, 1000 items at a time 202 | # 203 | params.update({'search_type': 'scan', 'scroll': '10m', 'size': qsize}) 204 | 205 | if validate: 206 | command = '/_validate/query' 207 | params.update({'pretty': 'true', 'explain': 'true'}) 208 | 209 | # validate doesn't like "query" 210 | if 'query' in data: 211 | q = data.pop('query') 212 | data.update(q) 213 | 214 | # 215 | # this is actually {index}/{document-id}/_explain 216 | # 217 | # elif explain: 218 | # command = '/_explain' 219 | # params.update({'pretty': 'true'}) 220 | 221 | else: 222 | command = '/_search' 223 | 224 | if request.routing: 225 | command += '?routing=%s' % request.routing 226 | 227 | command_path = request.index.replace(".", "/") + command 228 | 229 | if self.debug: 230 | HTTPConnection.debuglevel = 1 231 | 232 | print() 233 | print("GET", command_path, params or '') 234 | print(" ", pprint.pformat(data)) 235 | params.update({'pretty': 'true'}) 236 | else: 237 | HTTPConnection.debuglevel = 0 238 | 239 | if self.print_query: 240 | print() 241 | print("; ", _csval(query)) 242 | print() 243 | 244 | total = None 245 | print_fields = True 246 | do_query = True 247 | 248 | while self.es and do_query: 249 | try: 250 | result = self.es.get(command_path, params=params, data=data) 251 | except ConnectionError as err: 252 | print("cannot connect to", self.es.url) 253 | print(err) 254 | return 255 | 256 | if self.debug: 257 | print() 258 | print("RESPONSE:", pprint.pformat(result)) 259 | print() 260 | 261 | if '_scroll_id' in result: 262 | # scan/scroll request 263 | params['scroll_id'] = result['_scroll_id'] 264 | 265 | if 'search_type' in params: 266 | params.pop('search_type') 267 | command_path = '_search/scroll' 268 | else: 269 | # done 270 | do_query = False 271 | 272 | if 'valid' in result: 273 | if 'explanations' in result: 274 | for e in result['explanations']: 275 | print() 276 | for k, v in e.iteritems(): 277 | print(k, ':', v) 278 | else: 279 | print("valid:", result['valid']) 280 | return 281 | 282 | if 'error' in result: 283 | print("ERROR:", result['error']) 284 | return 285 | 286 | if 'shards' in result and 'failures' in result['_shards']: 287 | failures = result['_shards']['failures'] 288 | for f in failures: 289 | print("ERROR:", f['reason']) 290 | return 291 | 292 | if 'hits' in result: 293 | hits = result['hits'] 294 | total = hits['total'] 295 | 296 | if data_fields and not self.v5: 297 | if print_fields: 298 | print_fields = False 299 | print(_csvline(data_fields)) 300 | 301 | for _ in hits['hits']: 302 | result_fields = _['fields'] if 'fields' in _ else {} 303 | print(_csvline([_.get(x) or result_fields.get(x) for x in data_fields])) 304 | else: 305 | if hits['hits']: 306 | if print_fields: 307 | print_fields = False 308 | print(_csvline(hits['hits'][0]['_source'].keys())) 309 | else: 310 | do_query = False 311 | 312 | for _ in hits['hits']: 313 | print(_csvline([_csval(x) for x in _['_source'].values()])) 314 | 315 | if 'facets' in result: 316 | for facet in result['facets']: 317 | print() 318 | print("%s,count" % _csval(facet)) 319 | 320 | for _ in result['facets'][facet]['terms']: 321 | t = _['term'] 322 | c = _['count'] 323 | print("%s,%s" % (_csval(t), c)) 324 | 325 | if do_query and self.debug: 326 | print() 327 | print("GET", command_path, params or '') 328 | print(" ", pprint.pformat(data)) 329 | 330 | if total is not None: 331 | print() 332 | print("total: ", total) 333 | -------------------------------------------------------------------------------- /elseql/version.py: -------------------------------------------------------------------------------- 1 | __version__ = '0.3.0' 2 | 3 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | from elseql import __version__ 5 | import sys 6 | 7 | try: 8 | from setuptools import setup 9 | except ImportError: 10 | from distutils.core import setup 11 | 12 | if sys.version_info <= (2, 5): 13 | error = "ERROR: elseql %s requires Python Version 2.6 or above...exiting." % __version__ 14 | print >> sys.stderr, error 15 | sys.exit(1) 16 | 17 | #elif sys.version_info < (3, 0): 18 | # pyparsing_version = "< 2.0.0" 19 | else: 20 | pyparsing_version = "" 21 | 22 | SETUP_OPTIONS = dict( 23 | name='elseql', 24 | version=__version__, 25 | description='SQL-like command line client for ElasticSearch', 26 | long_description=open("README.md").read(), 27 | author='Raffaele Sena', 28 | author_email='raff367@gmail.com', 29 | url='https://github.com/raff/elseql', 30 | license="MIT", 31 | platforms="Posix; MacOS X; Windows", 32 | classifiers=[ 33 | 'Development Status :: 4 - Beta', 34 | 'Environment :: Console', 35 | 'Intended Audience :: Developers', 36 | 'License :: OSI Approved :: MIT License', 37 | 'Operating System :: OS Independent', 38 | 'Topic :: Internet', 39 | 'Topic :: Utilities', 40 | 'Topic :: Database :: Front-Ends', 41 | 'Topic :: Internet :: WWW/HTTP :: Indexing/Search', 42 | 'Programming Language :: Python :: 2', 43 | 'Programming Language :: Python :: 2.6', 44 | 'Programming Language :: Python :: 2.7' 45 | ], 46 | 47 | packages=['elseql' 48 | ], 49 | 50 | data_files=[('.', ['./README.md']) 51 | ], 52 | 53 | install_requires=['pyparsing' + pyparsing_version, 54 | 'rawes', 55 | 'cmd2', 56 | ], 57 | 58 | entry_points=""" 59 | [console_scripts] 60 | elseql=elseql.elseql:run_command 61 | """ 62 | ) 63 | 64 | 65 | def do_setup(): 66 | setup(**SETUP_OPTIONS) 67 | 68 | if __name__ == '__main__': 69 | do_setup() 70 | -------------------------------------------------------------------------------- /win32/README.md: -------------------------------------------------------------------------------- 1 | Build Windows executable using pyinstaller 2 | ========================================== 3 | 4 | #### INSTALLATION 5 | * Install elseql package 6 | * Install pyinstaller from http://www.pyinstaller.org/ 7 | * Install pywin32 from http://sourceforge.net/projects/pywin32/files/ 8 | 9 | #### PACKAGING 10 | * run 'pyinstaller elseql.spec' 11 | 12 | #### RUN 13 | The folder dist/elseql contains all you need to run elseql on Windows without having python installed. Copy it to your destionation machine and run elseql.exe 14 | -------------------------------------------------------------------------------- /win32/elseql.spec: -------------------------------------------------------------------------------- 1 | # -*- mode: python -*- 2 | a = Analysis(['elseql_run.py'], 3 | hiddenimports=['rawes','pyparsing','cmd2'], 4 | hookspath=None) 5 | pyz = PYZ(a.pure) 6 | exe = EXE(pyz, 7 | a.scripts, 8 | exclude_binaries=1, 9 | name=os.path.join('build\\pyi.win32\\elseql', 'elseql.exe'), 10 | debug=False, 11 | strip=None, 12 | upx=True, 13 | console=True ) 14 | coll = COLLECT(exe, 15 | a.binaries, 16 | a.zipfiles, 17 | a.datas, 18 | strip=None, 19 | upx=True, 20 | name=os.path.join('dist', 'elseql')) 21 | -------------------------------------------------------------------------------- /win32/elseql_run.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # Copyright (c) 2012 Raffaele Sena https://github.com/raff 3 | # 4 | # Permission is hereby granted, free of charge, to any person obtaining a 5 | # copy of this software and associated documentation files (the 6 | # "Software"), to deal in the Software without restriction, including 7 | # without limitation the rights to use, copy, modify, merge, publish, dis- 8 | # tribute, sublicense, and/or sell copies of the Software, and to permit 9 | # persons to whom the Software is furnished to do so, subject to the fol- 10 | # lowing conditions: 11 | # 12 | # The above copyright notice and this permission notice shall be included 13 | # in all copies or substantial portions of the Software. 14 | # 15 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS 16 | # OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- 17 | # ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT 18 | # SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, 19 | # WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 21 | 22 | if __name__ == "__main__": 23 | from elseql import elseql 24 | elseql.run_command() 25 | --------------------------------------------------------------------------------