├── .gitignore ├── .tool-versions ├── .travis.yml ├── CONTRIBUTING.md ├── LICENSE.md ├── Makefile ├── NOTICE ├── README.md ├── __init__.py ├── requirements.txt ├── setup.py ├── sqlparse.iml └── sqlparse ├── __init__.py ├── _old └── sql_grammar_old.py ├── builders ├── __init__.py ├── base.py ├── mongo_builder.py └── sqlalchemy_builder.py ├── grammar.py ├── nodes.py ├── nodevisitor.py ├── test ├── __init__.py ├── base.py ├── mongo_builder_test.py ├── sqlalchemy_builder_test.py └── sqlparse_test.py └── visitors.py /.gitignore: -------------------------------------------------------------------------------- 1 | .DS_Store 2 | Thumbs.db 3 | *.log 4 | *.pyc 5 | /sqlparse.egg-info/ 6 | /.idea/workspace.xml 7 | /.idea/ 8 | ideas.txt 9 | /.venv/ 10 | *.iml 11 | -------------------------------------------------------------------------------- /.tool-versions: -------------------------------------------------------------------------------- 1 | python 3.7.2 2 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: python 2 | before_script: 3 | script: make && make test 4 | branches: 5 | only: 6 | - master 7 | notifications: 8 | recipients: 9 | - flushot@gmail.com 10 | email: 11 | on_success: change 12 | on_failure: always 13 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contributing 2 | 3 | No guidelines yet. This is an early project. 4 | 5 | Please create issues or send pull requests with anything you'd like to see added/fixed. -------------------------------------------------------------------------------- /LICENSE.md: -------------------------------------------------------------------------------- 1 | Copyright 2014 Chris Lyon 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | .PHONY: deps develop 2 | 3 | all: deps 4 | PYTHONPATH=.venv ; . .venv/bin/activate 5 | 6 | .venv: 7 | if [ ! -e ".venv/bin/activate_this.py" ] ; then virtualenv --clear .venv ; fi 8 | 9 | deps: .venv 10 | PYTHONPATH=.venv ; . .venv/bin/activate && .venv/bin/pip install -U -r requirements.txt 11 | 12 | develop: .venv 13 | . .venv/bin/activate && .venv/bin/python setup.py develop 14 | 15 | test: .venv 16 | . .venv/bin/activate && .venv/bin/python setup.py test $* 17 | 18 | clean: 19 | rm -rf .venv *.egg-info *.log build 20 | rm -f `find . -name \*.pyc -print0 | xargs -0` 21 | -------------------------------------------------------------------------------- /NOTICE: -------------------------------------------------------------------------------- 1 | sqlparse is an early draft, and shouldn't be used in production. -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # SqlParse 2 | 3 | Parses and transforms SQL queries. 4 | 5 | This project started is an experiement, and isn't in a stable state. 6 | Please don't use it in production code yet. 7 | 8 | ## What's supported already? 9 | 10 | * Parser 11 | * Grammar: SELECT ... FROM ... [ WHERE ... ] [ ORDER BY ... ] 12 | * Where clause expression tree 13 | * Source tables/models (in FROM) 14 | * Projections 15 | 16 | * Query builder 17 | * Transform SQL into Mongo queries 18 | * Transform SQL into SqlAlchemy queries 19 | 20 | ## Roadmap 21 | 22 | Here's a list of features that are currently in the plans for development: 23 | 24 | * Grammar 25 | * Aliases 26 | * Joins 27 | * Unions 28 | * Pivots 29 | * Functions 30 | * Bitwise operators 31 | * Math expressions 32 | * GROUP BY and HAVING 33 | * LIMIT and OFFSET 34 | * EXCEPT and INTERSECT 35 | 36 | * Parser 37 | * Null-safe equality 38 | * Type checking 39 | * Logical optimizations 40 | * Multiple tables/models 41 | 42 | * General 43 | * Better test coverage 44 | * Sphinx documentation 45 | 46 | ## Tests 47 | 48 | 49 | 50 | To run unit tests: 51 | 52 | `./setup.py test` or `make test` 53 | 54 | ## Examples 55 | 56 | Parsing SQL query into a pyparsing parse tree: 57 | 58 | >>> ast = sqlparse.parseString('select a from b where c = 1 and d = 2 or e = "f"') 59 | >>> print ast.asXML('query') 60 | 61 | 62 | a 63 | 64 | 65 | b
66 | 67 | (and (= c 1) (or (= d 2) (= e "f"))) 68 | 69 | >>> print ast.tables.asXML('tables') 70 | 71 | b
72 | 73 | 74 | Building a SqlAlchemy query object from a parsed SQL query: 75 | 76 | >>> builder = sqlparse.bulders.SqlAlchemyQueryBuilder(sa_session, globals()) 77 | >>> sqlalchemy_query = builder.parse_and_build(""" 78 | ... select * from User where 79 | ... not (last_name = 'Jacob' or 80 | ... (first_name != 'Chris' and last_name != 'Lyon')) and 81 | ... not is_active = 1 82 | ... """) 83 | >>> for result in sqlalchemy_query.all(): 84 | ... # do something 85 | 86 | Building a MongoDB query object from a parsed SQL query: 87 | 88 | >>> builder = sqlparse.builders.MongoQueryBuilder(pymongo_database) 89 | >>> mongo_query = builder.parse_and_build(""" 90 | ... select * from User where 91 | ... not (last_name = 'Jacob' or 92 | ... (first_name != 'Chris' and last_name != 'Lyon')) and 93 | ... not is_active = 1 94 | ... """) 95 | >>> print json.dumps(mongo_query, indent=4) 96 | { 97 | "$and": [ 98 | { 99 | "$nor": [ 100 | { 101 | "$or": [ 102 | { 103 | "last_name": "Jacob" 104 | }, 105 | { 106 | "$and": [ 107 | { 108 | "first_name": { 109 | "$ne": "Chris" 110 | } 111 | }, 112 | { 113 | "last_name": { 114 | "$ne": "Lyon" 115 | } 116 | } 117 | ] 118 | } 119 | ] 120 | } 121 | ] 122 | }, 123 | { 124 | "$nor": [ 125 | { 126 | "is_active": 1 127 | } 128 | ] 129 | } 130 | ] 131 | } 132 | 133 | ## Documentation 134 | 135 | No documentation exists yet, except for what you see in this README file. 136 | 137 | ## License 138 | 139 | Copyright 2014 Chris Lyon 140 | 141 | Licensed under the Apache License, Version 2.0 (the "License"); 142 | you may not use this file except in compliance with the License. 143 | You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 144 | 145 | Unless required by applicable law or agreed to in writing, software 146 | distributed under the License is distributed on an "AS IS" BASIS, 147 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 148 | See the License for the specific language governing permissions and 149 | limitations under the License. 150 | -------------------------------------------------------------------------------- /__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Flushot/sqlparse/58bf73c9f106c3834e4e1028e98b3c0af3a59ff7/__init__.py -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | pyparsing 2 | sqlalchemy 3 | pymongo 4 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | from setuptools import setup 3 | 4 | import sqlparse 5 | 6 | setup( 7 | name='sqlparse', 8 | version=sqlparse.__version__, 9 | 10 | author='Chris Lyon', 11 | author_email='flushot@gmail.com', 12 | 13 | description='SQL parser and query builder', 14 | long_description=open('README.md').read(), 15 | 16 | url='https://github.com/Flushot/sqlparse', 17 | 18 | license='Apache License 2.0', 19 | classifiers=[ 20 | 'Intended Audience :: Developers' 21 | 'Development Status :: 2 - Pre-Alpha', 22 | 'License :: OSI Approved :: Apache Software License', 23 | ], 24 | 25 | install_requires=[ 26 | 'pyparsing', 27 | 'sqlalchemy', 28 | 'pymongo' 29 | ], 30 | 31 | test_suite='sqlparse' 32 | ) 33 | -------------------------------------------------------------------------------- /sqlparse.iml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | -------------------------------------------------------------------------------- /sqlparse/__init__.py: -------------------------------------------------------------------------------- 1 | from . import builders, nodes, visitors, grammar 2 | 3 | 4 | __version__ = '0.2.0' 5 | 6 | __all__ = [ 7 | 'builders', 8 | 'nodes', 9 | 'visitors', 10 | 'parse_string' 11 | ] 12 | 13 | 14 | def parse_string(query_string): 15 | """ 16 | Parses :query_string: into an AST 17 | """ 18 | return grammar.sqlQuery.parseString(query_string) 19 | -------------------------------------------------------------------------------- /sqlparse/_old/sql_grammar_old.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # 3 | # Copyright 2014 Chris Lyon 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | # 17 | 18 | # Contains an old grammar that only worked with INSERT statements. 19 | # TODO: Move this into sql_grammar.py 20 | 21 | from pyparsing import Word, White, alphas, nums, alphanums, Literal, CaselessLiteral, \ 22 | Group, OneOrMore, ZeroOrMore, StringEnd, Forward, Optional, ParseException 23 | import json 24 | 25 | # Macros 26 | def delim(literal): 27 | return Literal(literal).suppress() 28 | def quoted(token): 29 | return ( apos + token + apos ) | ( quot + token + quot ) 30 | def escaped(token): 31 | return ( backtick + token + backtick ) | token 32 | def commaSeparated(token): 33 | return ( OneOrMore(token + Literal(',').suppress()) + token ) | token 34 | 35 | # Delimiter 36 | eol = delim(';') 37 | apos = delim("'") 38 | quot = delim('"') 39 | backtick = delim('`') 40 | lpar = delim('(') 41 | rpar = delim(')') 42 | 43 | relset = escaped(Word(alphas, alphanums)) 44 | 45 | # todo: include nums, alphas, strings, etc. 46 | integer = Word(nums).setParseAction(lambda s,l,t: int(t[0])) 47 | decimal = Regex(r'^[+-]?\d+(\.\d*)?$').setParseAction(lambda s,l,t: float(t[0])) 48 | number = decimal | integer 49 | string = quoted(Regex('.*')) 50 | 51 | atom = number | string 52 | 53 | assign = relset + '=' + ( relset | atom ) 54 | 55 | # select X from X where X group by X having X order by X limit X,X 56 | select = Forward() 57 | proj = relset | atom | Literal('*') 58 | projs = Group(commaSeparated(proj)).setResultsName('projection') 59 | whereClause = CaselessLiteral('where') # incomplete 60 | select << CaselessLiteral('select').setResultsName('statementName') + \ 61 | projs + CaselessLiteral('from') + relset.setResultsName('relsetSource') 62 | 63 | # insert into X values (x,x,x) 64 | # insert into X select (x,x,x) ... 65 | # insert delayed into X values (x,x,x) 66 | # insert into X (x,x,x) select (x,x,x) ... 67 | # insert into X (x,x,x) values (x,x,x) 68 | # insert into X (x,x,x) values (x,x,x), (x,x,x) 69 | # insert high_priority into X values (x,x,x) on duplicate key update x=x,x=x 70 | insertCols = lpar + commaSeparated(relset) + rpar 71 | insertVals = lpar + Group(commaSeparated(relset | atom)) + rpar 72 | insertLockOption = Optional(CaselessLiteral('low_priority') | CaselessLiteral('delayed') | CaselessLiteral('high_priority')).setResultsName('lockOption') 73 | insert = CaselessLiteral('insert').setResultsName('statementName') + \ 74 | insertLockOption + Optional(CaselessLiteral('ignore').setParseAction(lambda s,l,t: True).setResultsName('ignoreOption')) + \ 75 | CaselessLiteral('into').suppress() + relset.setResultsName('relsetTarget') + \ 76 | insertCols.setResultsName('insertColumns') + \ 77 | ( select | (CaselessLiteral('values') + \ 78 | Group((commaSeparated(insertVals) | insertVals)).setResultsName('insertValues') ) ) + \ 79 | Optional(CaselessLiteral('on duplicate key update')) # col_name=expr, ... 80 | 81 | statement = Optional(White().suppress()) + ( select | insert ) #| assign 82 | 83 | sql = OneOrMore(statement + eol) | ( statement + Optional(eol) ) 84 | 85 | if __name__ == '__main__': 86 | # 87 | # test cases 88 | # 89 | for query in [ 90 | "select * from table", 91 | 92 | "select a, b, 1 from table", 93 | 94 | "insert into table (x) values (y)", 95 | 96 | "insert into tableA (a, `b`, c) select d, `e`, 1 from tableB", 97 | 98 | """ 99 | insert 100 | into table(a, b,c) 101 | values 102 | (d,e, f) 103 | """, 104 | 105 | "insert into `table` (a, b) values ('c', 2)", 106 | 107 | "insert into table (a,b) values(1,2),(3, 4), (5, 6)", 108 | 109 | "insert low_priority into table (a,b) values (1,2);" 110 | 111 | """ 112 | insert into table (a, b) values ('c', 'd'); 113 | insert into `table`(e,f) values(g,h) 114 | """ 115 | ]: 116 | print query 117 | try: 118 | toks = sql.parseString(query) 119 | if toks.statementName == 'insert': 120 | records = [ dict(zip(toks.insertColumns, valTuple)) for valTuple in toks.insertValues ] 121 | print json.dumps(records, indent=4) 122 | else: 123 | print toks 124 | print 'PASS' 125 | except ParseException, err: 126 | print 'FAIL : Syntax error in: "%s"' % err.line 127 | print ' ' + '-'*(err.col-1) + '^ %s' % err # ---^ pointer 128 | print '='*80 129 | -------------------------------------------------------------------------------- /sqlparse/builders/__init__.py: -------------------------------------------------------------------------------- 1 | from .base import QueryBuilder 2 | from .sqlalchemy_builder import SqlAlchemyQueryBuilder 3 | from .mongo_builder import MongoQueryBuilder 4 | 5 | __all__ = [ 6 | 'QueryBuilder', 7 | 'SqlAlchemyQueryBuilder', 8 | 'MongoQueryBuilder' 9 | ] 10 | -------------------------------------------------------------------------------- /sqlparse/builders/base.py: -------------------------------------------------------------------------------- 1 | import logging 2 | from abc import ABCMeta, abstractmethod 3 | 4 | import sqlparse 5 | import pyparsing 6 | 7 | logger = logging.getLogger(__name__) 8 | 9 | 10 | class QueryBuilder(object): 11 | __metaclass__ = ABCMeta 12 | _primitives = (int, float, str, bool) 13 | 14 | def __init__(self): 15 | self.model_class = None # deprecated 16 | self.model_classes = [] 17 | self.fields = [] 18 | 19 | @abstractmethod 20 | def parse_and_build(self, query_string): 21 | pass 22 | 23 | def _parse(self, query_string): 24 | try: 25 | ast = sqlparse.parse_string(query_string) 26 | except pyparsing.ParseException as err: 27 | msg = [ 28 | 'Parse Error: %s' % err, 29 | query_string, 30 | '-' * (err.col - 1) + '^', 31 | ] 32 | logger.error('\n'.join(msg)) 33 | raise 34 | 35 | return ast 36 | -------------------------------------------------------------------------------- /sqlparse/builders/mongo_builder.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | import logging 3 | 4 | import sqlparse 5 | from sqlparse import nodes 6 | from sqlparse.visitors import IdentifierAndValueVisitor 7 | from .base import QueryBuilder 8 | 9 | logger = logging.getLogger(__name__) 10 | 11 | 12 | class MongoQueryVisitor(IdentifierAndValueVisitor): 13 | # Map of SQL operators to MongoDB equivalents 14 | # TODO: Create node classes for these operators, rather than relying on operator.name 15 | OPERATORS = { 16 | 'not': '$nor', 17 | '!': '$nor', 18 | '!=': '$ne', 19 | '<>': '$ne', 20 | '<': '$lt', 21 | '<=': '$lte', 22 | '>': '$gt', 23 | '>=': '$gte', 24 | 'and': '$and', 25 | '&&': '$and', 26 | 'or': '$or', 27 | '||': '$or', 28 | 'in': '$in', 29 | 'mod': '$mod', 30 | '%': '$mod', 31 | 'like': '$regex', 32 | # Mongo doesn't support: + - * / ** << >> 33 | } 34 | 35 | def visit_UnaryOperator(self, node): 36 | op_name = self.OPERATORS.get(node.name) 37 | if op_name is None: 38 | raise ValueError('Mongo visitor does not implement "%s" unary operator') 39 | 40 | rhs_node = self.visit(node.rhs) 41 | if not isinstance(rhs_node, nodes.ListValue): 42 | rhs_node = [ rhs_node ] 43 | 44 | return { op_name: rhs_node } 45 | 46 | def visit_BinaryOperator(self, node): 47 | lhs_node = self.visit(node.lhs) 48 | rhs_node = self.visit(node.rhs) 49 | 50 | if node.name == '=': 51 | # Mongo treats equality struct different from other binary operators 52 | if isinstance(lhs_node, str): 53 | return {lhs_node: rhs_node} 54 | else: 55 | raise ValueError('lhs is an expression: %s' % lhs_node) 56 | 57 | elif node.name in ('xor', '^'): 58 | # Mongo lacks an XOR operator 59 | return { 60 | '$and': [ 61 | {'$or': [lhs_node, rhs_node]}, 62 | {'$and': [ 63 | {'$nor': [lhs_node]}, 64 | {'$nor': [rhs_node]} 65 | ]} 66 | ]} 67 | 68 | elif node.name == 'between': 69 | # Mongo lacks a BETWEEN operator 70 | return { 71 | '$and': [ 72 | {lhs_node: {'$gte': rhs_node.begin}}, 73 | {lhs_node: {'$lte': rhs_node.end}} 74 | ]} 75 | 76 | # Standard binary operator 77 | else: 78 | op_name = self.OPERATORS.get(node.name) 79 | if op_name is None: 80 | raise ValueError('Mongo visitor does not implement "%s" binary operator' % node.name) 81 | 82 | # AND and OR have list operands 83 | if op_name in ('$and', '$or'): 84 | return {op_name: [lhs_node, rhs_node]} 85 | # Everything else contains a { prop: expr } operand 86 | elif isinstance(lhs_node, str): 87 | return {lhs_node: {op_name: rhs_node}} 88 | else: 89 | raise ValueError('lhs is an expression: %s' % lhs_node) 90 | 91 | 92 | class MongoQueryBuilder(QueryBuilder): 93 | """ 94 | Builds a MongoDB query from a SQL query 95 | """ 96 | def parse_and_build(self, query_string): 97 | parse_tree = sqlparse.parse_string(query_string) 98 | filter_options = {} 99 | 100 | # collections 101 | self.model_class = self._get_collection_name(parse_tree) 102 | self.class_names = [self.model_class] 103 | 104 | # fields 105 | filter_fields = self._get_fields_option(parse_tree) 106 | self.fields = list(filter_fields.keys()) 107 | if filter_fields: 108 | filter_options['fields'] = filter_fields 109 | 110 | return self._get_filter_criteria(parse_tree), filter_options 111 | 112 | def _get_filter_criteria(self, parse_tree): 113 | """ 114 | Filter criteria specified in WHERE 115 | """ 116 | filter_criteria = MongoQueryVisitor().visit(parse_tree.where[0]) 117 | # print('WHERE: {}', json.dumps(filter_criteria, indent=4)) 118 | return filter_criteria 119 | 120 | def _get_collection_name(self, parse_tree): 121 | """ 122 | Collections specified in FROM 123 | """ 124 | collections = [str(table.name) for table in parse_tree.tables.values] 125 | if len(collections) == 0: 126 | raise ValueError('Collection name required in FROM clause') 127 | 128 | collection = collections[0] 129 | # print('FROM: {}', collection) 130 | 131 | # TODO: parse this as an Identifier instead of a str 132 | if not isinstance(collection, str): 133 | raise ValueError('collection name must be a string') 134 | 135 | if len(collections) > 1: 136 | raise ValueError('Mongo query requires single collection in FROM clause') 137 | 138 | return collection 139 | 140 | def _get_fields_option(self, parse_tree): 141 | """ 142 | Fields specified in SELECT 143 | """ 144 | fields = IdentifierAndValueVisitor().visit(parse_tree.columns) 145 | # print('SELECT: {}', fields) 146 | if not isinstance(fields, list): 147 | raise ValueError('SELECT must be a list') 148 | 149 | filter_fields = {} 150 | for field in fields: 151 | if field == '*': 152 | return {} 153 | 154 | filter_fields[field.name] = 1 155 | 156 | return filter_fields 157 | -------------------------------------------------------------------------------- /sqlparse/builders/sqlalchemy_builder.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | import operator 3 | import logging 4 | import inspect 5 | 6 | import sqlalchemy 7 | from sqlalchemy.orm.session import Session 8 | 9 | from sqlparse import nodes 10 | from sqlparse.visitors import IdentifierAndValueVisitor 11 | from .base import QueryBuilder 12 | 13 | logger = logging.getLogger(__name__) 14 | 15 | 16 | class SqlAlchemyQueryVisitor(IdentifierAndValueVisitor): 17 | BINARY_OPERATORS = { 18 | '=': operator.eq, 19 | '!=': operator.ne, 20 | '<>': operator.ne, 21 | '<': operator.lt, 22 | '<=': operator.le, 23 | '>': operator.gt, 24 | '>=': operator.ge, 25 | 'and': sqlalchemy.and_, 26 | '&&': sqlalchemy.and_, 27 | 'or': sqlalchemy.or_, 28 | '||': sqlalchemy.or_, 29 | 'in': lambda lhs, rhs: lhs.in_(rhs), 30 | 'between': lambda lhs, rhs: lhs.between(rhs.begin, rhs.end), 31 | 'like': lambda lhs, rhs: lhs.like(rhs), 32 | #'ilike': lambda lhs, rhs: lhs.ilike(rhs), # TODO: implement in grammar 33 | 34 | '+': operator.add, 35 | '-': operator.sub, 36 | '*': operator.mul, 37 | '/': operator.truediv, 38 | '%': operator.mod, 39 | '**': operator.pow, 40 | '<<': operator.lshift, 41 | '>>': operator.rshift 42 | } 43 | 44 | UNARY_OPERATORS = { 45 | '!': sqlalchemy.not_, 46 | 'not': sqlalchemy.not_, 47 | '~': operator.inv, 48 | '-': operator.neg, 49 | '+': operator.pos 50 | } 51 | 52 | def __init__(self, model_class): 53 | self.model_class = model_class 54 | 55 | def visit_UnaryOperator(self, node): 56 | op_func = self.UNARY_OPERATORS.get(node.name) 57 | if op_func is None: 58 | raise ValueError('SqlAlchemy visitor does not implement "%s" unary operator') 59 | 60 | return op_func(self.visit(node.rhs)) 61 | 62 | def visit_BinaryOperator(self, node): 63 | # XOR operator 64 | if node.name in ('xor', '^'): 65 | # SqlAlchemy doesn't support XOR operator 66 | lhs_node = self.visit(node.lhs) 67 | rhs_node = self.visit(node.rhs) 68 | return sqlalchemy.and_( 69 | sqlalchemy.or_(lhs_node, rhs_node), 70 | sqlalchemy.not_( 71 | sqlalchemy.and_(lhs_node, rhs_node))) 72 | 73 | # Regular operator 74 | else: 75 | op_func = self.BINARY_OPERATORS.get(node.name) 76 | if op_func is None: 77 | raise ValueError('Mongo visitor does not implement "%s" binary operator' % node.name) 78 | 79 | return op_func(self.visit(node.lhs), self.visit(node.rhs)) 80 | 81 | def visit_Identifier(self, node): 82 | # Ensure property is mapped in SqlAlchemy (and thus can be queried) 83 | mapped_properties = set([p.key for p in self.model_class.__mapper__.iterate_properties]) 84 | if node.name not in mapped_properties: 85 | raise ValueError('%s property is not a mapped relation, and can not be queried with SqlAlchemy' % node.name) 86 | 87 | # Class property that can be used in SqlAlchemy query expressions 88 | return getattr(self.model_class, node.name) 89 | 90 | 91 | class SqlAlchemyQueryBuilder(QueryBuilder): 92 | """ 93 | Builds a SqlAlchemy query from a SQL query 94 | """ 95 | def __init__(self, session, model_scope=None): 96 | super(SqlAlchemyQueryBuilder, self).__init__() 97 | 98 | if session is None: 99 | raise ValueError('session is required') 100 | if not isinstance(session, Session): 101 | raise ValueError('session must be a SqlAlchemy session object') 102 | 103 | # TODO: figure out if there's a way to do detached queries w/o depending on session (like hibernate) 104 | self.session = session 105 | 106 | if model_scope is None: 107 | model_scope = globals() 108 | 109 | self.model_scope = model_scope 110 | 111 | def parse_and_build(self, query_string): 112 | parse_tree = self._parse(query_string) 113 | 114 | self.model_class = self._get_model_class(parse_tree) 115 | 116 | self.fields = self._get_projection(parse_tree) 117 | query = self.session.query(self.model_class) 118 | 119 | criteria = self._get_filter_criteria(self.model_class, parse_tree) 120 | if criteria is not None: 121 | query = query.filter(criteria) 122 | 123 | return query 124 | 125 | def _get_model_class(self, parse_tree): 126 | class_names = [v.name for v in parse_tree.tables.values] 127 | if len(class_names) == 0: 128 | raise ValueError('Model name required in FROM clause') 129 | 130 | # TODO: support multiple classes and aliases 131 | if len(class_names) > 1: 132 | raise NotImplementedError('SqlAlchemy queries currently only support a single model class') 133 | 134 | class_name = class_names[0] 135 | # print('FROM: {}', class_name) 136 | 137 | klass = self.model_scope.get(class_name) 138 | if klass is None: 139 | raise ValueError('Model class %s not found in model_scope' % class_name) 140 | elif not inspect.isclass(klass): 141 | raise ValueError('Model class %s is not a class' % class_name) 142 | 143 | return klass 144 | 145 | def _get_projection(self, parse_tree): 146 | projection = IdentifierAndValueVisitor().visit(parse_tree.columns) 147 | # print('SELECT: {}', projection) 148 | if not isinstance(projection, list): 149 | raise ValueError('SELECT must be a list') 150 | 151 | fields = [] 152 | for field in projection: 153 | if not isinstance(field, nodes.Identifier): 154 | raise NotImplementedError('Only identifiers can be used in SELECT clause') 155 | fields.append(field.name) 156 | 157 | return fields 158 | 159 | def _get_filter_criteria(self, model_class, parse_tree): 160 | filter_criteria = SqlAlchemyQueryVisitor(model_class).visit(parse_tree.where[0]) 161 | print('WHERE: {}', filter_criteria) 162 | return filter_criteria 163 | -------------------------------------------------------------------------------- /sqlparse/grammar.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # 3 | # Copyright 2014 Chris Lyon 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | # 17 | import pyparsing 18 | from pyparsing import \ 19 | Forward, Group, Combine, Suppress, StringEnd, \ 20 | Optional, ZeroOrMore, OneOrMore, oneOf, \ 21 | operatorPrecedence, opAssoc, \ 22 | Word, Literal, CaselessLiteral, Regex, \ 23 | alphas, nums, alphanums, quotedString, \ 24 | restOfLine, quotedString, delimitedList, \ 25 | ParseResults, ParseException 26 | 27 | from .nodes import * 28 | 29 | ################################ 30 | # Terminals 31 | ################################ 32 | 33 | # Keywords 34 | WHERE = CaselessLiteral('where') 35 | FROM = CaselessLiteral('from') 36 | 37 | SELECT = CaselessLiteral('select') 38 | SELECT_DISTINCT = CaselessLiteral('distinct') 39 | SELECT_ALL = CaselessLiteral('all') 40 | AS = CaselessLiteral('as') 41 | 42 | WITH = CaselessLiteral('with') 43 | RECURSIVE = CaselessLiteral('recursive') 44 | 45 | PIVOT = CaselessLiteral('pivot') 46 | # UNPIVOT = CaselessLiteral('unpivot') 47 | PIVOT_IN = CaselessLiteral('in') 48 | PIVOT_FOR = CaselessLiteral('for') 49 | 50 | ORDER_BY = CaselessLiteral('order by') 51 | ORDER_ASC = CaselessLiteral('asc') 52 | ORDER_DESC = CaselessLiteral('desc') 53 | 54 | # Special values 55 | VAL_NULL = CaselessLiteral('null') 56 | VAL_TRUE = CaselessLiteral('true') 57 | VAL_FALSE = CaselessLiteral('false') 58 | VAL_UNKNOWN = CaselessLiteral('unknown') 59 | 60 | # Joins 61 | # JOIN = CaselessLiteral('join') 62 | # OUTER = CaselessLiteral('outer') 63 | # INNER = CaselessLiteral('inner') 64 | # LEFT_JOIN = CaselessLiteral('left') + Optional(OUTER) + JOIN 65 | # RIGHT_JOIN = CaselessLiteral('right') + Optional(OUTER) + JOIN 66 | # INNER_JOIN = CaselessLiteral(INNER) + JOIN 67 | # UNION = CaselessLiteral('union') 68 | # UNION_ALL = UNION + CaselessLiteral('all') 69 | # GROUP_BY = CaselessLiteral('group by') 70 | # HAVING = CaselessLiteral('having') 71 | # ORDER_BY = CaselessLiteral('order by') 72 | 73 | # Operators (name is operators.FUNC_NAME) 74 | OP_EQUAL = Literal('=') 75 | OP_VAL_NULLSAFE_EQUAL = Literal('<=>') 76 | OP_NOTEQUAL = (Literal('!=') | Literal('<>')) 77 | OP_GT = Literal('>').setName('gt') 78 | OP_LT = Literal('<').setName('lt') 79 | OP_GTE = Literal('>=').setName('ge') 80 | OP_LTE = Literal('<=').setName('le') 81 | OP_IN = CaselessLiteral('in') # sqlalchemy property: lhs.in_(rhs) 82 | OP_LIKE = CaselessLiteral('like') # sqlalchemy property: lhs.like(rhs), lhs.ilike(rhs) 83 | OP_IS = CaselessLiteral('is') # sqlalchemy or_(lhs == rhs, lhs == None) 84 | OP_BETWEEN = CaselessLiteral('between') # sqlalchemy: between 85 | OP_BETWEEN_AND = Suppress(CaselessLiteral('and')) 86 | 87 | # Math 88 | # OP_ADD = Literal('+') 89 | # OP_SUB = Literal('-') 90 | # OP_MUL = Literal('*') 91 | # OP_DIV = (Literal('/') | CaselessLiteral('div')) 92 | # OP_EXP = Literal('**') # standard? 93 | # addOp = OP_ADD | OP_SUB 94 | # multOp = OP_MUL | OP_DIV 95 | # OP_SHL = Literal('<<') 96 | # OP_SHR = Literal('>>') 97 | 98 | # Bitwise Operators 99 | # BITOP_AND = Literal('&') 100 | # BITOP_OR = Literal('|') 101 | # BITOP_NOT = Literal('~') # unary 102 | # BITOP_XOR = Literal('^') 103 | 104 | # Conjugates 105 | LOGOP_AND = (CaselessLiteral('and') | CaselessLiteral('&&')) 106 | LOGOP_OR = (CaselessLiteral('or') | CaselessLiteral('||')) 107 | LOGOP_NOT = (CaselessLiteral('not') | CaselessLiteral('!')) 108 | LOGOP_XOR = CaselessLiteral('xor') 109 | 110 | # SELECT Statement Operators 111 | SELECTOP_EXCEPT = CaselessLiteral('except') 112 | SELECTOP_INTERSECT = CaselessLiteral('intersect') 113 | SELECTOP_UNION = CaselessLiteral('union') 114 | SELECTOP_UNION_ALL = CaselessLiteral('all') 115 | 116 | # Grouping 117 | L_PAREN = Suppress('(') 118 | R_PAREN = Suppress(')') 119 | 120 | # Math 121 | E = CaselessLiteral('e') 122 | STAR = Literal('*') 123 | DOT = Literal('.') 124 | PLUS = Literal('+') 125 | MINUS = Literal('-') 126 | 127 | ################################ 128 | # Non-Terminals 129 | ################################ 130 | 131 | # TODO: WITH ( RECURSIVE ) 132 | 133 | sign = PLUS | MINUS 134 | 135 | selectStmt = Forward() # SELECT 136 | 137 | identifier = Word(alphas, alphanums + '_$')('identifier') # a, A1, a_1$ 138 | # alias = (Optional(AS) + identifier).setName('alias') 139 | 140 | # Projection 141 | columnName = delimitedList(identifier, DOT, combine=True).setParseAction(Identifier)('column') # TODO: x AS y, x y, x `y`, x 'y', `x`, 'x' 142 | columnNameList = Group(delimitedList(STAR | columnName)).setParseAction(ListValue) 143 | tableName = delimitedList(identifier, DOT, combine=True).setParseAction(Identifier)('table') 144 | tableNameList = Group(delimitedList(tableName)).setParseAction(ListValue) 145 | 146 | whereExpr = Forward() # WHERE 147 | 148 | # TODO: indirect comparisons (e.g. "table1.field1.xyz = 3" becomes "table1.any(field1.xyz == 3)") 149 | # TODO: math expression grammar (for both lval and rval) 150 | equalityOp = ( 151 | OP_VAL_NULLSAFE_EQUAL ^ 152 | OP_EQUAL ^ 153 | OP_NOTEQUAL ^ 154 | OP_LT ^ 155 | OP_GT ^ 156 | OP_GTE ^ 157 | OP_LTE 158 | ) 159 | likeOp = (Optional(LOGOP_NOT) + OP_LIKE) 160 | 161 | betweenOp = Optional(LOGOP_NOT) + OP_BETWEEN # [ NOT ] BETWEEN 162 | 163 | stringValue = quotedString.setParseAction(StringValue) 164 | 165 | realNumber = ( 166 | Combine( 167 | Optional(sign) + ( 168 | # decimal present 169 | ((Word(nums) + DOT + Optional(Word(nums)) | (DOT + Word(nums))) + 170 | Optional(E + Optional(sign) + Word(nums))) | 171 | # negative exp 172 | (Word(nums) + Optional(E + Optional(MINUS) + Word(nums))) 173 | ) 174 | ).setParseAction(RealValue) 175 | ).setName('real') # .1, 1.2, 1.2e3, -1.2e+3, 1.2e-3 176 | 177 | intNumber = ( 178 | Combine( 179 | Optional(sign) + 180 | Word(nums) 181 | # Optional(E + Optional(PLUS) + Word(nums)) # python int() doesn't grok this 182 | ).setParseAction(IntegerValue) 183 | ).setName('int') # -1 0 1 23 184 | 185 | number = intNumber ^ realNumber 186 | 187 | atom = ( 188 | number | 189 | stringValue('string') # normalize quotes 190 | ) 191 | 192 | groupSubSelectStmt = Group(R_PAREN + selectStmt + R_PAREN) # todo: subselect must have a LIMIT in this context 193 | 194 | columnRval = ( 195 | atom('value') | 196 | columnName('column') | 197 | groupSubSelectStmt('query') 198 | ) 199 | 200 | likePattern = ( 201 | stringValue('value') 202 | ) 203 | 204 | inOperand = Suppress(L_PAREN) + Group(delimitedList(columnRval))('value').setParseAction(ListValue) + Suppress(R_PAREN) 205 | 206 | # TODO: Functions: sum, avg, count, max, min, ifnull/isnull, if 207 | # current_date, current_time, current_timestamp, current_user 208 | # substring, regex, concat, group_concat 209 | # cast, convert 210 | whereCond = Forward() 211 | whereCond << ( 212 | Group(LOGOP_NOT + whereCond)('op').setParseAction(UnaryOperator) | 213 | Group(columnName('column') + equalityOp('op') + columnRval).setParseAction(BinaryOperator) | # x = y, x != y, etc. 214 | Group(columnName('column') + likeOp('op') + likePattern).setParseAction(BinaryOperator) | 215 | Group(columnName('column') + betweenOp('op') + Group(columnRval + OP_BETWEEN_AND + columnRval)('range').setParseAction(RangeValue)).setParseAction(BinaryOperator) | # x between y and z, x not between y and z 216 | Group(columnName('column') + Group( 217 | OP_IS + 218 | Optional(LOGOP_NOT))('op') + 219 | (VAL_NULL | VAL_TRUE | VAL_FALSE | VAL_UNKNOWN)('value') 220 | ) | # x is null, x is not null 221 | Group(columnName('column') + OP_IN('op') + inOperand).setParseAction(BinaryOperator) | 222 | # Group( columnName('column') + Combine( LOGOP_NOT + OP_IN )('op') + inOperand ) | 223 | (L_PAREN + whereExpr('expr') + R_PAREN) 224 | ) 225 | 226 | 227 | # logOp = operatorPrecedence( 228 | # whereExpr('expr'), [ 229 | # (LOGOP_NOT, 1, opAssoc.RIGHT, UnaryOperator), 230 | # (LOGOP_AND, 2, opAssoc.RIGHT, BinaryOperator), 231 | # (LOGOP_OR, 2, opAssoc.RIGHT, BinaryOperator) 232 | # ]) 233 | whereExpr << ( 234 | whereCond ^ 235 | # Group(LOGOP_NOT('op') + whereCond )('expr').setParseAction(UnaryOperator) ^ 236 | Group( 237 | whereCond + 238 | OneOrMore( 239 | LOGOP_AND('op') + whereExpr('expr') | 240 | LOGOP_XOR('op') + whereExpr('expr') | 241 | LOGOP_OR('op') + whereExpr('expr') 242 | ) 243 | ).setParseAction(BinaryOperator) 244 | ) 245 | 246 | columnProjection = ( 247 | Optional(SELECT_DISTINCT | SELECT_ALL).setResultsName('options') + 248 | columnNameList('columns') 249 | ) 250 | 251 | fromClause = Suppress(FROM) + tableNameList('tables') 252 | 253 | # TODO: ( LEFT | RIGHT ) ( INNER | OUTER ) JOIN 254 | 255 | # TODO: PIVOT, UNPIVOT 256 | pivotClause = Optional( 257 | Group( 258 | PIVOT + L_PAREN + Group(columnNameList) + 259 | PIVOT_FOR + columnName + 260 | PIVOT_IN + Group(columnNameList) + 261 | R_PAREN 262 | ) 263 | )('pivot') 264 | 265 | whereClause = Optional(Suppress(WHERE) + whereExpr)('where') 266 | 267 | # TODO: GROUP BY 268 | # TODO: HAVING 269 | # TODO: LIMIT, OFFSET 270 | 271 | # ORDER BY x, y ASC, d DESC, ... 272 | orderDirection = ORDER_ASC | ORDER_DESC 273 | orderByColumnList = Group(delimitedList(columnName('column') + Optional(orderDirection)('direction'))) 274 | orderByClause = Optional(Suppress(ORDER_BY) + orderByColumnList('order')) # todo: asc, desc 275 | 276 | selectStmt << ( 277 | Suppress(SELECT) + 278 | columnProjection + 279 | Optional( 280 | fromClause + 281 | # pivotClause + 282 | whereClause 283 | ) 284 | # orderByClause 285 | ) 286 | 287 | # UNION ( ALL ) 288 | unionOp = Combine(SELECTOP_UNION + Optional(SELECTOP_UNION_ALL)) 289 | 290 | # SELECT ... ( UNION | INTERSECT | EXCEPT ) SELECT ... 291 | selectStmts = selectStmt + ZeroOrMore((unionOp | SELECTOP_INTERSECT | SELECTOP_EXCEPT) + selectStmt) 292 | 293 | # Start symbol 294 | sqlQuery = selectStmts + StringEnd() 295 | 296 | # Ignore comments 297 | commentStart = Suppress(oneOf('-- #')) 298 | comment = commentStart + restOfLine 299 | sqlQuery.ignore(comment) 300 | -------------------------------------------------------------------------------- /sqlparse/nodes.py: -------------------------------------------------------------------------------- 1 | from abc import ABCMeta, abstractmethod 2 | from decimal import Decimal 3 | 4 | 5 | class ASTNode(object): 6 | """ 7 | Node in abstract syntax tree 8 | """ 9 | __metaclass__ = ABCMeta 10 | 11 | 12 | class Value(ASTNode): 13 | __metaclass__ = ABCMeta 14 | 15 | 16 | class StringValue(Value): 17 | """ 18 | 'value' 19 | "value" 20 | """ 21 | def __init__(self, tokens): 22 | self.value = tokens[0][1:-1] 23 | 24 | 25 | class IntegerValue(Value): 26 | """ 27 | ... -1 0 1 2 ... 28 | """ 29 | def __init__(self, tokens): 30 | self.value = int(tokens[0]) 31 | 32 | 33 | class RealValue(Value): 34 | """ 35 | .1 36 | 1.2 37 | -1.23 38 | 1.2e3 39 | 1.2e-3 40 | """ 41 | def __init__(self, tokens): 42 | self.value = Decimal(tokens[0]) 43 | 44 | 45 | class ListValue(Value): 46 | """ 47 | [x,y,...] 48 | (x,y,...) 49 | """ 50 | def __init__(self, tokens): 51 | self.values = list(tokens[0]) 52 | self.frozen = False 53 | 54 | def __repr__(self): 55 | return "'({})".format(' '.join(map(str, self.values))) 56 | 57 | 58 | class RangeValue(Value): 59 | """ 60 | [x...y] 61 | (x...y) 62 | between x and y 63 | """ 64 | def __init__(self, tokens): 65 | self.begin, self.end = tokens[0] 66 | 67 | def __repr__(self): 68 | return "{}...{}".format(self.begin, self.end) 69 | 70 | 71 | class Identifier(ASTNode): 72 | """ 73 | 74 | """ 75 | def __init__(self, tokens): 76 | self.name = tokens[0][0] 77 | 78 | 79 | class ModelIdentifier(Identifier): 80 | pass 81 | 82 | 83 | class ProjectionExpression(ASTNode): 84 | def __init__(self, tokens): 85 | self.projection = tokens[0] 86 | 87 | 88 | class PredicateExpression(ASTNode): 89 | """ 90 | Expression that can be used for filtering, such as in a SELECT, WHERE, 91 | ON clause, or HAVING function 92 | """ 93 | def __init__(self, tokens): 94 | self.expression = tokens[0] 95 | 96 | 97 | class Function(ASTNode): 98 | """ 99 | Commonly used for aggregate functions (e.g. MIN,MAX,AVG,etc.) 100 | 101 | f(x,y) 102 | """ 103 | __metaclass__ = ABCMeta 104 | 105 | def __init__(self, tokens): 106 | self.name, self.args = tokens[0] 107 | self.name = self.name.lower() 108 | 109 | 110 | class UnaryOperator(Function): 111 | """ 112 | -x +x ~x 113 | not 114 | """ 115 | def __init__(self, tokens): 116 | super(UnaryOperator, self).__init__(tokens) 117 | self.rhs = self.args 118 | 119 | def __repr__(self): 120 | return '({} {})'.format(self.name, self.args) 121 | 122 | 123 | class BinaryOperator(Function): 124 | """ 125 | x+y x-y x*y x**y x^y x/y 126 | | & << <<< >> >>> or xor and in 127 | """ 128 | def __init__(self, tokens): 129 | self.lhs, self.name, self.rhs = tokens[0] 130 | self.args = (self.lhs, self.rhs) 131 | # super(BinaryOperator, self).__init__() 132 | 133 | def __repr__(self): 134 | return '({} {} {})'.format(self.name, self.lhs, self.rhs) 135 | 136 | 137 | class Statement(ASTNode): 138 | def __init__(self, **kwargs): 139 | for k, v in kwargs.items(): 140 | setattr(self, k, v) 141 | -------------------------------------------------------------------------------- /sqlparse/nodevisitor.py: -------------------------------------------------------------------------------- 1 | ############################################################################### 2 | # 3 | # Copyright (c) 2011 Ruslan Spivak 4 | # 5 | # Permission is hereby granted, free of charge, to any person obtaining a copy 6 | # of this software and associated documentation files (the "Software"), to deal 7 | # in the Software without restriction, including without limitation the rights 8 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | # copies of the Software, and to permit persons to whom the Software is 10 | # furnished to do so, subject to the following conditions: 11 | # 12 | # The above copyright notice and this permission notice shall be included in 13 | # all copies or substantial portions of the Software. 14 | # 15 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 21 | # THE SOFTWARE. 22 | # 23 | ############################################################################### 24 | 25 | __author__ = 'Ruslan Spivak ' 26 | 27 | 28 | class ASTVisitor(object): 29 | """Base class for custom AST node visitors. 30 | 31 | Example: 32 | 33 | >>> from slimit.parser import Parser 34 | >>> from slimit.visitors.nodevisitor import ASTVisitor 35 | >>> 36 | >>> text = ''' 37 | ... var x = { 38 | ... "key1": "value1", 39 | ... "key2": "value2" 40 | ... }; 41 | ... ''' 42 | >>> 43 | >>> class MyVisitor(ASTVisitor): 44 | ... def visit_Object(self, node): 45 | ... '''Visit object literal.''' 46 | ... for prop in node: 47 | ... left, right = prop.left, prop.right 48 | ... print 'Property value: {}'.format(right.value) 49 | ... # visit all children in turn 50 | ... self.visit(prop) 51 | ... 52 | >>> 53 | >>> parser = Parser() 54 | >>> tree = parser.parse(text) 55 | >>> visitor = MyVisitor() 56 | >>> visitor.visit(tree) 57 | Property value: "value1" 58 | Property value: "value2" 59 | 60 | """ 61 | 62 | def visit(self, node): 63 | method = 'visit_{}'.format(node.__class__.__name__) 64 | return getattr(self, method, self.generic_visit)(node) 65 | 66 | def generic_visit(self, node): 67 | for child in node: 68 | self.visit(child) 69 | 70 | 71 | class NodeVisitor(object): 72 | """Simple node visitor.""" 73 | 74 | def visit(self, node): 75 | """Returns a generator that walks all children recursively.""" 76 | for child in node: 77 | yield child 78 | for subchild in self.visit(child): 79 | yield subchild 80 | 81 | 82 | def visit(node): 83 | visitor = NodeVisitor() 84 | for child in visitor.visit(node): 85 | yield child 86 | -------------------------------------------------------------------------------- /sqlparse/test/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Flushot/sqlparse/58bf73c9f106c3834e4e1028e98b3c0af3a59ff7/sqlparse/test/__init__.py -------------------------------------------------------------------------------- /sqlparse/test/base.py: -------------------------------------------------------------------------------- 1 | import os 2 | import unittest 3 | 4 | from pyparsing import ParseException 5 | import sqlparse 6 | 7 | 8 | class BuilderTestCase(unittest.TestCase): 9 | pass 10 | 11 | 12 | class ParserTestCase(unittest.TestCase): 13 | # Print results XML to console? 14 | PRINT_PARSE_RESULTS = bool(os.environ.get('PRINT_PARSE_RESULTS', False)) 15 | 16 | def assertParses(self, input_str: str, expect_error: bool = False): 17 | """ 18 | parses :input_str: and assets the parse succeeded 19 | (or failed if :expectError: is True) 20 | 21 | returns ParseResults with parse tree 22 | 23 | parameters 24 | ---------- 25 | input_str : str 26 | query string to parse 27 | expect_error : bool 28 | is this an intentionally malformed inputStr? 29 | if so, suppresses the ParseException 30 | """ 31 | if isinstance(input_str, list): 32 | return map(lambda i: self.assertParses(i, expect_error=expect_error), input_str) 33 | 34 | try: 35 | if self.PRINT_PARSE_RESULTS and not expect_error: 36 | print("\n{}".format(input_str)) 37 | 38 | tokens = sqlparse.parse_string(input_str) 39 | if self.PRINT_PARSE_RESULTS and not expect_error: 40 | print(tokens.asXML('query')) 41 | #print tokens.where.dump() 42 | 43 | #print tokens.where.dump() 44 | 45 | self.assertFalse(expect_error, input_str) # parsed without error = pass 46 | return tokens 47 | 48 | except ParseException as err: 49 | if self.PRINT_PARSE_RESULTS and not expect_error: 50 | # print(err.line) 51 | print('-' * (err.col - 1) + '^') 52 | print('ERROR: {}'.format(err)) 53 | self.assertTrue(expect_error, '{}, parsing "{}"'.format(err, input_str)) 54 | -------------------------------------------------------------------------------- /sqlparse/test/mongo_builder_test.py: -------------------------------------------------------------------------------- 1 | import itertools 2 | import logging 3 | 4 | import pymongo 5 | 6 | from .base import BuilderTestCase 7 | from sqlparse.builders import MongoQueryBuilder 8 | 9 | logger = logging.getLogger(__name__) 10 | 11 | 12 | class MongoQueryBuilderTest(BuilderTestCase): 13 | def setUp(self): 14 | self.client = pymongo.MongoClient('mongodb://localhost:27017') 15 | self.db = self.client.test 16 | 17 | self.collection = self.db['User'] 18 | self.collection.drop() 19 | 20 | names = { 21 | 'first': [ 22 | 'Chris', 23 | 'John', 24 | 'Bob' 25 | ], 26 | 'last': [ 27 | 'Jacob', 28 | 'Smith', 29 | 'Lyon' 30 | ] 31 | } 32 | 33 | for idx, (first_name, last_name) in enumerate(itertools.product(names['first'], names['last'])): 34 | self.collection.insert({ 35 | '_id': str(idx), 36 | 'first_name': first_name, 37 | 'last_name': last_name, 38 | 'is_active': (first_name == 'Chris') 39 | }) 40 | 41 | def tearDown(self): 42 | self.db = None 43 | self.client = None 44 | 45 | def test_SELECT(self): 46 | builder = MongoQueryBuilder() 47 | query, options = builder.parse_and_build(""" 48 | select a,b from User where 49 | not ( last_name= 'Jacob' or 50 | (first_name !='Chris' and last_name!='Lyon') ) and 51 | not is_active = 1 52 | """) 53 | 54 | self.assertEquals('User', builder.model_class) 55 | 56 | self.assertEquals(['a', 'b'], builder.fields) 57 | self.assertDictEqual({ 58 | 'fields': { 59 | 'a': 1, 60 | 'b': 1 61 | } 62 | }, options) 63 | 64 | self.assertDictEqual({ 65 | "$and": [ 66 | { 67 | "$nor": [ 68 | { 69 | "$or": [ 70 | { "last_name": "Jacob" }, 71 | { 72 | "$and": [ 73 | {"first_name": {"$ne": "Chris"}}, 74 | {"last_name": {"$ne": "Lyon"}} 75 | ] 76 | } 77 | ] 78 | } 79 | ] 80 | }, 81 | { 82 | "$nor": [ 83 | {"is_active": 1} 84 | ] 85 | } 86 | ] 87 | }, query) 88 | 89 | results = self.collection.find(query, options) 90 | # for result in results: 91 | # print(json.dumps(result)) 92 | 93 | self.assertEquals(4, results.count()) 94 | -------------------------------------------------------------------------------- /sqlparse/test/sqlalchemy_builder_test.py: -------------------------------------------------------------------------------- 1 | import itertools 2 | import logging 3 | 4 | import sqlalchemy.orm 5 | import sqlalchemy.ext.declarative 6 | 7 | from .base import BuilderTestCase 8 | from sqlparse.builders import SqlAlchemyQueryBuilder 9 | 10 | logger = logging.getLogger(__name__) 11 | 12 | 13 | class SqlAlchemyQueryBuilderTest(BuilderTestCase): 14 | def setUp(self): 15 | self.engine = sqlalchemy.create_engine('sqlite://', echo=False) 16 | 17 | Session = sqlalchemy.orm.sessionmaker(bind=self.engine) 18 | self.session = Session() 19 | 20 | Base = sqlalchemy.ext.declarative.declarative_base(bind=self.engine) 21 | 22 | class User(Base): 23 | __tablename__ = 'users' 24 | 25 | id = sqlalchemy.Column(sqlalchemy.Integer, primary_key=True) 26 | first_name = sqlalchemy.Column(sqlalchemy.String) 27 | last_name = sqlalchemy.Column(sqlalchemy.String) 28 | is_active = sqlalchemy.Column(sqlalchemy.Boolean) 29 | 30 | def __init__(self, **kwargs): 31 | for k, v in kwargs.items(): 32 | setattr(self, k, v) 33 | 34 | def __str__(self): 35 | return '%s %s' % (self.first_name, self.last_name) 36 | 37 | self.User = User 38 | self.model_scope = dict(User=self.User) 39 | 40 | # Add seed data 41 | Base.metadata.create_all() 42 | 43 | names = { 44 | 'first': [ 45 | 'Chris', 46 | 'John', 47 | 'Bob' 48 | ], 49 | 'last': [ 50 | 'Jacob', 51 | 'Smith', 52 | 'Lyon' 53 | ] 54 | } 55 | for first_name, last_name in itertools.product(names['first'], names['last']): 56 | self.session.add(User( 57 | first_name=first_name, 58 | last_name=last_name, 59 | is_active=(first_name == 'Chris'))) 60 | 61 | self.session.commit() 62 | 63 | def tearDown(self): 64 | pass 65 | 66 | def test_SELECT(self): 67 | builder = SqlAlchemyQueryBuilder(self.session, model_scope=self.model_scope) 68 | query = builder.parse_and_build(""" 69 | select a, b from User where 70 | not (last_name = 'Jacob' or 71 | (first_name != 'Chris' and last_name != 'Lyon')) and 72 | not is_active = 1 73 | """) 74 | 75 | self.assertEqual(['a', 'b'], builder.fields) 76 | 77 | results = query.all() 78 | # for user in results: 79 | # print(user.__dict__) 80 | 81 | self.assertEquals(2, len(results)) 82 | -------------------------------------------------------------------------------- /sqlparse/test/sqlparse_test.py: -------------------------------------------------------------------------------- 1 | from .base import ParserTestCase, unittest 2 | 3 | 4 | class TestSqlQueryGrammar(ParserTestCase): 5 | """ 6 | Test cases for grammar parsing 7 | """ 8 | @unittest.skip('unimplemented') 9 | def test_aliases(self): 10 | pass 11 | 12 | def test_SELECT_with_FROM_and_simple_join(self): 13 | # valid queries 14 | results = self.assertParses([ 15 | 16 | #'select 1 from a', # unsupported for now 17 | 18 | 'select * from xyzzy, ABC', 19 | 20 | 'select a, * from xyzzy, ABC', 21 | 22 | 'select *, a from xyzzy, ABC', 23 | 24 | 'select *, blah from xyzzy , abc', 25 | 26 | 'select a,b,c , D ,e from xyzzy,ABC', 27 | 28 | 'select all a,b,c from sys.blah ,Table2', 29 | 30 | ]) 31 | self.assertIsNotNone(results) 32 | for result in results: 33 | self.assertIsNotNone( result ) 34 | self.assertTrue( len(result.tables.values) > 1 ) 35 | self.assertTrue( len(result.columns.values) > 0 ) 36 | 37 | # invalid queries 38 | self.assertParses([ 39 | 40 | 'select * from a .b' 41 | 42 | ], expect_error=True) 43 | 44 | @unittest.skip('unimplemented') 45 | def test_SELECT_without_FROM(self): 46 | # standalone 'select x' 47 | pass 48 | 49 | @unittest.skip('unimplemented') 50 | def test_PIVOT(self): 51 | self.assertParses([ 52 | 53 | 'select a from b pivot ( q for col in (c1, c2, c3) ) where y = 1' 54 | 55 | ]) 56 | 57 | def test_explicit_JOIN(self): 58 | pass 59 | 60 | def test_WHERE_and_operators(self): 61 | self.assertParses([ 62 | 63 | 'select a from b where c = 1', 64 | 65 | 'select a from b where c = 1 and d = 2 and e = 3', 66 | 67 | 'select a from b where c <=> 1', 68 | 69 | 'select a from b where c is null', 70 | 71 | 'select a from b where c is not null', 72 | 73 | 'select a from b where b.a = "test"', 74 | 75 | 'select a from b where c = d or e = "f" or g = 1e2', 76 | 77 | 'select a from b where c > 1 and c < 2 and d >= 3 and d <= 4', 78 | 79 | 'select a from table where b = 3 and c = -1 or d = 1e-3 or e=-1.2e-3 or f!= -1.2e+3 or g =1.2e3', 80 | 81 | 'select a from b where i = 1 or i in (2,3, 4e2 ) and f in( .1, 1.2, -.1, +1.2, +1.2e+3, 1e-1, 1.2e-3, +1.2e-3,-4e2)', 82 | 83 | 'SELECT A from sys.blah where a in ("RED","GREEN", "BLUE")', 84 | 85 | 'select x from y where z between 10 and 30.5', 86 | 87 | 'select distinct a from b,x where ( c = 1 ) or ( d != 2 ) or e >= 3', 88 | 89 | 'select a from b where ( c > 1 ) or ( d = 2 ) or e < 3', 90 | 91 | 'SeLeCt * fRoM SYS.XYZZY where q is not null and q >= 1e2', 92 | 93 | 'select a from b where a not in (1,2,3.5) and d = 1', 94 | 95 | "select distinct A from sys.blah where a in (\"xx\",'yy', 'zz' ) and b in ( 10, 20,30,2.5 )", 96 | 97 | 'select a from table where b = 3 and c = -1 and d is null and e is not null', 98 | 99 | 'select a from b where c like \'%%blah%%\' and d not like "%%whatever" and c like \'l\'', 100 | 101 | 'select x from y,z where y.a != z.a or ( y.a > 3 and y.b = 1 ) and ( y.x <= a.x or ( y.x = 1 or y.y = 3 )) and z in (2,4,6)', 102 | 103 | ]) 104 | 105 | @unittest.skip('unimplemented') 106 | def test_subselect(self): 107 | self.assertParses([ 108 | 109 | 'select a from table where b = 3 and c = (select id from d where e = 1)', 110 | 111 | 'select a from b where c in (select d from e where f = 1)', 112 | 113 | ]) 114 | 115 | @unittest.skip('unimplemented') 116 | def test_GROUP_BY(self): 117 | pass 118 | 119 | @unittest.skip('unimplemented') 120 | def test_ORDER_BY(self): 121 | pass 122 | 123 | @unittest.skip('unimplemented') 124 | def test_HAVING(self): 125 | pass 126 | 127 | @unittest.skip('unimplemented') 128 | def test_LIMIT(self): 129 | pass 130 | 131 | @unittest.skip('unimplemented') 132 | def test_UNION_and_UNION_ALL(self): 133 | self.assertParses([ 134 | 135 | 'select a from b union select c from d', 136 | 137 | 'select a from b union all select c from d', 138 | 139 | 'select a from b union all (select c from d) except select e from f' 140 | 141 | ]) 142 | 143 | @unittest.skip('unimplemented') 144 | def test_EXCEPT(self): 145 | self.assertParses([ 146 | 147 | 'select a from b except select e from f' 148 | 149 | ]) 150 | 151 | @unittest.skip('unimplemented') 152 | def test_INTERSECT(self): 153 | self.assertParses([ 154 | 155 | 'select a from b intersect select e from f' 156 | 157 | ]) 158 | 159 | def test_comments(self): 160 | self.assertParses([ 161 | 162 | 'Select A , b,c from Sys.blah # ignored comment', 163 | 164 | 'select A,b from table1,table2 where table1.id = table2.id -- ignored comment' 165 | 166 | ]) 167 | 168 | if __name__ == '__main__': 169 | # import operator 170 | # import sqlalchemy 171 | 172 | unittest.main() 173 | 174 | # TODO: semantic analysis 175 | # TODO: dfs walk tokens.where and construct a sqlalchemy Query 176 | -------------------------------------------------------------------------------- /sqlparse/visitors.py: -------------------------------------------------------------------------------- 1 | from .nodevisitor import ASTVisitor 2 | 3 | 4 | class IdentifierVisitor(ASTVisitor): 5 | def visit_Identifier(self, node): 6 | # TODO: parse . notation 7 | return str(node.name) 8 | 9 | 10 | class ValueVisitor(ASTVisitor): 11 | def visit_StringValue(self, node): 12 | return str(node.value) 13 | 14 | def visit_IntegerValue(self, node): 15 | return int(node.value) 16 | 17 | def visit_RealValue(self, node): 18 | return str(node.value) # str = no precision loss 19 | 20 | def visit_ListValue(self, node): 21 | return list(node.values) 22 | 23 | def visit_RangeValue(self, node): 24 | raise NotImplementedError() 25 | 26 | 27 | class IdentifierAndValueVisitor(IdentifierVisitor, ValueVisitor): 28 | pass 29 | --------------------------------------------------------------------------------