├── test ├── test_helper.rb ├── formatter_test.rb ├── tokenizer_test.rb └── parser_test.rb ├── lib ├── sqlpp │ ├── version.rb │ ├── ast.rb │ ├── tokenizer.rb │ ├── formatter.rb │ └── parser.rb └── sqlpp.rb ├── Rakefile ├── bin └── sqlpp ├── sqlpp.gemspec ├── MIT-LICENSE └── README.md /test/test_helper.rb: -------------------------------------------------------------------------------- 1 | require 'sqlpp' 2 | require 'minitest/autorun' 3 | -------------------------------------------------------------------------------- /lib/sqlpp/version.rb: -------------------------------------------------------------------------------- 1 | module SQLPP 2 | module Version 3 | MAJOR = 1 4 | MINOR = 3 5 | TINY = 0 6 | 7 | STRING = [MAJOR, MINOR, TINY].join(".") 8 | end 9 | end 10 | -------------------------------------------------------------------------------- /lib/sqlpp.rb: -------------------------------------------------------------------------------- 1 | module SQLPP 2 | class Exception < RuntimeError; end 3 | end 4 | 5 | require 'sqlpp/tokenizer' 6 | require 'sqlpp/parser' 7 | require 'sqlpp/formatter' 8 | require 'sqlpp/version' 9 | -------------------------------------------------------------------------------- /Rakefile: -------------------------------------------------------------------------------- 1 | require 'rake/testtask' 2 | 3 | task default: :test 4 | 5 | Rake::TestTask.new do |t| 6 | t.libs << "test" 7 | t.test_files = FileList['test/*_test.rb'] 8 | t.verbose = true 9 | end 10 | -------------------------------------------------------------------------------- /bin/sqlpp: -------------------------------------------------------------------------------- 1 | #!/bin/sh ruby 2 | 3 | require 'sqlpp' 4 | 5 | sql = STDIN.read 6 | 7 | if ARGV.grep(/^-[h?]$/).any? 8 | puts "SQLPP (SQL Pretty Printer)" 9 | puts 10 | puts "Usage: #{$0} -h -? -wp < SQL" 11 | puts 12 | puts " -h or -?: this list of options" 13 | puts " -wp: wrap the projection lists" 14 | puts 15 | exit 16 | end 17 | 18 | projections = ARGV.include?("-wp") ? :wrap : nil 19 | 20 | ast = SQLPP::Parser.parse(sql) 21 | puts SQLPP::Formatter.new(projections: projections).format(ast) 22 | -------------------------------------------------------------------------------- /sqlpp.gemspec: -------------------------------------------------------------------------------- 1 | lib = File.expand_path('../lib', __FILE__) 2 | $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib) 3 | require "sqlpp/version" 4 | 5 | Gem::Specification.new do |gem| 6 | gem.version = SQLPP::Version::STRING 7 | gem.name = "sqlpp" 8 | gem.authors = ["Jamis Buck"] 9 | gem.email = ["jamis@jamisbuck.org"] 10 | gem.homepage = "http://github.com/jamis/sqlpp" 11 | gem.summary = "A simplistic SQL parser and pretty-printer" 12 | gem.description = "A simplistic SQL parser and pretty-printer" 13 | gem.license = 'MIT' 14 | 15 | gem.files = `git ls-files`.split($\) 16 | gem.executables = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) } 17 | gem.test_files = gem.files.grep(%r{^test/}) 18 | gem.require_paths = ["lib"] 19 | 20 | ## 21 | # Development dependencies 22 | # 23 | gem.add_development_dependency "rake" 24 | gem.add_development_dependency "minitest" 25 | end 26 | -------------------------------------------------------------------------------- /lib/sqlpp/ast.rb: -------------------------------------------------------------------------------- 1 | module SQLPP 2 | module AST 3 | class Select < Struct.new(:projections, :froms, :wheres, :groups, :orders, :distinct, :limit, :offset) 4 | end 5 | 6 | class Expr < Struct.new(:left, :op, :right, :not) 7 | end 8 | 9 | class Unary < Struct.new(:op, :expr) 10 | end 11 | 12 | class Atom < Struct.new(:type, :left, :right) 13 | end 14 | 15 | class Parens < Struct.new(:value) 16 | end 17 | 18 | class As < Struct.new(:name, :expr) 19 | end 20 | 21 | class Alias < Struct.new(:name, :expr) 22 | end 23 | 24 | class Join < Struct.new(:type, :left, :right, :on) 25 | end 26 | 27 | class SortKey < Struct.new(:key, :options) 28 | end 29 | 30 | class Limit < Struct.new(:expr) 31 | end 32 | 33 | class Offset < Struct.new(:expr) 34 | end 35 | 36 | class Subscript < Struct.new(:left, :right) 37 | end 38 | 39 | class TypeCast < Struct.new(:value, :type) 40 | end 41 | end 42 | end 43 | -------------------------------------------------------------------------------- /MIT-LICENSE: -------------------------------------------------------------------------------- 1 | Copyright 2015 Jamis Buck 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining 4 | a copy of this software and associated documentation files (the 5 | "Software"), to deal in the Software without restriction, including 6 | without limitation the rights to use, copy, modify, merge, publish, 7 | distribute, sublicense, and/or sell copies of the Software, and to 8 | permit persons to whom the Software is furnished to do so, subject to 9 | the following conditions: 10 | 11 | The above copyright notice and this permission notice shall be 12 | included in all copies or substantial portions of the Software. 13 | 14 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 15 | EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 16 | MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 17 | NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE 18 | LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 19 | OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION 20 | WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 21 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # SQLPP 2 | 3 | SQLPP is a simplistic SQL parser and pretty-printer. 4 | 5 | ## Usage 6 | 7 | ```ruby 8 | require 'sqlpp' 9 | 10 | sql = "..." 11 | ast = SQLPP::Parser.parse(sql) 12 | 13 | puts SQLPP::Formatter.new.format(ast) 14 | 15 | # or, to wrap projection lists... 16 | puts SQLPP::Formatter.new(projections: :wrap).format(ast) 17 | ``` 18 | 19 | Or, you can use the included `bin/sqlpp` script to format SQL via STDIN: 20 | 21 | ```sh 22 | $ sqlpp -h 23 | SQLPP (SQL Pretty Printer) 24 | 25 | Usage: sqlpp -h -? -wp < SQL 26 | 27 | -h or -?: this list of options 28 | -wp: wrap the projection lists 29 | 30 | $ sqlpp -wp < query.sql 31 | ... 32 | ``` 33 | 34 | ## Output 35 | 36 | The formatter is not particularly sophisticated, and is optimized primarily for displaying queries with deeply nested subselects. The major query components (`FROM`, `WHERE`, `GROUP BY`, and `ORDER BY`) are printed on separate lines, with subselects indented. 37 | 38 | ```sql 39 | SELECT a, b, sum(c) 40 | FROM ( 41 | SELECT d, e, f 42 | FROM ( 43 | SELECT g, h, i 44 | FROM table 45 | WHERE id IN (1, 2, 3) 46 | ) a 47 | WHERE a.e = 5 48 | OR a.e = 7 49 | ) b 50 | WHERE b.c > 5 51 | GROUP BY a, b 52 | ORDER BY a ASC, b DESC 53 | ``` 54 | 55 | ## Caveats 56 | 57 | This implementation is far, far, far from complete. It currently accepts only `SELECT` statements, and even then will only recognize a subset of the valid SQL syntax. That said, it should be a pretty big subset. It's done well enough for what I've needed it for. 58 | 59 | If, however, you find that it doesn't recognize some syntax that you need, pull requests would be appreciated! 60 | 61 | ## License 62 | 63 | MIT. See `MIT-LICENSE`. 64 | 65 | ## Author 66 | 67 | Jamis Buck (jamis@jamisbuck.org) 68 | -------------------------------------------------------------------------------- /test/formatter_test.rb: -------------------------------------------------------------------------------- 1 | require 'test_helper' 2 | 3 | class FormatterTest < Minitest::Test 4 | def test_format_select 5 | ast = _parser("select a, b, c from table where x > 5 and z between 1 and 2 or (y IS NULL) group by a, b order by z ASC").parse 6 | 7 | assert_equal <<-SQL, _format(ast) 8 | SELECT a, b, c 9 | FROM table 10 | WHERE x > 5 11 | AND z BETWEEN 1 AND 2 12 | OR (y IS NULL) 13 | GROUP BY a, b 14 | ORDER BY z ASC 15 | SQL 16 | end 17 | 18 | def test_format_subselect 19 | ast = _parser("select a, b, c from (select d,e,f from table where table.id in (1,2,3)) subselect where x > 5 group by a, b order by z ASC").parse 20 | 21 | assert_equal <<-SQL, _format(ast) 22 | SELECT a, b, c 23 | FROM ( 24 | SELECT d, e, f 25 | FROM table 26 | WHERE table.id IN (1, 2, 3) 27 | ) subselect 28 | WHERE x > 5 29 | GROUP BY a, b 30 | ORDER BY z ASC 31 | SQL 32 | end 33 | 34 | def test_format_wrap_projections 35 | ast = _parser("select a, b, c, d from foo").parse 36 | assert_equal <<-SQL, _format(ast, projections: :wrap) 37 | SELECT a, 38 | b, 39 | c, 40 | d 41 | FROM foo 42 | SQL 43 | end 44 | 45 | def test_format_select_distinct 46 | ast = _parser("select distinct a, b, c, d from foo").parse 47 | assert_equal <<-SQL, _format(ast) 48 | SELECT DISTINCT a, b, c, d 49 | FROM foo 50 | SQL 51 | end 52 | 53 | def test_format_count_distinct 54 | ast = _parser("select count(distinct id) from foo").parse 55 | assert_equal <<-SQL, _format(ast) 56 | SELECT count(DISTINCT id) 57 | FROM foo 58 | SQL 59 | end 60 | 61 | def test_format_limit 62 | ast = _parser("select * from foo limit 5").parse 63 | assert_equal <<-SQL, _format(ast) 64 | SELECT * 65 | FROM foo 66 | LIMIT 5 67 | SQL 68 | end 69 | 70 | def test_format_offset 71 | ast = _parser("select * from foo offset 5").parse 72 | assert_equal <<-SQL, _format(ast) 73 | SELECT * 74 | FROM foo 75 | OFFSET 5 76 | SQL 77 | end 78 | 79 | def test_format_limit_and_offset 80 | ast = _parser("select * from foo limit 10 offset 5").parse 81 | assert_equal <<-SQL, _format(ast) 82 | SELECT * 83 | FROM foo 84 | LIMIT 10 OFFSET 5 85 | SQL 86 | end 87 | 88 | def test_format_NOT_expr 89 | ast = _parser("select * from foo where x not in (5, 6)").parse 90 | assert_equal <<-SQL, _format(ast) 91 | SELECT * 92 | FROM foo 93 | WHERE x NOT IN (5, 6) 94 | SQL 95 | end 96 | 97 | def test_format_subscript_in_expr 98 | ast = _parser("select (array_agg(x))[0] from foo").parse 99 | assert_equal <<-SQL, _format(ast) 100 | SELECT (array_agg(x))[0] 101 | FROM foo 102 | SQL 103 | end 104 | 105 | def test_format_double_colon_typecast 106 | ast = _parser("select 'month'::interval from foo").parse 107 | assert_equal <<-SQL, _format(ast) 108 | SELECT 'month'::interval 109 | FROM foo 110 | SQL 111 | end 112 | 113 | def _parser(string) 114 | SQLPP::Parser.new(string) 115 | end 116 | 117 | def _format(ast, options={}) 118 | SQLPP::Formatter.new(options).format(ast) 119 | end 120 | end 121 | -------------------------------------------------------------------------------- /lib/sqlpp/tokenizer.rb: -------------------------------------------------------------------------------- 1 | require 'strscan' 2 | 3 | module SQLPP 4 | class Tokenizer 5 | class Exception < SQLPP::Exception; end 6 | class UnexpectedCharacter < Exception; end 7 | class EOFError < Exception; end 8 | 9 | class Token < Struct.new(:type, :text, :pos) 10 | end 11 | 12 | KEYWORDS = %w( 13 | and 14 | as 15 | asc 16 | between 17 | by 18 | case 19 | cross 20 | desc 21 | distinct 22 | else 23 | end 24 | first 25 | from 26 | full 27 | group 28 | having 29 | ilike 30 | in 31 | inner 32 | is 33 | join 34 | last 35 | left 36 | like 37 | limit 38 | not 39 | null 40 | nulls 41 | offset 42 | on 43 | or 44 | order 45 | outer 46 | right 47 | select 48 | then 49 | when 50 | where 51 | ) 52 | 53 | KEYWORDS_REGEX = Regexp.new('\b(' + KEYWORDS.join('|') + ')\b', Regexp::IGNORECASE) 54 | 55 | def initialize(string) 56 | @scanner = StringScanner.new(string) 57 | @buffer = [] 58 | end 59 | 60 | def next 61 | if @buffer.any? 62 | @buffer.pop 63 | else 64 | _scan 65 | end 66 | end 67 | 68 | def peek 69 | push(self.next) 70 | end 71 | 72 | def push(token) 73 | @buffer.push(token) 74 | token 75 | end 76 | 77 | def _scan 78 | pos = @scanner.pos 79 | 80 | if @scanner.eos? 81 | Token.new(:eof, nil, pos) 82 | elsif (key = @scanner.scan(KEYWORDS_REGEX)) 83 | Token.new(:key, key.downcase.to_sym, pos) 84 | elsif (num = @scanner.scan(/\d+(?:\.\d+)?/)) 85 | Token.new(:lit, num, pos) 86 | elsif (id = @scanner.scan(/\w+/)) 87 | Token.new(:id, id, pos) 88 | elsif (punct = @scanner.scan(/<=|<>|!=|>=|::/)) 89 | Token.new(:punct, punct, pos) 90 | elsif (punct = @scanner.scan(/[<>=\(\).*,\/+\-\[\]]/)) 91 | Token.new(:punct, punct, pos) 92 | elsif (delim = @scanner.scan(/["`]/)) 93 | contents = _scan_to_delim(delim, pos) 94 | Token.new(:id, "#{delim}#{contents}#{delim}", pos) 95 | elsif @scanner.scan(/'/) 96 | contents = _scan_to_delim("'", pos) 97 | Token.new(:lit, "'#{contents}'", pos) 98 | elsif (space = @scanner.scan(/\s+/)) 99 | Token.new(:space, space, pos) 100 | else 101 | raise UnexpectedCharacter, @scanner.rest 102 | end 103 | end 104 | 105 | def _scan_to_delim(delim, pos) 106 | escape, if_peek = case delim 107 | when '"', '`' then ["\\", nil] 108 | when "'" then ["'", "'"] 109 | end 110 | 111 | string = "" 112 | loop do 113 | ch = @scanner.getch 114 | 115 | if ch == escape && (if_peek.nil? || @scanner.peek(1) == if_peek) 116 | ch << @scanner.getch 117 | end 118 | 119 | case ch 120 | when nil then 121 | raise EOFError, "end of input reached in string started at #{pos} with #{delim.inspect}" 122 | when delim then 123 | return string 124 | else 125 | string << ch 126 | end 127 | end 128 | end 129 | end 130 | end 131 | -------------------------------------------------------------------------------- /test/tokenizer_test.rb: -------------------------------------------------------------------------------- 1 | require 'test_helper' 2 | 3 | class TokenizerTest < Minitest::Test 4 | def setup 5 | @tokenizer = nil 6 | end 7 | 8 | def test_next_should_return_next_token_in_stream 9 | _setup_tokenizer("select * from table") 10 | 11 | assert_equal :key, tokenizer.next.type 12 | assert_equal :space, tokenizer.next.type 13 | assert_equal :punct, tokenizer.next.type 14 | assert_equal :space, tokenizer.next.type 15 | assert_equal :key, tokenizer.next.type 16 | assert_equal :space, tokenizer.next.type 17 | assert_equal :id, tokenizer.next.type 18 | end 19 | 20 | def test_peek_should_not_advance_token_pointer 21 | _setup_tokenizer("select * from table") 22 | 23 | assert_equal :key, tokenizer.peek.type 24 | assert_equal :key, tokenizer.peek.type 25 | end 26 | 27 | def test_push_should_put_token_into_stream 28 | _setup_tokenizer("select * from table") 29 | 30 | tok = tokenizer.next 31 | tokenizer.push tok 32 | 33 | assert_equal :key, tokenizer.next.type 34 | end 35 | 36 | def test_it_should_recognize_keywords 37 | SQLPP::Tokenizer::KEYWORDS.each do |word| 38 | tok = _setup_tokenizer(word).next 39 | assert_token tok, type: :key, text: word.downcase.to_sym 40 | end 41 | end 42 | 43 | def test_it_should_recognize_identifiers 44 | _setup_tokenizer "word word123 \"quoted word\" \"with \\\"escape\\\" word\" `mysql word` `mysql \\`escape\\` word`" 45 | 46 | assert_token _next, type: :id, text: "word" 47 | _skip :space 48 | assert_token _next, type: :id, text: "word123" 49 | _skip :space 50 | assert_token _next, type: :id, text: '"quoted word"' 51 | _skip :space 52 | assert_token _next, type: :id, text: '"with \"escape\" word"' 53 | _skip :space 54 | assert_token _next, type: :id, text: '`mysql word`' 55 | _skip :space 56 | assert_token _next, type: :id, text: '`mysql \`escape\` word`' 57 | end 58 | 59 | def test_it_should_recognize_number_literals 60 | _setup_tokenizer "1 123 0.5 123.456" 61 | 62 | assert_token _next, type: :lit, text: "1"; _skip :space 63 | assert_token _next, type: :lit, text: "123"; _skip :space 64 | assert_token _next, type: :lit, text: "0.5"; _skip :space 65 | assert_token _next, type: :lit, text: "123.456" 66 | end 67 | 68 | def test_it_should_recognize_string_literals 69 | _setup_tokenizer "'hello' 'quoted ''string'' here'" 70 | 71 | assert_token _next, type: :lit, text: "'hello'"; _skip :space 72 | assert_token _next, type: :lit, text: "'quoted ''string'' here'" 73 | end 74 | 75 | def test_it_should_recognize_whitespace 76 | _setup_tokenizer " space\n " 77 | 78 | assert_token _next, type: :space, text: " "; _skip :id 79 | assert_token _next, type: :space, text: "\n " 80 | end 81 | 82 | def test_it_should_recognize_multichar_punctuation 83 | _setup_tokenizer "<= <> != >=" 84 | 85 | assert_token _next, type: :punct, text: "<="; _skip :space 86 | assert_token _next, type: :punct, text: "<>"; _skip :space 87 | assert_token _next, type: :punct, text: "!="; _skip :space 88 | assert_token _next, type: :punct, text: ">=" 89 | end 90 | 91 | def test_it_should_recognize_punctuation 92 | _setup_tokenizer "< > = ( ) . * , / + -" 93 | 94 | assert_token _next, type: :punct, text: "<"; _skip :space 95 | assert_token _next, type: :punct, text: ">"; _skip :space 96 | assert_token _next, type: :punct, text: "="; _skip :space 97 | assert_token _next, type: :punct, text: "("; _skip :space 98 | assert_token _next, type: :punct, text: ")"; _skip :space 99 | assert_token _next, type: :punct, text: "."; _skip :space 100 | assert_token _next, type: :punct, text: "*"; _skip :space 101 | assert_token _next, type: :punct, text: ","; _skip :space 102 | assert_token _next, type: :punct, text: "/"; _skip :space 103 | assert_token _next, type: :punct, text: "+"; _skip :space 104 | assert_token _next, type: :punct, text: "-" 105 | end 106 | 107 | def test_it_should_recognize_end_of_file 108 | _setup_tokenizer "done" 109 | 110 | _skip :id 111 | assert_token _next, type: :eof 112 | assert_token _next, type: :eof 113 | end 114 | 115 | attr_reader :tokenizer 116 | 117 | def _setup_tokenizer(string) 118 | @tokenizer = SQLPP::Tokenizer.new(string) 119 | end 120 | 121 | def _next 122 | tokenizer.next 123 | end 124 | 125 | def _skip(type) 126 | assert_token _next, type: type 127 | end 128 | 129 | def assert_token(token, type: nil, text: nil, pos: nil) 130 | assert_equal type, token.type if type 131 | assert_equal text, token.text if text 132 | assert_equal pos, token.pos if pos 133 | end 134 | end 135 | -------------------------------------------------------------------------------- /lib/sqlpp/formatter.rb: -------------------------------------------------------------------------------- 1 | module SQLPP 2 | class Formatter 3 | def initialize(projections: nil) 4 | @indent = nil 5 | @state = nil 6 | 7 | @projections = projections 8 | end 9 | 10 | def format(node) 11 | name = node.class.to_s.split(/::/).last 12 | send(:"_format_#{name}", node) 13 | end 14 | 15 | def _format_Select(node) 16 | output = "" 17 | 18 | if @indent.nil? 19 | @indent = 0 20 | else 21 | @indent += 2 22 | output << "\n" 23 | end 24 | 25 | output << (select = "#{_indent}SELECT ") 26 | output << "DISTINCT " if node.distinct 27 | link = "," 28 | link << ((@projections == :wrap) ? "\n#{" " * select.length}" : " ") 29 | output << node.projections.map { |c| format(c) }.join(link) 30 | output << "\n" 31 | 32 | if node.froms 33 | output << "#{_indent}FROM " 34 | output << node.froms.map { |c| format(c) }.join(", ") 35 | output << "\n" 36 | end 37 | 38 | if node.wheres 39 | save, @state = @state, :where 40 | output << "#{_indent}WHERE " 41 | output << format(node.wheres) 42 | output << "\n" 43 | @state = save 44 | end 45 | 46 | if node.groups 47 | output << "#{_indent}GROUP BY " 48 | output << node.groups.map { |c| format(c) }.join(", ") 49 | output << "\n" 50 | end 51 | 52 | if node.orders 53 | output << "#{_indent}ORDER BY " 54 | output << node.orders.map { |c| format(c) }.join(", ") 55 | output << "\n" 56 | end 57 | 58 | if node.limit || node.offset 59 | output << _indent 60 | 61 | if node.limit 62 | output << format(node.limit) 63 | output << " " if node.offset 64 | end 65 | 66 | output << format(node.offset) if node.offset 67 | 68 | output << "\n" 69 | end 70 | 71 | @indent -= 2 72 | @indent = nil if @indent < 0 73 | 74 | output << _indent 75 | end 76 | 77 | def _format_Expr(node) 78 | output = format(node.left) 79 | if node.op 80 | op = node.op.to_s.upcase 81 | 82 | if @state == :where && %w(AND OR).include?(op) 83 | output << "\n#{_indent}" 84 | else 85 | output << " " 86 | end 87 | 88 | output << "NOT " if node.not 89 | output << op << " " 90 | output << format(node.right) 91 | end 92 | output 93 | end 94 | 95 | def _format_Unary(node) 96 | op = node.op.to_s.upcase 97 | output = op 98 | output << " " if op =~ /\w/ 99 | output << format(node.expr) 100 | end 101 | 102 | def _format_Atom(node) 103 | output = "" 104 | 105 | case node.type 106 | when :range 107 | output << format(node.left) << " AND " << format(node.right) 108 | when :list 109 | output << "(" << node.left.map { |c| format(c) }.join(", ") << ")" 110 | when :func 111 | output << format(node.left) << "(" 112 | output << node.right.map { |c| format(c) }.join(", ") 113 | output << ")" 114 | when :lit 115 | output << node.left 116 | when :attr 117 | output << node.left 118 | output << "." << node.right if node.right 119 | when :case 120 | output << "CASE " 121 | output << format(node.left) << " " if node.left 122 | node.right.each do |child| 123 | if child.is_a?(Array) 124 | output << "WHEN " << format(child[0]) << " " 125 | output << "THEN " << format(child[1]) << " " 126 | else 127 | output << "ELSE " << format(child) << " " 128 | end 129 | end 130 | output << "END" 131 | else 132 | raise ArgumentError, "unknown atom type #{node.type.inspect}" 133 | end 134 | 135 | output 136 | end 137 | 138 | def _format_Parens(node) 139 | "(" + format(node.value) + ")" 140 | end 141 | 142 | def _format_As(node) 143 | format(node.expr) + " AS " + format(node.name) 144 | end 145 | 146 | def _format_Alias(node) 147 | format(node.expr) + " " + format(node.name) 148 | end 149 | 150 | def _format_Join(node) 151 | output = "" 152 | 153 | output << format(node.left) 154 | output << "\n#{_indent}" 155 | output << node.type.upcase << " JOIN " 156 | output << format(node.right) 157 | output << "\n#{_indent}ON " << format(node.on) if node.on 158 | 159 | output 160 | end 161 | 162 | def _format_SortKey(node) 163 | output = "" 164 | output << format(node.key) 165 | 166 | if node.options.any? 167 | output << " " 168 | output << node.options.map { |opt| opt.upcase }.join(" ") 169 | end 170 | 171 | output 172 | end 173 | 174 | def _format_Limit(node) 175 | "LIMIT #{format(node.expr)}" 176 | end 177 | 178 | def _format_Offset(node) 179 | "OFFSET #{format(node.expr)}" 180 | end 181 | 182 | def _format_String(string) 183 | string 184 | end 185 | 186 | def _format_Subscript(node) 187 | output = "" 188 | output << format(node.left) 189 | output << "[" << format(node.right) << "]" 190 | output 191 | end 192 | 193 | def _format_TypeCast(node) 194 | format(node.value) << '::' << format(node.type) 195 | end 196 | 197 | def _indent 198 | " " * (@indent || 0) 199 | end 200 | end 201 | end 202 | -------------------------------------------------------------------------------- /test/parser_test.rb: -------------------------------------------------------------------------------- 1 | require 'test_helper' 2 | 3 | class ParserTest < Minitest::Test 4 | def test_it_should_parse_number_as_expression 5 | expr = _parser("5").parse_expression 6 | assert_instance_of SQLPP::AST::Atom, expr 7 | assert_equal :lit, expr.type 8 | assert_equal "5", expr.left 9 | end 10 | 11 | def test_it_should_parse_string_as_expression 12 | expr = _parser("'hello'").parse_expression 13 | assert_instance_of SQLPP::AST::Atom, expr 14 | assert_equal :lit, expr.type 15 | assert_equal "'hello'", expr.left 16 | end 17 | 18 | def test_it_should_parse_id_as_expression 19 | expr = _parser("hello").parse_expression 20 | assert_instance_of SQLPP::AST::Atom, expr 21 | assert_equal :attr, expr.type 22 | assert_equal "hello", expr.left 23 | end 24 | 25 | def test_it_should_parse_star_as_expression 26 | expr = _parser("*").parse_expression 27 | assert_instance_of SQLPP::AST::Atom, expr 28 | assert_equal :lit, expr.type 29 | assert_equal "*", expr.left 30 | end 31 | 32 | def test_it_should_parse_function_as_expression 33 | expr = _parser("sum(donuts)").parse_expression 34 | assert_instance_of SQLPP::AST::Atom, expr 35 | assert_equal :func, expr.type 36 | assert_equal "sum", expr.left 37 | assert_equal 1, expr.right.count 38 | 39 | assert_instance_of SQLPP::AST::Atom, expr.right[0] 40 | assert_equal :attr, expr.right[0].type 41 | assert_equal "donuts", expr.right[0].left 42 | end 43 | 44 | def test_it_should_parse_parenthesized_expression 45 | expr = _parser("(donuts)").parse_expression 46 | assert_instance_of SQLPP::AST::Parens, expr 47 | assert_instance_of SQLPP::AST::Atom, expr.value 48 | assert_equal :attr, expr.value.type 49 | assert_equal "donuts", expr.value.left 50 | end 51 | 52 | def test_it_should_parse_qualified_references 53 | expr = _parser("foo.bar").parse_expression 54 | assert_instance_of SQLPP::AST::Atom, expr 55 | assert_equal :attr, expr.type 56 | assert_equal "foo", expr.left 57 | assert_equal "bar", expr.right 58 | end 59 | 60 | def test_it_should_treat_null_as_literal 61 | expr = _parser("NULL").parse_expression 62 | assert_instance_of SQLPP::AST::Atom, expr 63 | assert_equal :lit, expr.type 64 | assert_equal "NULL", expr.left 65 | end 66 | 67 | def test_it_should_parse_case_with_root_expression 68 | expr = _parser("case x when 5 then 1 end").parse_expression 69 | assert_instance_of SQLPP::AST::Atom, expr 70 | assert_equal :case, expr.type 71 | assert_equal "x", expr.left.left 72 | assert_equal 1, expr.right.length 73 | assert_equal 2, expr.right[0].length 74 | assert_equal "5", expr.right[0][0].left 75 | assert_equal "1", expr.right[0][1].left 76 | end 77 | 78 | def test_it_should_parse_case_without_root_expression 79 | expr = _parser("case when 5 then 1 end").parse_expression 80 | assert_instance_of SQLPP::AST::Atom, expr 81 | assert_equal :case, expr.type 82 | assert_nil expr.left 83 | assert_equal 1, expr.right.length 84 | assert_equal 2, expr.right[0].length 85 | assert_equal "5", expr.right[0][0].left 86 | assert_equal "1", expr.right[0][1].left 87 | end 88 | 89 | def test_it_should_parse_case_with_else_expression 90 | expr = _parser("case when 5 then 1 else 3 end").parse_expression 91 | assert_instance_of SQLPP::AST::Atom, expr 92 | assert_equal :case, expr.type 93 | assert_nil expr.left 94 | assert_equal 2, expr.right.length 95 | assert_equal 2, expr.right[0].length 96 | assert_equal "5", expr.right[0][0].left 97 | assert_equal "1", expr.right[0][1].left 98 | assert_equal "3", expr.right[1].left 99 | end 100 | 101 | def test_it_should_parse_unary_plus 102 | expr = _parser("+x").parse_expression 103 | assert_instance_of SQLPP::AST::Unary, expr 104 | assert_equal "+", expr.op 105 | assert_equal "x", expr.expr.left 106 | end 107 | 108 | def test_it_should_parse_unary_minus 109 | expr = _parser("-x").parse_expression 110 | assert_instance_of SQLPP::AST::Unary, expr 111 | assert_equal "-", expr.op 112 | assert_equal "x", expr.expr.left 113 | end 114 | 115 | def test_it_should_parse_arithmatic 116 | %w(+ - * /).each do |op| 117 | expr = _parser("x #{op} y").parse_expression 118 | assert_instance_of SQLPP::AST::Expr, expr 119 | assert_instance_of SQLPP::AST::Atom, expr.left 120 | assert_equal op, expr.op 121 | assert_instance_of SQLPP::AST::Atom, expr.right 122 | end 123 | end 124 | 125 | def test_it_should_parse_between 126 | expr = _parser("x between y and z").parse_expression 127 | assert_instance_of SQLPP::AST::Expr, expr 128 | assert_instance_of SQLPP::AST::Atom, expr.left 129 | assert_equal :between, expr.op 130 | assert_instance_of SQLPP::AST::Atom, expr.right 131 | assert_equal :range, expr.right.type 132 | assert_instance_of SQLPP::AST::Atom, expr.right.left 133 | assert_instance_of SQLPP::AST::Atom, expr.right.right 134 | end 135 | 136 | def test_it_should_parse_like 137 | expr = _parser("x like y").parse_expression 138 | assert_instance_of SQLPP::AST::Expr, expr 139 | assert_instance_of SQLPP::AST::Atom, expr.left 140 | assert_equal :like, expr.op 141 | assert_instance_of SQLPP::AST::Atom, expr.right 142 | end 143 | 144 | def test_it_should_parse_ilike 145 | expr = _parser("x ilike y").parse_expression 146 | assert_instance_of SQLPP::AST::Expr, expr 147 | assert_instance_of SQLPP::AST::Atom, expr.left 148 | assert_equal :ilike, expr.op 149 | assert_instance_of SQLPP::AST::Atom, expr.right 150 | end 151 | 152 | def test_it_should_parse_in 153 | expr = _parser("x in (1,2,3,4,5)").parse_expression 154 | assert_instance_of SQLPP::AST::Expr, expr 155 | assert_instance_of SQLPP::AST::Atom, expr.left 156 | assert_equal :in, expr.op 157 | assert_instance_of SQLPP::AST::Atom, expr.right 158 | assert_equal :list, expr.right.type 159 | assert_equal 5, expr.right.left.length 160 | end 161 | 162 | def test_it_should_parse_boolean_operations 163 | %w(< <= <> != = >= >).each do |op| 164 | expr = _parser("x #{op} y").parse_expression 165 | assert_instance_of SQLPP::AST::Expr, expr 166 | assert_instance_of SQLPP::AST::Atom, expr.left 167 | assert_equal op, expr.op 168 | assert_instance_of SQLPP::AST::Atom, expr.right 169 | end 170 | end 171 | 172 | def test_it_should_parse_expr_operations 173 | [:is, "is not", :and, :or].each do |op| 174 | expr = _parser("x #{op} y").parse_expression 175 | assert_instance_of SQLPP::AST::Expr, expr 176 | assert_instance_of SQLPP::AST::Atom, expr.left 177 | assert_equal op, expr.op 178 | assert_instance_of SQLPP::AST::Atom, expr.right 179 | end 180 | end 181 | 182 | def test_it_should_parse_distinct_expression 183 | expr = _parser("count(distinct id)").parse_expression 184 | assert_instance_of SQLPP::AST::Atom, expr 185 | assert_equal :func, expr.type 186 | assert_equal 1, expr.right.count 187 | 188 | assert_instance_of SQLPP::AST::Unary, expr.right[0] 189 | assert_equal :distinct, expr.right[0].op 190 | assert_instance_of SQLPP::AST::Atom, expr.right[0].expr 191 | assert_equal :attr, expr.right[0].expr.type 192 | assert_equal "id", expr.right[0].expr.left 193 | end 194 | 195 | def test_from_should_recognize_single_attr 196 | from = _parser("x").parse_from 197 | assert_instance_of SQLPP::AST::Atom, from 198 | end 199 | 200 | def test_from_should_recognize_alias 201 | from = _parser("x y").parse_from 202 | assert_instance_of SQLPP::AST::Alias, from 203 | assert_equal "y", from.name 204 | assert_instance_of SQLPP::AST::Atom, from.expr 205 | end 206 | 207 | def test_from_should_recognize_as 208 | from = _parser("x AS y").parse_from 209 | assert_instance_of SQLPP::AST::As, from 210 | assert_equal "y", from.name 211 | assert_instance_of SQLPP::AST::Atom, from.expr 212 | end 213 | 214 | def test_from_may_be_parenthesized 215 | from = _parser("( x ) AS y").parse_from 216 | assert_instance_of SQLPP::AST::As, from 217 | assert_equal "y", from.name 218 | assert_instance_of SQLPP::AST::Parens, from.expr 219 | assert_instance_of SQLPP::AST::Atom, from.expr.value 220 | end 221 | 222 | def test_from_may_be_a_subselect 223 | from = _parser("(select * from x) as y").parse_from 224 | assert_instance_of SQLPP::AST::As, from 225 | assert_instance_of SQLPP::AST::Parens, from.expr 226 | assert_instance_of SQLPP::AST::Select, from.expr.value 227 | end 228 | 229 | def test_from_may_join_to_another_entity 230 | [ "inner", 231 | "left outer", 232 | "right outer", 233 | "full outer", 234 | "cross" 235 | ].each do |type| 236 | from = _parser("x #{type} join y AS z ON x.id = z.id").parse_from 237 | assert_instance_of SQLPP::AST::Join, from 238 | assert_instance_of SQLPP::AST::Atom, from.left 239 | assert_equal type, from.type 240 | assert_instance_of SQLPP::AST::As, from.right 241 | assert_instance_of SQLPP::AST::Expr, from.on 242 | end 243 | end 244 | 245 | def test_chained_join_will_build_tree 246 | from = _parser("x inner join y on x.id = y.id inner join z on x.id = z.id").parse_from 247 | assert_instance_of SQLPP::AST::Join, from 248 | assert_instance_of SQLPP::AST::Join, from.left 249 | end 250 | 251 | def test_accepts_select_with_projections 252 | s = _parser("select x, y").parse_select 253 | assert_instance_of SQLPP::AST::Select, s 254 | assert_equal 2, s.projections.length 255 | end 256 | 257 | def test_accepts_select_with_froms 258 | s = _parser("select from x, y").parse_select 259 | assert_instance_of SQLPP::AST::Select, s 260 | assert_equal 2, s.froms.length 261 | end 262 | 263 | def test_accepts_select_with_where 264 | s = _parser("select from x where x > 5 and z < 2").parse_select 265 | assert_instance_of SQLPP::AST::Select, s 266 | assert_instance_of SQLPP::AST::Expr, s.wheres 267 | end 268 | 269 | def test_accepts_select_with_group_by 270 | s = _parser("select * from x group by a, b").parse_select 271 | assert_instance_of SQLPP::AST::Select, s 272 | assert_equal 2, s.groups.length 273 | end 274 | 275 | def test_accepts_select_with_order_by 276 | s = _parser("select * from x order by a ASC, b DESC NULLS LAST").parse_select 277 | assert_instance_of SQLPP::AST::Select, s 278 | assert_equal 2, s.orders.length 279 | assert_instance_of SQLPP::AST::SortKey, s.orders[0] 280 | assert_equal [:asc], s.orders[0].options 281 | assert_instance_of SQLPP::AST::SortKey, s.orders[1] 282 | assert_equal [:desc, "nulls last"], s.orders[1].options 283 | end 284 | 285 | def test_accepts_select_distinct 286 | s = _parser("select distinct * from x").parse_select 287 | assert_instance_of SQLPP::AST::Select, s 288 | assert_equal s.distinct, true 289 | end 290 | 291 | def test_parse_should_recognize_select 292 | s = _parser("select * from x").parse 293 | assert_instance_of SQLPP::AST::Select, s 294 | end 295 | 296 | def test_accepts_select_with_limit 297 | s = _parser("select * from x limit 5").parse_select 298 | assert_instance_of SQLPP::AST::Limit, s.limit 299 | assert_instance_of SQLPP::AST::Atom, s.limit.expr 300 | assert_equal :lit, s.limit.expr.type 301 | assert_equal "5", s.limit.expr.left 302 | end 303 | 304 | def test_accepts_select_with_offset 305 | s = _parser("select * from x offset 5").parse_select 306 | assert_instance_of SQLPP::AST::Offset, s.offset 307 | assert_instance_of SQLPP::AST::Atom, s.offset.expr 308 | assert_equal :lit, s.offset.expr.type 309 | assert_equal "5", s.offset.expr.left 310 | end 311 | 312 | def test_accepts_not_in_syntax 313 | s = _parser("select * from x where x.id not in (5)").parse_select 314 | assert s.wheres.not 315 | end 316 | 317 | def test_accepts_array_subscript_in_expression 318 | expr = _parser("a[5]").parse_expression 319 | assert_instance_of SQLPP::AST::Subscript, expr 320 | assert_instance_of SQLPP::AST::Atom, expr.left 321 | assert_instance_of SQLPP::AST::Atom, expr.right 322 | assert_equal :attr, expr.left.type 323 | assert_equal "a", expr.left.left 324 | assert_equal :lit, expr.right.type 325 | assert_equal "5", expr.right.left 326 | end 327 | 328 | def test_accepts_double_colon_as_typecast 329 | expr = _parser("'month'::INTERVAL").parse_expression 330 | assert_instance_of SQLPP::AST::TypeCast, expr 331 | assert_instance_of SQLPP::AST::Atom, expr.value 332 | assert_instance_of SQLPP::AST::Atom, expr.type 333 | assert_equal :lit, expr.value.type 334 | assert_equal "'month'", expr.value.left 335 | assert_equal :attr, expr.type.type 336 | assert_equal "INTERVAL", expr.type.left 337 | end 338 | 339 | def test_accept_empty_parameter_list 340 | expr = _parser("NOW()").parse_expression 341 | assert_instance_of SQLPP::AST::Atom, expr 342 | assert_equal :func, expr.type 343 | assert_equal "NOW", expr.left 344 | assert_equal 0, expr.right.count 345 | end 346 | 347 | def _parser(string) 348 | SQLPP::Parser.new(string) 349 | end 350 | end 351 | -------------------------------------------------------------------------------- /lib/sqlpp/parser.rb: -------------------------------------------------------------------------------- 1 | require 'sqlpp/ast' 2 | 3 | # select := 'SELECT' optional_distinct 4 | # optional_projections 5 | # optional_froms 6 | # optional_wheres 7 | # optional_groups 8 | # optional_orders 9 | # optional_limit 10 | # optional_offset 11 | # 12 | # optional_distinct := '' 13 | # | 'DISTINCT' 14 | # 15 | # optional_projections := '' 16 | # | list 17 | # 18 | # optional_froms := '' 19 | # | 'FROM' froms 20 | # 21 | # optional_wheres := '' 22 | # | 'WHERE' expr1 23 | # 24 | # optional_groups := '' 25 | # | 'GROUP' 'BY' list 26 | # 27 | # optional_orders := '' 28 | # | 'ORDER' 'BY' sort_keys 29 | # 30 | # optional_limit := '' 31 | # | 'LIMIT' expr4 32 | # 33 | # optional_offset := '' 34 | # | 'OFFSET' expr4 35 | # 36 | # sort_keys := sort_key 37 | # | sort_key ',' sort_keys 38 | # 39 | # sort_key := expr1 40 | # | expr1 sort_options 41 | # 42 | # sort_options := sort_option 43 | # | sort_option ' ' sort_options 44 | # 45 | # sort_option := 'ASC' | 'DESC' | 'NULLS FIRST' | 'NULLS LAST' 46 | # 47 | # froms := from 48 | # | from ',' froms 49 | # 50 | # from := entity 51 | # | entity optional_join_expr 52 | # 53 | # optional_join_expr := '' 54 | # | 'LEFT' 'JOIN' from 'ON' expr 55 | # | 'INNER' 'JOIN' from 'ON' expr 56 | # | 'OUTER' 'JOIN' from 'ON' expr 57 | # | 'FULL' 'OUTER' 'JOIN' from 'ON' expr 58 | # 59 | # entity := '(' from ')' 60 | # | id 61 | # | select_stmt 62 | # 63 | # expr1 := expr2 64 | # | expr2 op expr1 65 | # 66 | # op := 'AND' | 'OR' | 'IS' | 'IS NOT' 67 | # 68 | # expr2 := expr3 optional_op 69 | # 70 | # optional_op := '' 71 | # | 'NOT' optional_op 72 | # | 'BETWEEN' expr3 AND expr3 73 | # | 'NOT IN' '(' list ')' 74 | # | 'IN' '(' list ')' 75 | # | bop expr3 76 | # 77 | # bop := '<' | '<=' | '<>' | '=' | '>=' | '>' 78 | # 79 | # expr3 := expr4 80 | # | expr4 op2 expr3 81 | # | unary expr3 82 | # 83 | # op2 := '+' | '-' | '*' | '/' 84 | # 85 | # unary := '+' | '-' | 'NOT' | 'DISTINCT' 86 | # 87 | # expr4 := lit 88 | # | id 89 | # | id '.' id 90 | # | id '(' args ')' 91 | # | 'CASE' case_stmt 'END' 92 | # | '(' expr1 ')' 93 | # | expr4 '[' expr1 ']' 94 | # | expr4 '::' expr4 95 | # 96 | # list := expr1 97 | # | expr1 ',' list 98 | 99 | module SQLPP 100 | class Parser 101 | class Exception < SQLPP::Exception; end 102 | class UnexpectedToken < Exception; end 103 | class TrailingTokens < Exception; end 104 | 105 | def self.parse(string) 106 | parser = new(string) 107 | parser.parse 108 | end 109 | 110 | def initialize(string) 111 | @tokenizer = SQLPP::Tokenizer.new(string) 112 | end 113 | 114 | def parse 115 | _eat :space 116 | 117 | token = _peek(:key) 118 | raise UnexpectedToken, token.inspect unless token 119 | 120 | case token.text 121 | when :select then parse_select 122 | else raise UnexpectedToken, token.inspect 123 | end 124 | end 125 | 126 | # --- exposed for testing purposes --- 127 | 128 | def parse_expression 129 | _parse_expr1 130 | ensure 131 | _ensure_stream_empty! 132 | end 133 | 134 | def parse_from 135 | _parse_from 136 | ensure 137 | _ensure_stream_empty! 138 | end 139 | 140 | def parse_select 141 | _parse_select 142 | ensure 143 | _ensure_stream_empty! 144 | end 145 | 146 | # --- internal use --- 147 | 148 | def _parse_select 149 | _expect :key, :select 150 | select = AST::Select.new 151 | 152 | _eat :space 153 | if _eat(:key, :distinct) 154 | select.distinct = true 155 | _eat :space 156 | end 157 | 158 | if !_peek(:key, /^(from|where)$/) && !_peek(:eof) 159 | list = [] 160 | 161 | loop do 162 | expr = _parse_expr1 163 | _eat :space 164 | if _peek(:key, :as) 165 | _next 166 | _eat :space 167 | name = _expect(:id) 168 | expr = AST::As.new(name.text, expr) 169 | end 170 | list.push expr 171 | break unless _eat(:punct, ",") 172 | end 173 | _eat :space 174 | 175 | select.projections = list 176 | end 177 | 178 | if _eat(:key, :from) 179 | list = [] 180 | 181 | loop do 182 | _eat :space 183 | list << _parse_from 184 | _eat :space 185 | break unless _eat(:punct, ',') 186 | end 187 | _eat :space 188 | 189 | select.froms = list 190 | end 191 | 192 | if _eat(:key, :where) 193 | select.wheres = _parse_expr1 194 | _eat :space 195 | end 196 | 197 | if _eat(:key, :group) 198 | _eat :space 199 | _expect :key, :by 200 | _eat :space 201 | select.groups = _parse_list 202 | end 203 | 204 | if _eat(:key, :order) 205 | _eat :space 206 | _expect :key, :by 207 | _eat :space 208 | 209 | list = [] 210 | loop do 211 | key = AST::SortKey.new(_parse_expr1, []) 212 | list << key 213 | 214 | _eat :space 215 | 216 | if (dir = _eat(:key, /^(asc|desc)$/)) 217 | _eat :space 218 | key.options << dir.text 219 | end 220 | 221 | if (opt = _eat(:key, :nulls)) 222 | opt = opt.text.to_s 223 | _eat :space 224 | sort = _eat(:key, /^(first|last)$/) 225 | opt << " " << sort.text.to_s if sort 226 | key.options << opt 227 | end 228 | 229 | _eat :space 230 | break unless _eat(:punct, ",") 231 | end 232 | 233 | select.orders = list 234 | end 235 | 236 | if _eat(:key, :limit) 237 | _eat :space 238 | atom = _parse_atom 239 | _eat :space 240 | 241 | select.limit = AST::Limit.new(atom) 242 | end 243 | 244 | if _eat(:key, :offset) 245 | _eat :space 246 | atom = _parse_atom 247 | _eat :space 248 | 249 | select.offset = AST::Offset.new(atom) 250 | end 251 | 252 | select 253 | end 254 | 255 | def _parse_from 256 | entity = _parse_entity 257 | 258 | loop do 259 | _eat :space 260 | 261 | if (which = _eat(:key, /^(inner|cross|left|right|full|outer)$/)) 262 | type = which.text.to_s 263 | 264 | if type == "full" || type == "left" || type == "right" 265 | _eat :space 266 | _expect :key, :outer 267 | type << " outer" 268 | end 269 | 270 | _eat :space 271 | _expect :key, :join 272 | 273 | entity = AST::Join.new(type.downcase, entity, _parse_from) 274 | 275 | _eat :space 276 | if _eat(:key, :on) 277 | _eat :space 278 | entity.on = _parse_expr1 279 | end 280 | 281 | else 282 | break 283 | end 284 | end 285 | 286 | entity 287 | end 288 | 289 | def _parse_entity 290 | _eat :space 291 | 292 | entity = if _eat(:punct, '(') 293 | from = _parse_from 294 | _eat :space 295 | _expect :punct, ')' 296 | AST::Parens.new(from) 297 | 298 | elsif _peek(:key, :select) 299 | _parse_select 300 | 301 | else 302 | id = _expect(:id) 303 | AST::Atom.new(:attr, id.text) 304 | end 305 | 306 | _eat :space 307 | if _eat(:key, :as) 308 | _eat :space 309 | id = _expect(:id) 310 | AST::As.new(id.text, entity) 311 | elsif (id = _eat(:id)) 312 | AST::Alias.new(id.text, entity) 313 | else 314 | entity 315 | end 316 | end 317 | 318 | def _parse_expr1 319 | _eat :space 320 | 321 | left = _parse_expr2 322 | _eat :space 323 | 324 | if (op = _eat(:key, /^(and|or|is)$/i)) 325 | op = op.text 326 | 327 | if op == :is 328 | _eat :space 329 | op2 = _eat(:key, :not) 330 | op = "#{op} #{op2.text}" if op2 331 | end 332 | 333 | right = _parse_expr1 334 | 335 | AST::Expr.new(left, op, right) 336 | else 337 | left 338 | end 339 | end 340 | 341 | def _parse_expr2 342 | _eat :space 343 | 344 | left = _parse_expr3 345 | _eat :space 346 | 347 | not_kw = _eat(:key, :not) 348 | _eat :space if not_kw 349 | 350 | if (op = _eat(:key, :between)) 351 | op = op.text 352 | 353 | _eat :space 354 | lo = _parse_expr3 355 | 356 | _eat :space 357 | _expect :key, :and 358 | 359 | _eat :space 360 | hi = _parse_expr3 361 | 362 | right = AST::Atom.new(:range, lo, hi) 363 | 364 | elsif (op = _eat(:key, :in)) 365 | op = op.text 366 | 367 | _eat :space 368 | _expect :punct, "(" 369 | 370 | right = AST::Atom.new(:list, _parse_list) 371 | _eat :space 372 | _expect :punct, ")" 373 | 374 | elsif (op = _eat(:punct, /<=|<>|>=|=|<|>/) || _eat(:key, /^i?like$/)) 375 | op = op.text 376 | right = _parse_expr3 377 | end 378 | 379 | if right 380 | AST::Expr.new(left, op, right, not_kw != nil) 381 | elsif not_kw 382 | raise UnexpectedToken, "got #{not_kw.inspect}" 383 | else 384 | left 385 | end 386 | end 387 | 388 | def _parse_expr3 389 | _eat :space 390 | 391 | if (op = (_eat(:punct, /[-+]/) || _eat(:key, /^(not|distinct)$/))) 392 | _eat :space 393 | AST::Unary.new(op.text, _parse_expr3) 394 | 395 | else 396 | atom = _parse_atom 397 | _eat :space 398 | 399 | if _eat(:punct, "[") 400 | subscript = _parse_expr1 401 | _eat :space 402 | _expect(:punct, "]") 403 | _eat :space 404 | 405 | atom = AST::Subscript.new(atom, subscript) 406 | end 407 | 408 | if _eat(:punct, "::") 409 | _eat :space 410 | type = _parse_atom 411 | _eat :space 412 | 413 | atom = AST::TypeCast.new(atom, type) 414 | end 415 | 416 | if (op = _eat(:punct, /[-+*\/]/)) 417 | _eat :space 418 | AST::Expr.new(atom, op.text, _parse_expr3) 419 | else 420 | atom 421 | end 422 | end 423 | end 424 | 425 | def _parse_atom 426 | if (lit = _eat(:lit)) 427 | AST::Atom.new(:lit, lit.text) 428 | 429 | elsif _eat(:key, :case) 430 | _parse_case 431 | 432 | elsif _eat(:punct, "(") 433 | expr = _parse_expr1 434 | _eat :space 435 | _expect(:punct, ")") 436 | AST::Parens.new(expr) 437 | 438 | elsif _eat(:key, :null) 439 | AST::Atom.new(:lit, "NULL") 440 | 441 | elsif _eat(:punct, "*") 442 | AST::Atom.new(:lit, "*") 443 | 444 | else 445 | id = _expect(:id) 446 | 447 | if _eat(:punct, "(") 448 | args = _parse_list 449 | _expect(:punct, ")") 450 | AST::Atom.new(:func, id.text, args) 451 | elsif _eat(:punct, '.') 452 | id2 = _eat(:id) || _eat(:punct, '*') 453 | 454 | if !id2 455 | raise UnexpectedToken, "expected id or *, got #{_peek.inspect}" 456 | end 457 | 458 | AST::Atom.new(:attr, id.text, id2.text) 459 | else 460 | AST::Atom.new(:attr, id.text) 461 | end 462 | end 463 | end 464 | 465 | def _parse_case 466 | _expect :space 467 | 468 | kase = AST::Atom.new(:case) 469 | unless _peek(:key, :when) 470 | kase.left = _parse_expr1 471 | _eat :space 472 | end 473 | 474 | cases = [] 475 | while _eat(:key, :when) 476 | condition = _parse_expr1 477 | _eat :space 478 | _expect :key, :then 479 | result = _parse_expr1 480 | cases << [condition, result] 481 | _eat :space 482 | end 483 | 484 | if _eat(:key, :else) 485 | cases << _parse_expr1 486 | _eat :space 487 | end 488 | 489 | _expect :key, :end 490 | 491 | kase.right = cases 492 | kase 493 | end 494 | 495 | # list := '' 496 | # | expr 497 | # | expr ',' args 498 | def _parse_list 499 | _eat :space 500 | args = [] 501 | return args if _peek(:punct, ')') 502 | 503 | loop do 504 | args << _parse_expr1 505 | 506 | _eat :space 507 | if _eat(:punct, ",") 508 | _eat :space 509 | else 510 | break 511 | end 512 | end 513 | 514 | args 515 | end 516 | 517 | def _eat(type_or_types, pattern=nil) 518 | _next if _peek(type_or_types, pattern) 519 | end 520 | 521 | def _peek(type_or_types, pattern=nil) 522 | token = _next 523 | _match(token, type_or_types, pattern) 524 | ensure 525 | @tokenizer.push(token) 526 | end 527 | 528 | def _match(token, type_or_types, pattern=nil) 529 | types = type_or_types.is_a?(Array) ? type_or_types : [ type_or_types ] 530 | 531 | if types.include?(token.type) && (pattern.nil? || pattern === token.text) 532 | token 533 | else 534 | nil 535 | end 536 | end 537 | 538 | def _expect(type_or_types, pattern=nil) 539 | token = _next 540 | 541 | if !_match(token, type_or_types, pattern) 542 | raise UnexpectedToken, "expected #{type_or_types.inspect}(#{pattern.inspect}), got #{token.inspect}" 543 | end 544 | 545 | token 546 | end 547 | 548 | def _next 549 | @tokenizer.next 550 | end 551 | 552 | def _ensure_stream_empty! 553 | unless _peek(:eof) 554 | raise TrailingTokens, _next.inspect 555 | end 556 | end 557 | 558 | end 559 | end 560 | --------------------------------------------------------------------------------