├── .travis.yml ├── parser ├── cpp │ ├── build.sh │ ├── compile.sh │ ├── prepare-javacc-grammar.sh │ ├── generate_parser.sh │ ├── parser.h │ ├── tokenize.cc │ ├── main.cc │ ├── ParseErrorHandler.cc │ ├── AstNode.h │ ├── pom.xml │ ├── TARGETS │ └── javacc-options.txt ├── grammar │ ├── prepare-javacc-grammar.sh │ ├── notes │ ├── datetimespec.txt │ ├── regexp.txt │ ├── presto-extensions.txt │ ├── javacc-options-java.txt │ ├── reservedwords.txt │ ├── lexical-elements.txt │ └── nonreservedwords.txt ├── src │ ├── test │ │ └── java │ │ │ └── com │ │ │ └── facebook │ │ │ └── coresql │ │ │ └── parser │ │ │ ├── sqllogictest │ │ │ ├── resources │ │ │ │ └── config.properties │ │ │ └── java │ │ │ │ ├── SqlLogicTestModule.java │ │ │ │ ├── SqlLogicTestResult.java │ │ │ │ ├── SqlLogicTestConfig.java │ │ │ │ ├── SqlLogicTestStatement.java │ │ │ │ └── SqlLogicTest.java │ │ │ ├── TestKeywords.java │ │ │ └── TestSqlParser.java │ └── main │ │ └── java │ │ └── com │ │ └── facebook │ │ └── coresql │ │ └── parser │ │ ├── Location.java │ │ ├── ParserHelper.java │ │ ├── AstNode.java │ │ └── Unparser.java └── pom.xml ├── README.md ├── .gitignore ├── linter ├── src │ ├── main │ │ └── java │ │ │ └── com │ │ │ └── facebook │ │ │ └── coresql │ │ │ └── linter │ │ │ ├── warning │ │ │ ├── WarningCollector.java │ │ │ ├── StandardWarningCode.java │ │ │ ├── WarningCollectorConfig.java │ │ │ ├── CoreSqlWarning.java │ │ │ ├── WarningCode.java │ │ │ └── DefaultWarningCollector.java │ │ │ └── lint │ │ │ ├── LintingVisitor.java │ │ │ └── MixedAndOr.java │ └── test │ │ └── java │ │ └── com │ │ └── facebook │ │ └── coresql │ │ └── linter │ │ ├── warning │ │ └── TestWarningCollector.java │ │ └── lint │ │ └── TestMixedAndOr.java └── pom.xml ├── rewriter ├── src │ ├── main │ │ └── java │ │ │ └── com │ │ │ └── facebook │ │ │ └── coresql │ │ │ └── rewriter │ │ │ ├── Rewriter.java │ │ │ ├── RewriteResult.java │ │ │ └── OrderByRewriter.java │ └── test │ │ └── java │ │ └── com │ │ └── facebook │ │ └── coresql │ │ └── rewriter │ │ └── TestOrderByRewriter.java └── pom.xml ├── pom.xml └── LICENSE /.travis.yml: -------------------------------------------------------------------------------- 1 | language: java 2 | jdk: 3 | - openjdk8 4 | -------------------------------------------------------------------------------- /parser/cpp/build.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | echo 'Generating parser ...' 4 | mvn clean generate-sources 5 | echo 'Compiling ...' 6 | ./compile.sh 7 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # SQL Language Frontend 2 | 3 | A Modern SQL frontend based on SQL16 with extensions for streaming, graph, rich types, etc, including parser, resolver, rewriters, etc. 4 | -------------------------------------------------------------------------------- /parser/cpp/compile.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | SRC_DIR=./target/generated-sources/javacc 3 | TARGET_DIR=./target 4 | clang++ -fbracket-depth=1024 -funsigned-char -I. -I./$SRC_DIR -o $TARGET_DIR/sqlparser -std=c++11 ParseErrorHandler.cc main.cc $SRC_DIR/*.cc 5 | -------------------------------------------------------------------------------- /parser/grammar/prepare-javacc-grammar.sh: -------------------------------------------------------------------------------- 1 | # Concatenate all the fragments into a .jj file. 2 | gendir='../target/generated-sources/javacc' 3 | mkdir -p $gendir 4 | cat javacc-options-java.txt nonreservedwords.txt reservedwords.txt sql-spec.txt presto-extensions.txt lexical-elements.txt > $gendir/parser_tmp.jjt 5 | -------------------------------------------------------------------------------- /parser/src/test/java/com/facebook/coresql/parser/sqllogictest/resources/config.properties: -------------------------------------------------------------------------------- 1 | test-files-root-path=src/test/java/com/facebook/coresql/parser/sqllogictest/resources/tests 2 | result-path=src/test/java/com/facebook/coresql/parser/sqllogictest/resources 3 | max-num-files-to-visit=10 4 | database-dialect=ALL 5 | -------------------------------------------------------------------------------- /parser/cpp/prepare-javacc-grammar.sh: -------------------------------------------------------------------------------- 1 | # Concatenate all the fragments into a .jj file. 2 | pwd 3 | GRAMMAR_DIR='../grammar' 4 | GEN_DIR='target/generated-sources/javacc' 5 | mkdir -p $GEN_DIR 6 | cat ./javacc-options.txt $GRAMMAR_DIR/nonreservedwords.txt $GRAMMAR_DIR/reservedwords.txt $GRAMMAR_DIR/sql-spec.txt $GRAMMAR_DIR/presto-extensions.txt $GRAMMAR_DIR/lexical-elements.txt > $GEN_DIR/parser_tmp.jjt 7 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.iml 2 | *.ipr 3 | *.iws 4 | target/ 5 | /var 6 | /*/var/ 7 | /presto-product-tests/**/var/ 8 | pom.xml.versionsBackup 9 | test-output/ 10 | test-reports/ 11 | /atlassian-ide-plugin.xml 12 | .idea 13 | .DS_Store 14 | .classpath 15 | .settings 16 | .project 17 | temp-testng-customsuite.xml 18 | test-output 19 | .externalToolBuilders 20 | *~ 21 | benchmark_outputs 22 | *.pyc 23 | *.class 24 | .checkstyle 25 | .mvn/timing.properties 26 | .editorconfig 27 | node_modules 28 | -------------------------------------------------------------------------------- /parser/cpp/generate_parser.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | env | grep DIR 4 | mkdir -p ${INSTALL_DIR} 5 | cat \ 6 | ${SRCDIR}/javacc-options.txt \ 7 | ${SRCDIR}/kw.txt \ 8 | ${SRCDIR}/sql-spec.txt \ 9 | ${SRCDIR}/presto-extensions.txt \ 10 | ${SRCDIR}/nonreservedwords.txt \ 11 | ${SRCDIR}/lexical-elements.txt \ 12 | > ${INSTALL_DIR}/parser_tmp.jjt 13 | java -cp ${SRCDIR}/javacc/javacc.jar jjtree -OUTPUT_DIRECTORY=${INSTALL_DIR} ${INSTALL_DIR}/parser_tmp.jjt 14 | java -cp ${SRCDIR}/javacc/javacc.jar javacc -OUTPUT_DIRECTORY=${INSTALL_DIR} ${INSTALL_DIR}/parser_tmp.jj 15 | touch ../ready.txt 16 | -------------------------------------------------------------------------------- /parser/grammar/notes: -------------------------------------------------------------------------------- 1 | Here are some notes that explain the reason for some of the non-standard looking changes made in the grammar. Most of these are to facilitate building AST in the correcto form. 2 | 3 | JJTree: 4 | ------ 5 | 6 | * For things like: 7 | int ARRAY[10] 8 | that are suffxies on other valid entities - ARRAY[10] is a suffix for a valid basic type 'int' - we need to pop the node for 9 | the the int that's already on the stack first before making the tree for the array. 10 | 11 | * #Unsupported is used to build special AST node of kind Unsupported so we can have the full grammar in the source and handle unsupported features gracefully. 12 | 13 | 14 | Javacc: 15 | ------ 16 | 17 | * a -> b is potentially ambiguous as SQL allows pointer like member access using -> which conflicts with Presto lambda. we need to 18 | -------------------------------------------------------------------------------- /parser/cpp/parser.h: -------------------------------------------------------------------------------- 1 | #ifndef __PARSER_H_ 2 | #define __PARSER_H_ 3 | 4 | #include 5 | #include 6 | 7 | #include "ErrorHandler.h" 8 | #include "Token.h" 9 | #include "SqlParserConstants.h" 10 | #include "SqlParserTreeConstants.h" 11 | #include "SqlParser.h" 12 | 13 | using std::cerr; 14 | using std::cout; 15 | using std::deque; 16 | 17 | namespace commonsql { 18 | namespace parser { 19 | 20 | class ParseErrorHandler: public ErrorHandler { 21 | public: 22 | virtual void handleUnexpectedToken(int expectedKind, const JJString& expectedToken, Token* actual, SqlParser* parser); 23 | virtual void handleParseError(Token* last, Token* unexpected, const JJSimpleString& production, SqlParser* parser); 24 | virtual void handleOtherError(const JJString& message, SqlParser* parser); 25 | }; 26 | 27 | } 28 | } 29 | 30 | #endif 31 | -------------------------------------------------------------------------------- /linter/src/main/java/com/facebook/coresql/linter/warning/WarningCollector.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed under the Apache License, Version 2.0 (the "License"); 3 | * you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at 5 | * 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * 8 | * Unless required by applicable law or agreed to in writing, software 9 | * distributed under the License is distributed on an "AS IS" BASIS, 10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the License for the specific language governing permissions and 12 | * limitations under the License. 13 | */ 14 | 15 | package com.facebook.coresql.linter.warning; 16 | 17 | import com.facebook.coresql.parser.AstNode; 18 | 19 | import java.util.List; 20 | 21 | public interface WarningCollector 22 | { 23 | void add(WarningCode code, String warningMessage, AstNode node); 24 | 25 | List getAllWarnings(); 26 | 27 | void clearWarnings(); 28 | } 29 | -------------------------------------------------------------------------------- /linter/src/main/java/com/facebook/coresql/linter/warning/StandardWarningCode.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed under the Apache License, Version 2.0 (the "License"); 3 | * you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at 5 | * 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * 8 | * Unless required by applicable law or agreed to in writing, software 9 | * distributed under the License is distributed on an "AS IS" BASIS, 10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the License for the specific language governing permissions and 12 | * limitations under the License. 13 | */ 14 | 15 | package com.facebook.coresql.linter.warning; 16 | 17 | public enum StandardWarningCode 18 | { 19 | MIXING_AND_OR_WITHOUT_PARENTHESES(0x0000_0001); 20 | 21 | private final WarningCode warningCode; 22 | 23 | StandardWarningCode(int code) 24 | { 25 | warningCode = new WarningCode(code, name()); 26 | } 27 | 28 | public WarningCode getWarningCode() 29 | { 30 | return warningCode; 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /rewriter/src/main/java/com/facebook/coresql/rewriter/Rewriter.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed under the Apache License, Version 2.0 (the "License"); 3 | * you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at 5 | * 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * 8 | * Unless required by applicable law or agreed to in writing, software 9 | * distributed under the License is distributed on an "AS IS" BASIS, 10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the License for the specific language governing permissions and 12 | * limitations under the License. 13 | */ 14 | 15 | package com.facebook.coresql.rewriter; 16 | 17 | import com.facebook.coresql.parser.Unparser; 18 | 19 | import java.util.Optional; 20 | 21 | public abstract class Rewriter 22 | extends Unparser 23 | { 24 | /** 25 | * Attempts to rewrite a SQL statement, storing both the name of the rewriter and the rewritten statement in a RewriteResult object. 26 | * 27 | * @return An Optional object containing information about the rewrite 28 | */ 29 | public abstract Optional rewrite(); 30 | } 31 | -------------------------------------------------------------------------------- /linter/src/main/java/com/facebook/coresql/linter/warning/WarningCollectorConfig.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed under the Apache License, Version 2.0 (the "License"); 3 | * you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at 5 | * 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * 8 | * Unless required by applicable law or agreed to in writing, software 9 | * distributed under the License is distributed on an "AS IS" BASIS, 10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the License for the specific language governing permissions and 12 | * limitations under the License. 13 | */ 14 | 15 | package com.facebook.coresql.linter.warning; 16 | 17 | import static com.google.common.base.Preconditions.checkArgument; 18 | 19 | public class WarningCollectorConfig 20 | { 21 | private int maxWarnings = Integer.MAX_VALUE; 22 | 23 | public WarningCollectorConfig setMaxWarnings(int maxWarnings) 24 | { 25 | checkArgument(maxWarnings >= 0, "maxWarnings must be >= 0"); 26 | this.maxWarnings = maxWarnings; 27 | return this; 28 | } 29 | 30 | public int getMaxWarnings() 31 | { 32 | return maxWarnings; 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /parser/src/test/java/com/facebook/coresql/parser/sqllogictest/java/SqlLogicTestModule.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed under the Apache License, Version 2.0 (the "License"); 3 | * you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at 5 | * 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * 8 | * Unless required by applicable law or agreed to in writing, software 9 | * distributed under the License is distributed on an "AS IS" BASIS, 10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the License for the specific language governing permissions and 12 | * limitations under the License. 13 | */ 14 | 15 | package com.facebook.coresql.parser.sqllogictest.java; 16 | 17 | import com.google.inject.Binder; 18 | import com.google.inject.Module; 19 | 20 | import static com.facebook.airlift.configuration.ConfigBinder.configBinder; 21 | import static com.google.inject.Scopes.SINGLETON; 22 | 23 | public class SqlLogicTestModule 24 | implements Module 25 | { 26 | @Override 27 | public void configure(Binder binder) 28 | { 29 | configBinder(binder).bindConfig(SqlLogicTestConfig.class); 30 | binder.bind(SqlLogicTest.class).in(SINGLETON); 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /parser/src/main/java/com/facebook/coresql/parser/Location.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed under the Apache License, Version 2.0 (the "License"); 3 | * you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at 5 | * 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * 8 | * Unless required by applicable law or agreed to in writing, software 9 | * distributed under the License is distributed on an "AS IS" BASIS, 10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the License for the specific language governing permissions and 12 | * limitations under the License. 13 | */ 14 | 15 | package com.facebook.coresql.parser; 16 | 17 | import static java.lang.String.format; 18 | 19 | public class Location 20 | { 21 | private final int beginLine; 22 | private final int beginColumn; 23 | private final int endLine; 24 | private final int endColumn; 25 | 26 | public Location(int beginLine, int beginColumn, int endLine, int endColumn) 27 | { 28 | this.beginLine = beginLine; 29 | this.beginColumn = beginColumn; 30 | this.endLine = endLine; 31 | this.endColumn = endColumn; 32 | } 33 | 34 | @Override 35 | public String toString() 36 | { 37 | return format("%d:%d-%d:%d", beginLine, beginColumn, endLine, endColumn); 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /parser/cpp/tokenize.cc: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | 7 | #include "SqlParserConstants.h" 8 | #include "CharStream.h" 9 | #include "CharStream.h" 10 | #include "Token.h" 11 | #include "SqlParser.h" 12 | #include "SqlParserTokenManager.h" 13 | 14 | #include 15 | 16 | using namespace commonsql::parser; 17 | using namespace std; 18 | 19 | JAVACC_STRING_TYPE ReadFileFully(char *file_name) { 20 | JAVACC_STRING_TYPE s; 21 | #if WIDE_CHAR 22 | wifstream fp_in; 23 | #else 24 | ifstream fp_in; 25 | #endif 26 | fp_in.open(file_name, ios::in); 27 | // Very inefficient. 28 | while (!fp_in.eof()) { 29 | s += fp_in.get(); 30 | } 31 | return s; 32 | } 33 | 34 | int main(int argc, char **argv) { 35 | if (argc < 2) { 36 | cout << "Usage: sqlparser " << endl; 37 | exit(1); 38 | } 39 | JAVACC_STRING_TYPE s = ReadFileFully(argv[1]); 40 | 41 | clock_t start,finish; 42 | double time; 43 | start = clock(); 44 | 45 | int cnt = 0; 46 | for (int i = 0; i < 2; i++) { 47 | CharStream *stream = new CharStream(s.c_str(), s.size() - 1, 1, 1); 48 | SqlParserTokenManager *scanner = new SqlParserTokenManager(stream); 49 | Token *t; 50 | cnt = 0; 51 | while ((t = scanner->getNextToken())->kind != _EOF) cnt++; 52 | } 53 | 54 | finish = clock(); 55 | time = (double(finish)-double(start))/CLOCKS_PER_SEC; 56 | printf ("Tokens: %d, avg scanning time: %lfms\n", cnt, (time*1000)/2); 57 | } 58 | -------------------------------------------------------------------------------- /parser/src/main/java/com/facebook/coresql/parser/ParserHelper.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed under the Apache License, Version 2.0 (the "License"); 3 | * you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at 5 | * 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * 8 | * Unless required by applicable law or agreed to in writing, software 9 | * distributed under the License is distributed on an "AS IS" BASIS, 10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the License for the specific language governing permissions and 12 | * limitations under the License. 13 | */ 14 | package com.facebook.coresql.parser; 15 | 16 | import java.io.StringReader; 17 | 18 | public class ParserHelper 19 | { 20 | private ParserHelper() {} 21 | 22 | public static AstNode parseStatement(String sql) 23 | { 24 | SqlParser parser = new SqlParser(new SqlParserTokenManager(new SimpleCharStream(new StringReader(sql), 1, 1))); 25 | try { 26 | parser.direct_SQL_statement(); 27 | return parser.getResult(); 28 | } 29 | catch (ParseException pe) { 30 | return null; 31 | } 32 | } 33 | 34 | public static AstNode parseExpression(String expression) 35 | { 36 | SqlParser parser = new SqlParser(new SqlParserTokenManager(new SimpleCharStream(new StringReader(expression), 1, 1))); 37 | try { 38 | parser.derived_column(); 39 | return parser.getResult(); 40 | } 41 | catch (ParseException pe) { 42 | return null; 43 | } 44 | } 45 | } 46 | -------------------------------------------------------------------------------- /parser/cpp/main.cc: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | 7 | #include "SqlParserConstants.h" 8 | #include "CharStream.h" 9 | #include "CharStream.h" 10 | #include "Token.h" 11 | #include "SqlParser.h" 12 | #include "SqlParserTokenManager.h" 13 | #include "parser.h" 14 | 15 | #include 16 | 17 | using namespace commonsql::parser; 18 | using namespace std; 19 | 20 | JAVACC_STRING_TYPE ReadFileFully(char *file_name) { 21 | JAVACC_STRING_TYPE s; 22 | ifstream fp_in; 23 | fp_in.open(file_name, ios::in); 24 | // Very inefficient. 25 | while (!fp_in.eof()) { 26 | s += fp_in.get(); 27 | } 28 | return s; 29 | } 30 | 31 | int main(int argc, char **argv) { 32 | if (argc < 2) { 33 | cout << "Usage: sqlparser " << endl; 34 | exit(1); 35 | } 36 | JAVACC_STRING_TYPE s = ReadFileFully(argv[1]); 37 | 38 | clock_t start,finish; 39 | double time; 40 | start = clock(); 41 | 42 | for (int i = 0; i < 1; i++) { 43 | CharStream *stream = new CharStream(s.c_str(), s.size() - 1, 1, 1); 44 | SqlParserTokenManager *scanner = new SqlParserTokenManager(stream); 45 | SqlParser parser(scanner); 46 | parser.setErrorHandler(new ParseErrorHandler()); 47 | parser.compilation_unit(); 48 | SimpleNode *root = (SimpleNode*)parser.jjtree.peekNode(); 49 | if (root) { 50 | JAVACC_STRING_TYPE buffer; 51 | root->dumpToBuffer(" ", "\n", &buffer); 52 | printf("%s\n", buffer.c_str()); 53 | } 54 | } 55 | 56 | finish = clock(); 57 | time = (double(finish)-double(start))/CLOCKS_PER_SEC; 58 | printf ("Avg parsing time: %lfms\n", (time*1000)/1); 59 | } 60 | -------------------------------------------------------------------------------- /rewriter/src/main/java/com/facebook/coresql/rewriter/RewriteResult.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed under the Apache License, Version 2.0 (the "License"); 3 | * you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at 5 | * 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * 8 | * Unless required by applicable law or agreed to in writing, software 9 | * distributed under the License is distributed on an "AS IS" BASIS, 10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the License for the specific language governing permissions and 12 | * limitations under the License. 13 | */ 14 | 15 | package com.facebook.coresql.rewriter; 16 | 17 | import com.fasterxml.jackson.annotation.JsonCreator; 18 | import com.fasterxml.jackson.annotation.JsonProperty; 19 | 20 | import static java.util.Objects.requireNonNull; 21 | 22 | public class RewriteResult 23 | { 24 | private final String nameOfRewrite; 25 | private final String rewrittenSql; 26 | 27 | @JsonCreator 28 | public RewriteResult( 29 | @JsonProperty("nameOfRewrite") String nameOfRewrite, 30 | @JsonProperty("rewrittenSql") String rewrittenSql) 31 | { 32 | this.nameOfRewrite = requireNonNull(nameOfRewrite, "name of rewrite is null"); 33 | this.rewrittenSql = requireNonNull(rewrittenSql, "rewritten sql statement is null"); 34 | } 35 | 36 | @JsonProperty("nameOfRewrite") 37 | public String getNameOfRewrite() 38 | { 39 | return nameOfRewrite; 40 | } 41 | 42 | @JsonProperty("rewrittenSql") 43 | public String getRewrittenSql() 44 | { 45 | return rewrittenSql; 46 | } 47 | } 48 | -------------------------------------------------------------------------------- /linter/src/main/java/com/facebook/coresql/linter/warning/CoreSqlWarning.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed under the Apache License, Version 2.0 (the "License"); 3 | * you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at 5 | * 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * 8 | * Unless required by applicable law or agreed to in writing, software 9 | * distributed under the License is distributed on an "AS IS" BASIS, 10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the License for the specific language governing permissions and 12 | * limitations under the License. 13 | */ 14 | 15 | package com.facebook.coresql.linter.warning; 16 | 17 | import com.facebook.coresql.parser.Location; 18 | 19 | import static java.lang.String.format; 20 | import static java.util.Objects.requireNonNull; 21 | 22 | public final class CoreSqlWarning 23 | { 24 | private final WarningCode warningCode; 25 | private final String message; 26 | private final Location location; 27 | 28 | public CoreSqlWarning(WarningCode warningCode, String message, 29 | int beginLine, 30 | int beginColumn, 31 | int endLine, 32 | int endColumn) 33 | { 34 | this.warningCode = requireNonNull(warningCode, "Warning code is null"); 35 | this.message = requireNonNull(message, "Warning message is null"); 36 | this.location = new Location(beginLine, beginColumn, endLine, endColumn); 37 | } 38 | 39 | public WarningCode getWarningCode() 40 | { 41 | return warningCode; 42 | } 43 | 44 | @Override 45 | public String toString() 46 | { 47 | return format("Warning@%s message: %s [%s]", location, message, warningCode); 48 | } 49 | } 50 | -------------------------------------------------------------------------------- /linter/src/test/java/com/facebook/coresql/linter/warning/TestWarningCollector.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed under the Apache License, Version 2.0 (the "License"); 3 | * you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at 5 | * 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * 8 | * Unless required by applicable law or agreed to in writing, software 9 | * distributed under the License is distributed on an "AS IS" BASIS, 10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the License for the specific language governing permissions and 12 | * limitations under the License. 13 | */ 14 | package com.facebook.coresql.linter.warning; 15 | 16 | import com.facebook.coresql.parser.AstNode; 17 | import org.testng.annotations.Test; 18 | 19 | import static com.facebook.coresql.parser.ParserHelper.parseStatement; 20 | import static org.testng.Assert.assertEquals; 21 | 22 | public class TestWarningCollector 23 | { 24 | private static final AstNode AST_NODE = parseStatement("SELECT true or false and false;"); 25 | private static final WarningCollector COLLECTOR = new DefaultWarningCollector(new WarningCollectorConfig().setMaxWarnings(1)); 26 | 27 | private static void assertCollectorHasOneWarning() 28 | { 29 | assertEquals(COLLECTOR.getAllWarnings().size(), 1); 30 | } 31 | 32 | @Test 33 | public void addWarningTest() 34 | { 35 | COLLECTOR.add(new WarningCode(0, "test code"), "msg", AST_NODE); 36 | assertCollectorHasOneWarning(); 37 | } 38 | 39 | @Test 40 | public void testWarningsExceedMaxInConfig() 41 | { 42 | COLLECTOR.add(new WarningCode(0, "test code"), "msg", AST_NODE); 43 | COLLECTOR.add(new WarningCode(0, "test code"), "msg", AST_NODE); 44 | assertCollectorHasOneWarning(); 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /linter/src/main/java/com/facebook/coresql/linter/lint/LintingVisitor.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed under the Apache License, Version 2.0 (the "License"); 3 | * you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at 5 | * 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * 8 | * Unless required by applicable law or agreed to in writing, software 9 | * distributed under the License is distributed on an "AS IS" BASIS, 10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the License for the specific language governing permissions and 12 | * limitations under the License. 13 | */ 14 | 15 | package com.facebook.coresql.linter.lint; 16 | 17 | import com.facebook.coresql.linter.warning.WarningCode; 18 | import com.facebook.coresql.linter.warning.WarningCollector; 19 | import com.facebook.coresql.parser.AstNode; 20 | import com.facebook.coresql.parser.SqlParserDefaultVisitor; 21 | 22 | public abstract class LintingVisitor 23 | extends SqlParserDefaultVisitor 24 | { 25 | private final WarningCollector warningCollector; 26 | 27 | public LintingVisitor(WarningCollector collector) 28 | { 29 | this.warningCollector = collector; 30 | } 31 | 32 | public void addWarningToCollector(WarningCode code, String warningMessage, AstNode node) 33 | { 34 | warningCollector.add(code, warningMessage, node); 35 | } 36 | 37 | /** 38 | * Entry point to recursive visiting routine. We recurse, add any warnings to the current collector, then return. 39 | * 40 | * @param node The root of the AST we're validating 41 | */ 42 | public void lint(AstNode node) 43 | { 44 | node.jjtAccept(this, null); 45 | } 46 | 47 | public WarningCollector getWarningCollector() 48 | { 49 | return warningCollector; 50 | } 51 | } 52 | -------------------------------------------------------------------------------- /linter/src/main/java/com/facebook/coresql/linter/warning/WarningCode.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed under the Apache License, Version 2.0 (the "License"); 3 | * you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at 5 | * 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * 8 | * Unless required by applicable law or agreed to in writing, software 9 | * distributed under the License is distributed on an "AS IS" BASIS, 10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the License for the specific language governing permissions and 12 | * limitations under the License. 13 | */ 14 | 15 | package com.facebook.coresql.linter.warning; 16 | 17 | import java.util.Objects; 18 | 19 | import static java.util.Objects.requireNonNull; 20 | 21 | public class WarningCode 22 | { 23 | private final int code; 24 | private final String name; 25 | 26 | public WarningCode(int code, String name) 27 | { 28 | this.code = code; 29 | this.name = requireNonNull(name, "name is null"); 30 | } 31 | 32 | public int getCode() 33 | { 34 | return code; 35 | } 36 | 37 | public String getName() 38 | { 39 | return name; 40 | } 41 | 42 | @Override 43 | public String toString() 44 | { 45 | return name + ", " + code; 46 | } 47 | 48 | @Override 49 | public boolean equals(Object obj) 50 | { 51 | if (this == obj) { 52 | return true; 53 | } 54 | if (obj == null || getClass() != obj.getClass()) { 55 | return false; 56 | } 57 | 58 | WarningCode that = (WarningCode) obj; 59 | return this.code == that.code && Objects.equals(this.name, that.name); 60 | } 61 | 62 | @Override 63 | public int hashCode() 64 | { 65 | return Objects.hash(code, name); 66 | } 67 | } 68 | -------------------------------------------------------------------------------- /parser/grammar/datetimespec.txt: -------------------------------------------------------------------------------- 1 | date_string() ::= 2 | "'" unquoted_date_string() "'" 3 | ; 4 | 5 | time_string() ::= 6 | "'" unquoted_time_string() "'" 7 | ; 8 | 9 | timestamp_string() ::= 10 | "'" unquoted_timestamp_string() "'" 11 | ; 12 | 13 | time_zone_interval() ::= 14 | sign() hours_value() ":" minutes_value() 15 | ; 16 | 17 | date_value() ::= 18 | years_value() "-" months_value() "-" days_value() 19 | ; 20 | 21 | time_value() ::= 22 | hours_value() ":" minutes_value() ":" seconds_value() 23 | ; 24 | 25 | interval_string() ::= 26 | "'" unquoted_interval_string() "'" 27 | ; 28 | 29 | unquoted_date_string() ::= 30 | date_value() 31 | ; 32 | 33 | unquoted_time_string() ::= 34 | time_value() [ time_zone_interval() ] 35 | ; 36 | 37 | unquoted_timestamp_string() ::= 38 | unquoted_date_string() space() unquoted_time_string() 39 | ; 40 | 41 | unquoted_interval_string() ::= 42 | [ sign() ] ( year_month_literal() | day_time_literal() ) 43 | ; 44 | 45 | year_month_literal() ::= 46 | years_value() [ "-" months_value() ] 47 | | months_value() 48 | ; 49 | 50 | day_time_literal() ::= 51 | day_time_interval() 52 | | time_interval() 53 | ; 54 | 55 | day_time_interval() ::= 56 | days_value() [ space() hours_value() [ ":" minutes_value() 57 | [ ":" seconds_value() ] ] ] 58 | ; 59 | 60 | time_interval() ::= 61 | hours_value() [ ":" minutes_value() [ ":" seconds_value() ] ] 62 | | minutes_value() [ ":" seconds_value() ] 63 | | seconds_value() 64 | ; 65 | 66 | years_value() ::= 67 | datetime_value() 68 | ; 69 | 70 | months_value() ::= 71 | datetime_value() 72 | ; 73 | 74 | days_value() ::= 75 | datetime_value() 76 | ; 77 | 78 | hours_value() ::= 79 | datetime_value() 80 | ; 81 | 82 | minutes_value() ::= 83 | datetime_value() 84 | ; 85 | 86 | seconds_value() ::= 87 | seconds_integer_value() [ "." [ seconds_fraction() ] ] 88 | ; 89 | 90 | seconds_integer_value() ::= 91 | unsigned_integer() 92 | ; 93 | 94 | seconds_fraction() ::= 95 | unsigned_integer() 96 | ; 97 | 98 | datetime_value() ::= 99 | unsigned_integer() 100 | ; 101 | 102 | -------------------------------------------------------------------------------- /parser/cpp/ParseErrorHandler.cc: -------------------------------------------------------------------------------- 1 | #include "parser.h" 2 | #include "ErrorHandler.h" 3 | #include "Token.h" 4 | #include "SqlParserConstants.h" 5 | #include "SqlParser.h" 6 | 7 | #include 8 | 9 | using std::cerr; 10 | using std::cout; 11 | 12 | namespace commonsql { 13 | namespace parser { 14 | 15 | inline bool isKeyword(Token* t) { 16 | return t->kind >= MIN_RESERVED_WORD and t->kind <= MAX_RESERVED_WORD; 17 | } 18 | 19 | // Since we have a lookahead of 3, check the next 2 tokens to see if someone is usign a keyword as identifier. 20 | inline Token* getPossibleKeyword(SqlParser* parser) { 21 | if (isKeyword(parser->getToken(1))) return parser->getToken(1); 22 | if (isKeyword(parser->getToken(2))) return parser->getToken(2); 23 | return nullptr; 24 | } 25 | 26 | void ParseErrorHandler::handleUnexpectedToken(int expectedKind, const JJString& expectedToken, Token* actual, SqlParser* parser) { 27 | Token* possibleKeyword = getPossibleKeyword(parser); 28 | if (possibleKeyword == nullptr) { 29 | cerr << actual->beginLine 30 | << ":" 31 | << actual->beginColumn 32 | << " Unexpected token: \"" 33 | << (*actual).image << "\" after: \"" 34 | << parser->getToken(0)->image 35 | << "\". Expecting: " 36 | << expectedKind 37 | << "(" 38 | << (expectedKind <= 0 ? "EOF" : tokenImage[expectedKind]) 39 | << ")\n"; 40 | } else { 41 | cerr << actual->beginLine << ":" << actual->beginColumn << " Unexpected keyword: \"" << (*possibleKeyword).image << "\"" << "\n"; 42 | } 43 | } 44 | 45 | void ParseErrorHandler::handleParseError(Token* last, Token* unexpected, const JJSimpleString& production, SqlParser* parser) { 46 | cerr << last->beginLine << ":" << last->beginColumn << " Unexpected token: " << (*unexpected).image << " after: " << (*last).image << " while parsing: " << production << "\n"; 47 | } 48 | 49 | void ParseErrorHandler::handleOtherError(const JJString& message, SqlParser* parser) { 50 | cerr << "Error: %s\n" << message.c_str() << "\n"; 51 | }; 52 | 53 | } 54 | } 55 | -------------------------------------------------------------------------------- /parser/grammar/regexp.txt: -------------------------------------------------------------------------------- 1 | // Used in SIMILAR etc. For now, we call it unsupported. 2 | void regular_expression() #Unsupported: 3 | {} 4 | { 5 | regular_term() ( "|" regular_term() )* 6 | } 7 | 8 | void regular_term(): 9 | {} 10 | { 11 | ( regular_factor() )+ 12 | } 13 | 14 | void regular_factor(): 15 | {} 16 | { 17 | regular_primary() 18 | ( "*" 19 | | "+" 20 | | "?" 21 | | repeat_factor() 22 | ) 23 | } 24 | 25 | void repeat_factor(): 26 | {} 27 | { 28 | "(" low_value() [ upper_limit() ] ")" 29 | } 30 | 31 | void upper_limit(): 32 | {} 33 | { 34 | "," [ high_value() ] 35 | } 36 | 37 | void low_value(): 38 | {} 39 | { 40 | 41 | } 42 | 43 | void high_value(): 44 | {} 45 | { 46 | 47 | } 48 | 49 | void regular_primary(): 50 | {} 51 | { 52 | character_specifier() 53 | | "%" 54 | | regular_character_set() 55 | | "(" regular_expression() ")" 56 | } 57 | 58 | void character_specifier(): 59 | {} 60 | { 61 | // TODO(kaikalur) 62 | //non_escaped_character() 63 | //| escaped_character() 64 | character_string_literal() // temp 65 | } 66 | 67 | /* 68 | void non_escaped_character(): 69 | {} 70 | { 71 | //!! See the Syntax Rules. 72 | character_string_literal() // temp 73 | } 74 | 75 | void escaped_character(): 76 | {} 77 | { 78 | //!! See the Syntax Rules. 79 | character_string_literal() // temp 80 | } 81 | */ 82 | 83 | void regular_character_set(): 84 | {} 85 | { 86 | "_" 87 | | "[" ( character_enumeration() )+ "]" 88 | | "[" "^" ( character_enumeration() )+ "]" 89 | | "[" ( character_enumeration_include() )+ 90 | "^" ( character_enumeration_exclude() )+ "]" 91 | } 92 | 93 | void character_enumeration_include(): 94 | {} 95 | { 96 | character_enumeration() 97 | } 98 | 99 | void character_enumeration_exclude(): 100 | {} 101 | { 102 | character_enumeration() 103 | } 104 | 105 | void character_enumeration(): 106 | {} 107 | { 108 | character_specifier() 109 | | character_specifier() "-" character_specifier() 110 | | "[" ":" identifier() ":" "]" 111 | } 112 | 113 | -------------------------------------------------------------------------------- /linter/src/main/java/com/facebook/coresql/linter/lint/MixedAndOr.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed under the Apache License, Version 2.0 (the "License"); 3 | * you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at 5 | * 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * 8 | * Unless required by applicable law or agreed to in writing, software 9 | * distributed under the License is distributed on an "AS IS" BASIS, 10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the License for the specific language governing permissions and 12 | * limitations under the License. 13 | */ 14 | 15 | package com.facebook.coresql.linter.lint; 16 | 17 | import com.facebook.coresql.linter.warning.WarningCollector; 18 | import com.facebook.coresql.parser.AndExpression; 19 | import com.facebook.coresql.parser.OrExpression; 20 | 21 | import static com.facebook.coresql.linter.warning.StandardWarningCode.MIXING_AND_OR_WITHOUT_PARENTHESES; 22 | import static com.facebook.coresql.parser.SqlParserTreeConstants.JJTANDEXPRESSION; 23 | import static com.facebook.coresql.parser.SqlParserTreeConstants.JJTOREXPRESSION; 24 | 25 | /** 26 | * A visitor that validates an AST built from an SQL string. Right now, it validates 27 | * a single clause: don't mix AND and OR without parentheses. 28 | */ 29 | public class MixedAndOr 30 | extends LintingVisitor 31 | { 32 | private static final String WARNING_MESSAGE = "Mixing AND and OR without parentheses."; 33 | 34 | public MixedAndOr(WarningCollector collector) 35 | { 36 | super(collector); 37 | } 38 | 39 | @Override 40 | public void visit(OrExpression node, Void data) 41 | { 42 | if (node.jjtGetParent().getId() == JJTANDEXPRESSION) { 43 | super.addWarningToCollector(MIXING_AND_OR_WITHOUT_PARENTHESES.getWarningCode(), 44 | WARNING_MESSAGE, 45 | node); 46 | } 47 | defaultVisit(node, data); 48 | } 49 | 50 | @Override 51 | public void visit(AndExpression node, Void data) 52 | { 53 | if (node.jjtGetParent().getId() == JJTOREXPRESSION) { 54 | super.addWarningToCollector(MIXING_AND_OR_WITHOUT_PARENTHESES.getWarningCode(), 55 | WARNING_MESSAGE, 56 | node); 57 | } 58 | defaultVisit(node, data); 59 | } 60 | } 61 | -------------------------------------------------------------------------------- /parser/src/test/java/com/facebook/coresql/parser/sqllogictest/java/SqlLogicTestResult.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed under the Apache License, Version 2.0 (the "License"); 3 | * you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at 5 | * 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * 8 | * Unless required by applicable law or agreed to in writing, software 9 | * distributed under the License is distributed on an "AS IS" BASIS, 10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the License for the specific language governing permissions and 12 | * limitations under the License. 13 | */ 14 | 15 | package com.facebook.coresql.parser.sqllogictest.java; 16 | 17 | import com.facebook.coresql.parser.sqllogictest.java.SqlLogicTest.DatabaseDialect; 18 | import com.google.common.collect.ImmutableList; 19 | 20 | import java.util.ArrayList; 21 | import java.util.List; 22 | 23 | import static java.util.Objects.requireNonNull; 24 | 25 | public class SqlLogicTestResult 26 | { 27 | private final List erroneousStatements; 28 | private final DatabaseDialect databaseDialect; 29 | private int numFilesVisited; 30 | private int numStatementsFound; 31 | 32 | public SqlLogicTestResult(DatabaseDialect databaseDialect) 33 | { 34 | this.databaseDialect = requireNonNull(databaseDialect); 35 | this.erroneousStatements = new ArrayList<>(); 36 | } 37 | 38 | public DatabaseDialect getDatabaseDialect() 39 | { 40 | return databaseDialect; 41 | } 42 | 43 | public void addErroneousStatements(List erroneousStatements) 44 | { 45 | this.erroneousStatements.addAll(erroneousStatements); 46 | } 47 | 48 | public List getErroneousStatements() 49 | { 50 | return ImmutableList.copyOf(erroneousStatements); 51 | } 52 | 53 | public int getNumFilesVisited() 54 | { 55 | return numFilesVisited; 56 | } 57 | 58 | public void incrementNumFilesVisited() 59 | { 60 | numFilesVisited += 1; 61 | } 62 | 63 | public int getNumStatementsFound() 64 | { 65 | return numStatementsFound; 66 | } 67 | 68 | public void incrementNumStatementsFound(int incrementAmount) 69 | { 70 | this.numStatementsFound += incrementAmount; 71 | } 72 | 73 | public int getNumStatementsParsed() 74 | { 75 | return numStatementsFound - erroneousStatements.size(); 76 | } 77 | } 78 | -------------------------------------------------------------------------------- /linter/src/main/java/com/facebook/coresql/linter/warning/DefaultWarningCollector.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed under the Apache License, Version 2.0 (the "License"); 3 | * you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at 5 | * 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * 8 | * Unless required by applicable law or agreed to in writing, software 9 | * distributed under the License is distributed on an "AS IS" BASIS, 10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the License for the specific language governing permissions and 12 | * limitations under the License. 13 | */ 14 | 15 | package com.facebook.coresql.linter.warning; 16 | 17 | import com.facebook.coresql.parser.AstNode; 18 | import com.google.common.collect.ImmutableList; 19 | import com.google.common.collect.LinkedListMultimap; 20 | import com.google.common.collect.Multimap; 21 | 22 | import java.util.List; 23 | 24 | import static java.util.Objects.requireNonNull; 25 | 26 | public class DefaultWarningCollector 27 | implements WarningCollector 28 | { 29 | private final Multimap warnings = LinkedListMultimap.create(); 30 | private final WarningCollectorConfig config; 31 | 32 | public DefaultWarningCollector(WarningCollectorConfig config) 33 | { 34 | this.config = requireNonNull(config, "config is null"); 35 | } 36 | 37 | @Override 38 | public void add(WarningCode code, String warningMessage, AstNode node) 39 | { 40 | requireNonNull(code, "warning code is null"); 41 | requireNonNull(warningMessage, "warning message is null"); 42 | requireNonNull(node, "node is null"); 43 | addWarningIfNumWarningsLessThanConfig(new CoreSqlWarning(code, 44 | warningMessage, 45 | node.beginToken.beginLine, 46 | node.beginToken.beginColumn, 47 | node.beginToken.endLine, 48 | node.beginToken.endColumn)); 49 | } 50 | 51 | @Override 52 | public List getAllWarnings() 53 | { 54 | return ImmutableList.copyOf(warnings.values()); 55 | } 56 | 57 | @Override 58 | public void clearWarnings() 59 | { 60 | warnings.clear(); 61 | } 62 | 63 | private void addWarningIfNumWarningsLessThanConfig(CoreSqlWarning coreSqlWarning) 64 | { 65 | if (warnings.size() < config.getMaxWarnings()) { 66 | warnings.put(coreSqlWarning.getWarningCode(), coreSqlWarning); 67 | } 68 | } 69 | } 70 | -------------------------------------------------------------------------------- /parser/cpp/AstNode.h: -------------------------------------------------------------------------------- 1 | #ifndef _ASTNODE_H_ 2 | #define _ASTNODE_H_ 3 | 4 | #include 5 | 6 | #include "Token.h" 7 | #include "Node.h" 8 | #include "SimpleNode.h" 9 | 10 | using std::string; 11 | 12 | namespace commonsql { 13 | namespace parser { 14 | 15 | class AstNode; 16 | 17 | class AstNode : public SimpleNode { 18 | private: 19 | int op; 20 | bool negated; 21 | 22 | public: 23 | Token* beginToken; 24 | Token* endToken; 25 | virtual ~AstNode() {} 26 | AstNode(int id) : SimpleNode(id) {} 27 | AstNode(SqlParser* parser, int id) : SimpleNode(parser, id) {} 28 | int NumChildren() const { return jjtGetNumChildren(); } 29 | AstNode* GetChild(int i) const { return static_cast(jjtGetChild(i)); } 30 | int Kind() const { return id; } 31 | JJString GetImage() const { return NumChildren() == 0 ? beginToken->image : ""; } 32 | JJString toString(const JJString& prefix) const { 33 | return SimpleNode::toString(prefix) + 34 | string(" (") + 35 | std::to_string(beginToken->beginLine) + 36 | string(":") + 37 | std::to_string(beginToken->beginColumn) + 38 | string(" - ") + 39 | std::to_string(endToken->endLine) + 40 | string(":") + 41 | std::to_string(endToken->endColumn) + 42 | string(")"); 43 | } 44 | 45 | // Other utility functions. 46 | bool IsNegatableOperator() 47 | { 48 | switch (Kind()) { 49 | case JJTBETWEEN: 50 | case JJTLIKE: 51 | case JJTINPREDICATE: 52 | case JJTISNULL: 53 | return true; 54 | default: 55 | return false; 56 | } 57 | } 58 | 59 | bool IsOperator() const 60 | { 61 | switch (Kind()) { 62 | case JJTADDITIVEEXPRESSION: 63 | case JJTMULTIPLICATIVEEXPRESSION: 64 | case JJTOREXPRESSION: 65 | case JJTANDEXPRESSION: 66 | case JJTNOTEXPRESSION: 67 | case JJTCOMPARISON: 68 | return true; 69 | default: 70 | return false; 71 | } 72 | } 73 | 74 | void SetOperator(int op) 75 | { 76 | this->op = op; 77 | } 78 | 79 | int GetOperator() const 80 | { 81 | assert(IsOperator()); 82 | return op; 83 | } 84 | 85 | JJString GetFunctionName() const 86 | { 87 | if (Kind() == JJTFUNCTIONCALL) { 88 | return GetChild(0)->GetImage(); 89 | } 90 | 91 | if (Kind() == JJTBUILTINFUNCTIONCALL) { 92 | return beginToken->image; 93 | } 94 | 95 | return ""; 96 | } 97 | 98 | void SetNegated(bool negated) { this->negated = negated; } 99 | bool IsNegated() const { return negated; } 100 | }; 101 | 102 | } 103 | } 104 | 105 | #endif 106 | -------------------------------------------------------------------------------- /parser/src/test/java/com/facebook/coresql/parser/TestKeywords.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed under the Apache License, Version 2.0 (the "License"); 3 | * you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at 5 | * 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * 8 | * Unless required by applicable law or agreed to in writing, software 9 | * distributed under the License is distributed on an "AS IS" BASIS, 10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the License for the specific language governing permissions and 12 | * limitations under the License. 13 | */ 14 | package com.facebook.coresql.parser; 15 | 16 | import org.testng.annotations.Test; 17 | 18 | import static com.facebook.coresql.parser.ParserHelper.parseStatement; 19 | import static com.facebook.coresql.parser.SqlParserConstants.MAX_NON_RESERVED_WORD; 20 | import static com.facebook.coresql.parser.SqlParserConstants.MAX_RESERVED_WORD; 21 | import static com.facebook.coresql.parser.SqlParserConstants.MIN_NON_RESERVED_WORD; 22 | import static com.facebook.coresql.parser.SqlParserConstants.MIN_RESERVED_WORD; 23 | import static com.facebook.coresql.parser.SqlParserConstants.tokenImage; 24 | import static org.testng.Assert.assertFalse; 25 | import static org.testng.Assert.assertNotNull; 26 | import static org.testng.Assert.assertNull; 27 | 28 | public class TestKeywords 29 | { 30 | private static String getTokenImageWithoutQuotes(int kind) 31 | { 32 | return tokenImage[kind].substring(1, tokenImage[kind].length() - 1); 33 | } 34 | 35 | @Test 36 | public void testReservedWords() 37 | { 38 | for (int i = MIN_RESERVED_WORD + 1; i < MAX_RESERVED_WORD; i++) { 39 | assertNull(parseStatement("select 1 as " + getTokenImageWithoutQuotes(i) + ";"), 40 | getTokenImageWithoutQuotes(i) + " should be a reserved word."); 41 | } 42 | } 43 | 44 | @Test 45 | public void testNonReservedWords() 46 | { 47 | // The last one is a weird one - "COUNT" so we ignore it for now. 48 | for (int i = MIN_NON_RESERVED_WORD + 1; i < MAX_NON_RESERVED_WORD - 1; i++) { 49 | assertNotNull(parseStatement("select 1 as " + getTokenImageWithoutQuotes(i) + ";"), 50 | getTokenImageWithoutQuotes(i) + " should NOT be a reserved word."); 51 | } 52 | } 53 | 54 | @Test 55 | public void testNoExtraneousKeywords() 56 | { 57 | for (int i = MAX_RESERVED_WORD + 1; i < tokenImage.length; i++) { 58 | // All string literal tokens start with quote and if it's a keyword first char is a letter (somewhat loosely). 59 | assertFalse( 60 | tokenImage[i].charAt(0) == '"' && Character.isLetter(tokenImage[i].charAt(1)), 61 | tokenImage[i] + " should be in one of the reserved word or non-reserved word group of tokens."); 62 | } 63 | } 64 | } 65 | -------------------------------------------------------------------------------- /parser/src/test/java/com/facebook/coresql/parser/sqllogictest/java/SqlLogicTestConfig.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed under the Apache License, Version 2.0 (the "License"); 3 | * you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at 5 | * 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * 8 | * Unless required by applicable law or agreed to in writing, software 9 | * distributed under the License is distributed on an "AS IS" BASIS, 10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the License for the specific language governing permissions and 12 | * limitations under the License. 13 | */ 14 | 15 | package com.facebook.coresql.parser.sqllogictest.java; 16 | 17 | import com.facebook.airlift.configuration.Config; 18 | import com.facebook.airlift.configuration.ConfigDescription; 19 | import com.facebook.coresql.parser.sqllogictest.java.SqlLogicTest.DatabaseDialect; 20 | 21 | import javax.validation.constraints.Min; 22 | import javax.validation.constraints.NotNull; 23 | 24 | import java.nio.file.Path; 25 | import java.nio.file.Paths; 26 | 27 | import static com.facebook.coresql.parser.sqllogictest.java.SqlLogicTest.DatabaseDialect.ALL; 28 | 29 | public class SqlLogicTestConfig 30 | { 31 | private Path testFilesRootPath = Paths.get("src/test/java/com/facebook/coresql/parser/sqllogictest/resources/tests"); 32 | private Path resultPath = Paths.get("src/test/java/com/facebook/coresql/parser/sqllogictest/resources"); 33 | private DatabaseDialect databaseDialect = ALL; 34 | private int maxNumFilesToVisit = 10; 35 | 36 | @Config("test-files-root-path") 37 | public SqlLogicTestConfig setTestFilesRootPath(String testFilesRootPath) 38 | { 39 | this.testFilesRootPath = Paths.get(testFilesRootPath); 40 | return this; 41 | } 42 | 43 | @NotNull 44 | public Path getTestFilesRootPath() 45 | { 46 | return testFilesRootPath; 47 | } 48 | 49 | @Config("result-path") 50 | public SqlLogicTestConfig setResultPath(String resultPath) 51 | { 52 | this.resultPath = Paths.get(resultPath); 53 | return this; 54 | } 55 | 56 | @NotNull 57 | public Path getResultPath() 58 | { 59 | return resultPath; 60 | } 61 | 62 | @Config("max-num-files-to-visit") 63 | @ConfigDescription("The max number of test files to test against") 64 | public SqlLogicTestConfig setMaxNumFilesToVisit(int maxNumFilesToVisit) 65 | { 66 | this.maxNumFilesToVisit = maxNumFilesToVisit; 67 | return this; 68 | } 69 | 70 | @Min(1) 71 | public int getMaxNumFilesToVisit() 72 | { 73 | return maxNumFilesToVisit; 74 | } 75 | 76 | @Config("database-dialect") 77 | @ConfigDescription("The database dialect to filter test queries over") 78 | public SqlLogicTestConfig setDatabaseDialect(DatabaseDialect databaseDialect) 79 | { 80 | this.databaseDialect = databaseDialect; 81 | return this; 82 | } 83 | 84 | @NotNull 85 | public DatabaseDialect getDatabaseDialect() 86 | { 87 | return databaseDialect; 88 | } 89 | } 90 | -------------------------------------------------------------------------------- /parser/src/test/java/com/facebook/coresql/parser/sqllogictest/java/SqlLogicTestStatement.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed under the Apache License, Version 2.0 (the "License"); 3 | * you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at 5 | * 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * 8 | * Unless required by applicable law or agreed to in writing, software 9 | * distributed under the License is distributed on an "AS IS" BASIS, 10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the License for the specific language governing permissions and 12 | * limitations under the License. 13 | */ 14 | package com.facebook.coresql.parser.sqllogictest.java; 15 | 16 | import com.facebook.coresql.parser.sqllogictest.java.SqlLogicTest.DatabaseDialect; 17 | 18 | import java.util.HashSet; 19 | import java.util.Optional; 20 | import java.util.Set; 21 | 22 | import static com.facebook.coresql.parser.sqllogictest.java.SqlLogicTest.CoreSqlParsingError; 23 | import static java.util.Objects.requireNonNull; 24 | 25 | public class SqlLogicTestStatement 26 | { 27 | private final Optional onlyIfDialect; 28 | private final Optional error; 29 | private final Set skipIfSet; 30 | private final String statement; 31 | 32 | private SqlLogicTestStatement(Builder builder) 33 | { 34 | this.onlyIfDialect = builder.onlyIfDialect; 35 | this.error = builder.error; 36 | this.skipIfSet = builder.skipIfSet; 37 | this.statement = builder.statement; 38 | } 39 | 40 | public static Builder builder() 41 | { 42 | return new Builder(); 43 | } 44 | 45 | public Optional getCurrStatementsOnlyIfDialect() 46 | { 47 | return onlyIfDialect; 48 | } 49 | 50 | public Optional getError() 51 | { 52 | return error; 53 | } 54 | 55 | public Set getSkipIfSet() 56 | { 57 | return skipIfSet; 58 | } 59 | 60 | public String getStatement() 61 | { 62 | return statement; 63 | } 64 | 65 | public static class Builder 66 | { 67 | private Optional onlyIfDialect; 68 | private Optional error; 69 | private final Set skipIfSet; 70 | private String statement; 71 | 72 | public Builder() 73 | { 74 | this.onlyIfDialect = Optional.empty(); 75 | this.error = Optional.empty(); 76 | this.skipIfSet = new HashSet<>(); 77 | } 78 | 79 | public SqlLogicTestStatement build() 80 | { 81 | return new SqlLogicTestStatement(this); 82 | } 83 | 84 | public void setOnlyIfDialect(DatabaseDialect onlyIfDialect) 85 | { 86 | this.onlyIfDialect = Optional.of(onlyIfDialect); 87 | } 88 | 89 | public void addDialectToSkipIfSet(DatabaseDialect dialect) 90 | { 91 | this.skipIfSet.add(requireNonNull(dialect)); 92 | } 93 | 94 | public Builder setError(Optional error) 95 | { 96 | this.error = error; 97 | return this; 98 | } 99 | 100 | public Builder setStatement(String statement) 101 | { 102 | this.statement = requireNonNull(statement); 103 | return this; 104 | } 105 | } 106 | } 107 | -------------------------------------------------------------------------------- /parser/grammar/presto-extensions.txt: -------------------------------------------------------------------------------- 1 | void use_statement() #UseStatement: 2 | {} 3 | { 4 | "USE" identifier_chain() 5 | } 6 | 7 | void lambda() #Lambda(2): 8 | {} 9 | { 10 | lambda_params() lambda_body() 11 | } 12 | 13 | void lambda_body() #LambdaBody: 14 | {} 15 | { 16 | "->" value_expression() 17 | } 18 | 19 | void lambda_params() #LambdaParams: 20 | {} 21 | { 22 | ( actual_identifier() )#LambdaParam(0) 23 | | "(" [ ( actual_identifier() )#LambdaParam(0) ( "," ( actual_identifier() #LambdaParam(0) ) )* ] ")" 24 | } 25 | 26 | void if_not_exists(): 27 | {} 28 | { 29 | "IF" "NOT" "EXISTS" 30 | } 31 | 32 | void identifier_suffix_chain(): 33 | {} 34 | { 35 | ( ( "@" | ":" ) [ actual_identifier() ] )+ 36 | } 37 | 38 | void limit_clause() #LimitClause: 39 | {} 40 | { 41 | "LIMIT" ( | "ALL" ) 42 | } 43 | 44 | void presto_generic_type(): 45 | {} 46 | { 47 | presto_array_type() 48 | | presto_map_type() 49 | | ( "(" data_type() ( "," data_type() )* ")" )#ParameterizedType 50 | } 51 | 52 | void presto_array_type() #ArrayType(): 53 | {} 54 | { 55 | "ARRAY" "<" data_type() ">" // Non-standard 56 | | "ARRAY" "(" data_type() ")" // Non-standard 57 | } 58 | 59 | void presto_map_type() #MapType(): 60 | {} 61 | { 62 | "MAP" "<" data_type() "," data_type() ">" // Non-standard 63 | | "MAP" "(" data_type() "," data_type() ")" // Non-standard 64 | } 65 | 66 | void percent_operator(): 67 | {} 68 | { 69 | 70 | } 71 | 72 | void distinct(): 73 | {} 74 | { 75 | "DISTINCT" 76 | } 77 | 78 | void grouping_expression(): 79 | {} 80 | { 81 | value_expression() 82 | } 83 | 84 | void count(): 85 | {} 86 | { 87 | "COUNT" "(" ")" 88 | | "\"COUNT\"" "(" [ set_quantifier() ] [ value_expression() | "*" ] ")" // Just weird 89 | } 90 | 91 | void table_description(): 92 | {} 93 | { 94 | "COMMENT" character_string_literal() 95 | } 96 | 97 | void routine_description(): 98 | {} 99 | { 100 | "COMMENT" character_string_literal() 101 | } 102 | 103 | void column_description(): 104 | {} 105 | { 106 | "COMMENT" character_string_literal() 107 | } 108 | 109 | void presto_aggregation_function(): 110 | {} 111 | { 112 | "NUMERIC_HISTOGRAM" 113 | | "HISTOGRAM" 114 | | "APPROEX_PERCENTILE" 115 | | "MAP_AGG" 116 | | "SET_AGG" 117 | | "MAP_UNION" 118 | } 119 | 120 | void presto_aggregations(): 121 | {} 122 | { 123 | presto_aggregation_function() 124 | "(" [ [ set_quantifier() ] value_expression() ( "," value_expression() )* ] ")" 125 | } 126 | 127 | void try_cast() #TryExpression: 128 | {} 129 | { 130 | "TRY_CAST" ( "(" cast_operand() "AS" cast_target() ")" )#CastExpression 131 | } 132 | 133 | void varbinary(): 134 | {} 135 | { 136 | "VARBINARY" 137 | } 138 | 139 | void table_attributes(): 140 | {} 141 | { 142 | "(" actual_identifier() "=" value_expression() ( "," actual_identifier() "=" value_expression() )* ")" // Non-standard 143 | } 144 | 145 | void or_replace(): 146 | {} 147 | { 148 | "OR" "REPLACE" 149 | } 150 | 151 | void udaf_filter(): 152 | {} 153 | { 154 | filter_clause() 155 | } 156 | 157 | void extra_args_to_agg(): 158 | {} 159 | { 160 | ( "," value_expression() )+ 161 | } 162 | 163 | void weird_identifiers(): 164 | {} 165 | { 166 | "_" 167 | } 168 | 169 | TOKEN: 170 | { 171 | )? > { setKindToIdentifier(matchedToken); } 172 | | { setUnicodeLiteralType(matchedToken); } 173 | } 174 | -------------------------------------------------------------------------------- /parser/cpp/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4.0.0 4 | 5 | com.facebook.presto 6 | presto-coresql-parser-cpp 7 | presto-coresql-parser-cpp 8 | 0.1-SNAPSHOT 9 | 10 | 11 | 12 | 13 | org.codehaus.mojo 14 | exec-maven-plugin 15 | 1.6.0 16 | 17 | 18 | Build-parser 19 | generate-sources 20 | 21 | exec 22 | 23 | 24 | 25 | 26 | ./prepare-javacc-grammar.sh 27 | . 28 | 29 | 30 | 31 | org.javacc.plugin 32 | javacc-maven-plugin 33 | 3.0.2 34 | 35 | 36 | jcc7jcc 37 | generate-sources 38 | 39 | jjtree 40 | 41 | 42 | target/generated-sources/javacc 43 | c++ 44 | target/generated-sources/javacc 45 | 46 | 47 | 48 | 49 | jcc7javacc 50 | generate-sources 51 | 52 | javacc 53 | 54 | 55 | target/generated-sources/javacc 56 | c++ 57 | target/generated-sources/javacc 58 | 59 | 60 | 61 | 62 | 63 | 64 | 65 | 66 | 67 | org.javacc.plugin 68 | javacc-maven-plugin 69 | 70 | 71 | net.java.dev.javacc 72 | javacc 73 | 7.0.10 74 | runtime 75 | 76 | 77 | 78 | 79 | org.gaul 80 | modernizer-maven-plugin 81 | 2.1.0 82 | 83 | 1.8 84 | com.facebook.coresql.parser 85 | 86 | 87 | 88 | 89 | 90 | 91 | -------------------------------------------------------------------------------- /linter/src/test/java/com/facebook/coresql/linter/lint/TestMixedAndOr.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed under the Apache License, Version 2.0 (the "License"); 3 | * you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at 5 | * 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * 8 | * Unless required by applicable law or agreed to in writing, software 9 | * distributed under the License is distributed on an "AS IS" BASIS, 10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the License for the specific language governing permissions and 12 | * limitations under the License. 13 | */ 14 | package com.facebook.coresql.linter.lint; 15 | 16 | import com.facebook.coresql.linter.warning.DefaultWarningCollector; 17 | import com.facebook.coresql.linter.warning.WarningCollectorConfig; 18 | import com.facebook.coresql.parser.AstNode; 19 | import org.testng.annotations.Test; 20 | 21 | import static com.facebook.coresql.linter.warning.StandardWarningCode.MIXING_AND_OR_WITHOUT_PARENTHESES; 22 | import static com.facebook.coresql.parser.ParserHelper.parseStatement; 23 | import static org.testng.Assert.assertEquals; 24 | 25 | public class TestMixedAndOr 26 | { 27 | private static final LintingVisitor LINTING_VISITOR = new MixedAndOr(new DefaultWarningCollector(new WarningCollectorConfig().setMaxWarnings(1))); 28 | private static final String[] NON_WARNING_SQL_STRINGS = new String[] { 29 | "SELECT (true or false) and false;", 30 | "SELECT true or false or true;", 31 | "SELECT true and false and false;", 32 | "SELECT a FROM T WHERE a.id = 2 or (a.id = 3 and a.age = 73);", 33 | "SELECT a FROM T WHERE (a.id = 2 or a.id = 3) and (a.age = 73 or a.age = 100);", 34 | "SELECT * from Evaluation e JOIN Value v ON e.CaseNum = v.CaseNum\n" + 35 | " AND e.FileNum = v.FileNum AND e.ActivityNum = v.ActivityNum;", 36 | "use a.b;", 37 | " SELECT 1;", 38 | "SELECT a FROM T;", 39 | "SELECT a FROM T WHERE p1 > p2;", 40 | "SELECT a, b, c FROM T WHERE c1 < c2 and c3 < c4;", 41 | "SELECT CASE a WHEN IN ( 1 ) THEN b ELSE c END AS x, b, c FROM T WHERE c1 < c2 and c3 < c4;", 42 | "SELECT T.* FROM T JOIN W ON T.x = W.x;", 43 | "SELECT NULL;", 44 | "SELECT ARRAY[x] FROM T;", 45 | "SELECT TRANSFORM(ARRAY[x], x -> x + 2) AS arra FROM T;", 46 | "CREATE TABLE T AS SELECT TRANSFORM(ARRAY[x], x -> x + 2) AS arra FROM T;", 47 | "INSERT INTO T SELECT TRANSFORM(ARRAY[x], x -> x + 2) AS arra FROM T;", 48 | "SELECT ROW_NUMBER() OVER(PARTITION BY x) FROM T;", 49 | "SELECT x, SUM(y) OVER (PARTITION BY y ORDER BY 1) AS min\n" + 50 | "FROM (values ('b',10), ('a', 10)) AS T(x, y)\n;", 51 | "SELECT\n" + 52 | " CAST(MAP() AS map>) AS \"bool_tensor_features\";", 53 | "SELECT f(f(f(f(f(f(f(f(f(f(f(f(f(f(f(f(f(f(f(f(f(f(f(f(f(f(f(f(f(f())))))))))))))))))))))))))))));", 54 | "SELECT abs, 2 as abs;", 55 | }; 56 | 57 | private static final String[] WARNING_SQL_STRINGS = new String[] { 58 | "SELECT true or false and false;", 59 | "SELECT a FROM T WHERE a.id = 2 or a.id = 3 and a.age = 73;", 60 | "SELECT a FROM T WHERE (a.id = 2 or a.id = 3) and a.age = 73 or a.age = 100;" 61 | }; 62 | 63 | private static void assertHasMixedAndOrWarnings(String statement, int expectedNumWarnings) 64 | { 65 | AstNode ast = parseStatement(statement); 66 | LINTING_VISITOR.lint(ast); 67 | assertEquals(LINTING_VISITOR.getWarningCollector().getAllWarnings().size(), expectedNumWarnings); 68 | LINTING_VISITOR.getWarningCollector().getAllWarnings().forEach(x -> 69 | assertEquals(x.getWarningCode(), MIXING_AND_OR_WITHOUT_PARENTHESES.getWarningCode())); 70 | } 71 | 72 | @Test 73 | public void testDoesntThrowsMixedAndOrWarning() 74 | { 75 | LINTING_VISITOR.getWarningCollector().clearWarnings(); 76 | for (String sql : NON_WARNING_SQL_STRINGS) { 77 | assertHasMixedAndOrWarnings(sql, 0); 78 | } 79 | } 80 | 81 | @Test 82 | public void testThrowsMixedAndOrWarning() 83 | { 84 | LINTING_VISITOR.getWarningCollector().clearWarnings(); 85 | for (String sql : WARNING_SQL_STRINGS) { 86 | assertHasMixedAndOrWarnings(sql, 1); 87 | } 88 | } 89 | } 90 | -------------------------------------------------------------------------------- /parser/src/test/java/com/facebook/coresql/parser/TestSqlParser.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed under the Apache License, Version 2.0 (the "License"); 3 | * you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at 5 | * 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * 8 | * Unless required by applicable law or agreed to in writing, software 9 | * distributed under the License is distributed on an "AS IS" BASIS, 10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the License for the specific language governing permissions and 12 | * limitations under the License. 13 | */ 14 | package com.facebook.coresql.parser; 15 | 16 | import com.facebook.coresql.parser.sqllogictest.java.SqlLogicTest; 17 | import org.testng.annotations.Test; 18 | 19 | import java.io.IOException; 20 | 21 | import static com.facebook.coresql.parser.ParserHelper.parseExpression; 22 | import static com.facebook.coresql.parser.ParserHelper.parseStatement; 23 | import static com.facebook.coresql.parser.SqlParserConstants.LESS_THAN; 24 | import static com.facebook.coresql.parser.SqlParserConstants.NOT; 25 | import static com.facebook.coresql.parser.SqlParserConstants.PLUS; 26 | import static com.facebook.coresql.parser.SqlParserConstants.SQRT; 27 | import static com.facebook.coresql.parser.SqlParserConstants.tokenImage; 28 | import static com.facebook.coresql.parser.Unparser.unparse; 29 | import static org.testng.Assert.assertEquals; 30 | import static org.testng.Assert.assertNotNull; 31 | 32 | public class TestSqlParser 33 | { 34 | private static final String[] TEST_SQL_TESTSTRINGS = new String[] { 35 | "use a.b;", 36 | " SELECT 1;", 37 | "SELECT a FROM T;", 38 | "SELECT a FROM T WHERE p1 > p2;", 39 | "SELECT a, b, c FROM T WHERE c1 < c2 and c3 < c4;", 40 | "SELECT CASE a WHEN IN ( 1 ) THEN b ELSE c END AS x, b, c FROM T WHERE c1 < c2 and c3 < c4;", 41 | "SELECT T.* FROM T JOIN W ON T.x = W.x;", 42 | "SELECT NULL;", 43 | "SELECT ARRAY[x] FROM T;", 44 | "SELECT TRANSFORM(ARRAY[x], x -> x + 2) AS arra FROM T;", 45 | "CREATE TABLE T AS SELECT TRANSFORM(ARRAY[x], x -> x + 2) AS arra FROM T;", 46 | "INSERT INTO T SELECT TRANSFORM(ARRAY[x], x -> x + 2) AS arra FROM T;", 47 | "SELECT ROW_NUMBER() OVER(PARTITION BY x) FROM T;", 48 | "SELECT x, SUM(y) OVER (PARTITION BY y ORDER BY 1) AS min\n" + 49 | "FROM (values ('b',10), ('a', 10)) AS T(x, y)\n;", 50 | "SELECT\n" + 51 | " CAST(MAP() AS map>) AS \"bool_tensor_features\";", 52 | "SELECT f(f(f(f(f(f(f(f(f(f(f(f(f(f(f(f(f(f(f(f(f(f(f(f(f(f(f(f(f(f())))))))))))))))))))))))))))));", 53 | "SELECT abs, 2 as abs;", 54 | "SELECT sqrt(x), power(y, 5), myFunction('a') FROM T;", 55 | }; 56 | 57 | private AstNode parse(String sql) 58 | { 59 | return parseStatement(sql); 60 | } 61 | 62 | @Test 63 | public void smokeTest() 64 | { 65 | for (String sql : TEST_SQL_TESTSTRINGS) { 66 | assertNotNull(parse(sql)); 67 | } 68 | } 69 | 70 | @Test 71 | public void parseUnparseTest() 72 | { 73 | for (String sql : TEST_SQL_TESTSTRINGS) { 74 | System.out.println(sql); 75 | AstNode ast = parse(sql); 76 | assertNotNull(ast); 77 | assertEquals(sql.trim(), unparse(ast).trim()); 78 | } 79 | } 80 | 81 | @Test 82 | public void sqlLogicTest() 83 | throws IOException 84 | { 85 | SqlLogicTest.execute(); 86 | } 87 | 88 | @Test 89 | public void testGetOperator() 90 | { 91 | assertEquals(parseExpression("x + 10").GetOperator(), PLUS); 92 | assertEquals(parseExpression("x < /*comment*/ 10").GetOperator(), LESS_THAN); 93 | assertEquals(parseExpression("NOT x").GetOperator(), NOT); 94 | } 95 | 96 | @Test 97 | public void testGetFunctionName() 98 | { 99 | assertEquals(parseExpression("SQRT(10)").GetFunctionName(), tokenImage[SQRT].substring(1, tokenImage[SQRT].length() - 1)); 100 | assertEquals(parseExpression("POW(x, 2)").GetFunctionName(), "POW"); 101 | assertEquals(parseExpression("PoW(x, 2)").GetFunctionName(), "PoW"); 102 | assertEquals(parseExpression("MyFunction('a')").GetFunctionName(), "MyFunction"); 103 | } 104 | 105 | @Test 106 | public void testIsNegated() 107 | { 108 | assertEquals(parseExpression("a LIKE B").IsNegated(), false); 109 | assertEquals(parseExpression("a NOT LIKE B").IsNegated(), true); 110 | assertEquals(parseExpression("a IS NULL").IsNegated(), false); 111 | assertEquals(parseExpression("a IS NOT NULL").IsNegated(), true); 112 | } 113 | } 114 | -------------------------------------------------------------------------------- /parser/grammar/javacc-options-java.txt: -------------------------------------------------------------------------------- 1 | options { 2 | STATIC = false; 3 | LOOKAHEAD=3; 4 | IGNORE_CASE=true; 5 | UNICODE_INPUT=true; 6 | ERROR_REPORTING=false; 7 | NODE_DEFAULT_VOID = true; 8 | NODE_SCOPE_HOOK = true; 9 | NODE_CLASS = "AstNode"; 10 | NODE_PREFIX = ""; 11 | MULTI = true; 12 | VISITOR = true; 13 | VISITOR_RETURN_TYPE = "void"; 14 | VISITOR_DATA_TYPE = "Void"; 15 | } 16 | 17 | PARSER_BEGIN(SqlParser) 18 | package com.facebook.coresql.parser; 19 | 20 | public class SqlParser { 21 | private boolean IsIdNonReservedWord() { 22 | int kind = getToken(1).kind; 23 | if (kind == regular_identifier || kind == delimited_identifier || kind == Unicode_delimited_identifier) return true; 24 | 25 | if (!(kind >= MIN_NON_RESERVED_WORD && kind <= MAX_NON_RESERVED_WORD)) return false; // Not a nonreserved word. 26 | 27 | // Some special cases. 28 | switch (kind) { 29 | // Some contextual keywords 30 | case GROUP: 31 | case ORDER: 32 | case PARTITION: 33 | return getToken(2).kind != BY; 34 | 35 | case LIMIT: 36 | return getToken(2).kind != unsigned_integer; 37 | 38 | case ROWS: 39 | return getToken(2).kind != BETWEEN; 40 | 41 | // Some builtin functions 42 | case TRIM: 43 | case POSITION: 44 | case MOD: 45 | case POWER: 46 | case RANK: 47 | case ROW_NUMBER: 48 | case FLOOR: 49 | case MIN: 50 | case MAX: 51 | case UPPER: 52 | case LOWER: 53 | case CARDINALITY: 54 | case ABS: 55 | return getToken(2).kind != lparen; 56 | 57 | default: 58 | return true; 59 | } 60 | } 61 | 62 | private boolean SyncToSemicolon() { 63 | while (getToken(1).kind != EOF && getToken(1).kind != SqlParserConstants.semicolon) getNextToken(); 64 | 65 | if (getToken(1).kind == semicolon) { 66 | getNextToken(); 67 | } 68 | 69 | return true; 70 | } 71 | 72 | private boolean NotEof() { 73 | return getToken(1).kind != EOF; 74 | } 75 | 76 | public void PushNode(Node node) { jjtree.pushNode(node); } 77 | public Node PopNode() { return jjtree.popNode(); } 78 | 79 | void jjtreeOpenNodeScope(Node node) { 80 | ((AstNode)node).beginToken = getToken(1); 81 | } 82 | 83 | void jjtreeCloseNodeScope(Node node) { 84 | AstNode astNode = ((AstNode)node); 85 | astNode.endToken = getToken(0); 86 | Token t = astNode.beginToken; 87 | 88 | // For some nodes, the node is opened after some children are already created. Reset the begin for those to be 89 | // the begin of the left-most child. 90 | if (astNode.NumChildren() > 0) { 91 | Token t0 = astNode.GetChild(0).beginToken; 92 | if (t0.beginLine < t.beginLine || (t0.beginLine == t.beginLine && t0.beginColumn < t.beginColumn)) { 93 | astNode.beginToken = t0; 94 | } 95 | } 96 | 97 | if (astNode.IsNegatableOperator()) { 98 | Token t1 = astNode.GetChild(0).endToken; 99 | 100 | if (astNode.Kind() == JJTISNULL) { 101 | // IsNull -- see if the penultimate token is NOT 102 | while (t1 != null && t1.kind != IS) { 103 | t1 = t1.next; 104 | } 105 | 106 | if (t1.next.kind == NOT) { 107 | astNode.SetNegated(true); 108 | } 109 | } 110 | else if (astNode.NumChildren() > 1) { 111 | Token t2 = astNode.GetChild(1).beginToken; 112 | while (t1.next != null && t1.next != t2) { 113 | if (t1.kind == NOT) { 114 | astNode.SetNegated(true); 115 | break; 116 | } 117 | t1 = t1.next; 118 | } 119 | } 120 | } 121 | else if (astNode.NumChildren() == 2 && astNode.IsOperator()) { 122 | // Hack locate the token just before the first token of the second operator 123 | Token t1 = astNode.GetChild(0).endToken; 124 | Token t2 = astNode.GetChild(1).beginToken; 125 | while (t1.next != null && t1.next != t2) { 126 | t1 = t1.next; 127 | } 128 | astNode.SetOperator(t1.kind); 129 | } 130 | else if (astNode.NumChildren() == 1 && astNode.IsOperator()) { 131 | astNode.SetOperator(astNode.beginToken.kind); 132 | } 133 | } 134 | 135 | public AstNode getResult() 136 | { 137 | return (AstNode) jjtree.popNode(); 138 | } 139 | } 140 | 141 | PARSER_END(SqlParser) 142 | 143 | TOKEN_MGR_DECLS: 144 | { 145 | void setKindToIdentifier(Token t) { 146 | t.kind = regular_identifier; 147 | } 148 | 149 | void setUnicodeLiteralType(Token t) { 150 | t.kind = unicode_literal; 151 | } 152 | 153 | void StoreImage(Token matchedToken) { 154 | matchedToken.image = image.toString(); 155 | } 156 | } 157 | 158 | // Temporary entry point 159 | Node CompilationUnit() #CompilationUnit: 160 | {} 161 | { 162 | ( 163 | LOOKAHEAD({ NotEof() }) 164 | try { 165 | direct_SQL_statement() 166 | } catch(ParseException pe) { 167 | System.err.println("Parse error: " + getToken(1).beginLine + ":" + getToken(1).beginColumn + " at token: " + getToken(1).image); 168 | SyncToSemicolon(); 169 | } 170 | )* 171 | 172 | 173 | 174 | { return jjtThis; } 175 | } 176 | -------------------------------------------------------------------------------- /rewriter/src/main/java/com/facebook/coresql/rewriter/OrderByRewriter.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed under the Apache License, Version 2.0 (the "License"); 3 | * you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at 5 | * 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * 8 | * Unless required by applicable law or agreed to in writing, software 9 | * distributed under the License is distributed on an "AS IS" BASIS, 10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the License for the specific language governing permissions and 12 | * limitations under the License. 13 | */ 14 | 15 | package com.facebook.coresql.rewriter; 16 | 17 | import com.facebook.coresql.parser.AstNode; 18 | import com.facebook.coresql.parser.OrderByClause; 19 | import com.facebook.coresql.parser.QuerySpecification; 20 | import com.facebook.coresql.parser.SimpleNode; 21 | import com.facebook.coresql.parser.SqlParserDefaultVisitor; 22 | import com.facebook.coresql.parser.Subquery; 23 | import com.facebook.coresql.parser.Unparser; 24 | import com.google.common.collect.ImmutableSet; 25 | 26 | import java.util.Optional; 27 | import java.util.Set; 28 | 29 | import static com.facebook.coresql.parser.SqlParserTreeConstants.JJTINSERT; 30 | import static com.facebook.coresql.parser.SqlParserTreeConstants.JJTLIMITCLAUSE; 31 | import static com.facebook.coresql.parser.SqlParserTreeConstants.JJTORDERBYCLAUSE; 32 | import static com.facebook.coresql.parser.SqlParserTreeConstants.JJTTABLEDEFINITION; 33 | import static java.util.Objects.requireNonNull; 34 | 35 | public class OrderByRewriter 36 | extends Rewriter 37 | { 38 | private final AstNode root; 39 | private final Set patternMatchedNodes; 40 | private static final String REWRITE_NAME = "ORDER BY without LIMIT"; 41 | 42 | public OrderByRewriter(AstNode root) 43 | { 44 | this.root = requireNonNull(root, "AST passed to rewriter was null"); 45 | this.patternMatchedNodes = new OrderByPatternMatcher(root).matchPattern(); 46 | } 47 | 48 | @Override 49 | public Optional rewrite() 50 | { 51 | if (patternMatchedNodes.isEmpty()) { 52 | return Optional.empty(); 53 | } 54 | String rewrittenSql = Unparser.unparse(root, this); 55 | return Optional.of(new RewriteResult(REWRITE_NAME, rewrittenSql)); 56 | } 57 | 58 | @Override 59 | public void visit(OrderByClause node, Void data) 60 | { 61 | if (patternMatchedNodes.contains(node)) { 62 | unparseUpto(node); 63 | moveToEndOfNode(node); 64 | } 65 | } 66 | 67 | private static class OrderByPatternMatcher 68 | extends SqlParserDefaultVisitor 69 | { 70 | private final AstNode root; 71 | private final ImmutableSet.Builder builder = ImmutableSet.builder(); 72 | private int depth; 73 | private static final int MINIMUM_SUBQUERY_DEPTH = 2; // Past this depth, all queries we encounter are subqueries 74 | 75 | public OrderByPatternMatcher(AstNode root) 76 | { 77 | this.root = requireNonNull(root, "AST passed to rewriter was null"); 78 | } 79 | 80 | public Set matchPattern() 81 | { 82 | root.jjtAccept(this, null); 83 | return builder.build(); 84 | } 85 | 86 | /** 87 | * Checks for a specific pattern in the AST this pattern matcher is traversing: 88 | * 1. Query has an ORDER BY 89 | * 2. Query does not have a LIMIT 90 | * 3. The query is a subquery OR the result of the query is used to CREATE a table or INSERT into a table 91 | * 92 | * @param node The node we're currently visiting 93 | * @return True if the pattern is matched at the current node, else false 94 | */ 95 | private boolean hasOrderByWithNoLimit(AstNode node) 96 | { 97 | if (depth > MINIMUM_SUBQUERY_DEPTH) { 98 | return hasOrderByAndDoesNotHaveLimit(node); 99 | } 100 | return hasOrderByAndDoesNotHaveLimit(node) && isUsedInInsertOrCreate(node); // Non-subquery check 101 | } 102 | 103 | private boolean hasOrderByAndDoesNotHaveLimit(AstNode node) 104 | { 105 | return node.hasChildOfKind(JJTORDERBYCLAUSE) && !node.hasChildOfKind(JJTLIMITCLAUSE); 106 | } 107 | 108 | private boolean isUsedInInsertOrCreate(AstNode node) 109 | { 110 | return (node.jjtGetParent().getId() == JJTTABLEDEFINITION || node.jjtGetParent().getId() == JJTINSERT); 111 | } 112 | 113 | @Override 114 | public void defaultVisit(SimpleNode node, Void data) 115 | { 116 | ++depth; 117 | node.childrenAccept(this, data); 118 | --depth; 119 | } 120 | 121 | @Override 122 | public void visit(QuerySpecification node, Void data) 123 | { 124 | if (hasOrderByWithNoLimit(node)) { 125 | builder.add(node.GetFirstChildOfKind(JJTORDERBYCLAUSE)); 126 | } 127 | defaultVisit(node, data); 128 | } 129 | 130 | @Override 131 | public void visit(Subquery node, Void data) 132 | { 133 | if (hasOrderByWithNoLimit(node)) { 134 | builder.add(node.GetFirstChildOfKind(JJTORDERBYCLAUSE)); 135 | } 136 | defaultVisit(node, data); 137 | } 138 | } 139 | } 140 | -------------------------------------------------------------------------------- /parser/cpp/TARGETS: -------------------------------------------------------------------------------- 1 | load("@fbcode_macros//build_defs:python_binary.bzl", "python_binary") 2 | load("@fbcode_macros//build_defs:python_library.bzl", "python_library") 3 | load("@fbcode_macros//build_defs:cpp_binary.bzl", "cpp_binary") 4 | load("@fbcode_macros//build_defs:cpp_library.bzl", "cpp_library") 5 | load("@fbcode_macros//build_defs:custom_rule.bzl", "custom_rule") 6 | load("@fbcode_macros//build_defs:native_rules.bzl", "buck_sh_binary") 7 | 8 | buck_sh_binary( 9 | name = "generate_parser", 10 | main = "generate_parser.sh" 11 | ) 12 | 13 | GENERATED_FILES = [ 14 | "gen/SimpleNode.cc", 15 | "gen/IntervalLiteral.cc", 16 | "gen/QuantifiedComparison.cc", 17 | "gen/Values.cc", 18 | "gen/WhereClause.cc", 19 | "gen/SecondField.cc", 20 | "gen/AndExpression.cc", 21 | "gen/UseStatement.cc", 22 | "gen/InvalueList.cc", 23 | "gen/FilterClause.cc", 24 | "gen/MultiplicativeExpression.cc", 25 | "gen/CompilationUnit.cc", 26 | "gen/CharStringLiteral.cc", 27 | "gen/TimestampLiteral.cc", 28 | "gen/UsingClause.cc", 29 | "gen/CurrentRow.cc", 30 | "gen/NonSecondDateTimeField.cc", 31 | "gen/SchemaQualifiedName.cc", 32 | "gen/InPredicate.cc", 33 | "gen/ArrayLiteral.cc", 34 | "gen/AliasedTable.cc", 35 | "gen/WhenOperand.cc", 36 | "gen/LanguageClause.cc", 37 | "gen/ElseClause.cc", 38 | "gen/Concatenation.cc", 39 | "gen/RowExression.cc", 40 | "gen/WindowFrameBetween.cc", 41 | "gen/NullLiteral.cc", 42 | "gen/WindowFrameFollowing.cc", 43 | "gen/TimeLiteral.cc", 44 | "gen/OrderingDirection.cc", 45 | "gen/BuiltinValue.cc", 46 | "gen/Subquery.cc", 47 | "gen/QuerySpecification.cc", 48 | "gen/OrderByClause.cc", 49 | "gen/LambdaParams.cc", 50 | "gen/UnboundedFollowing.cc", 51 | "gen/Star.cc", 52 | "gen/Comparison.cc", 53 | "gen/CastEpression.cc", 54 | "gen/SelectItem.cc", 55 | "gen/Unsupported.cc", 56 | "gen/FunctionCall.cc", 57 | "gen/ParenthesizedExpression.cc", 58 | "gen/FieldReference.cc", 59 | "gen/InvervalQualifier.cc", 60 | "gen/Join.cc", 61 | "gen/TryExpression.cc", 62 | "gen/CommaJoin.cc", 63 | "gen/FromClause.cc", 64 | "gen/Cube.cc", 65 | "gen/HavingClause.cc", 66 | "gen/AdditiveEpression.cc", 67 | "gen/NonSecondField.cc", 68 | "gen/NullOrdering.cc", 69 | "gen/Select.cc", 70 | "gen/TimeZoneField.cc", 71 | "gen/CatalogName.cc", 72 | "gen/DateLiteral.cc", 73 | "gen/FieldDefinition.cc", 74 | "gen/WindowFrameExtent.cc", 75 | "gen/OrExpression.cc", 76 | "gen/CaseExpression.cc", 77 | "gen/MapType.cc", 78 | "gen/ArrayType.cc", 79 | "gen/OnClause.cc", 80 | "gen/WindowFunction.cc", 81 | "gen/WhenClause.cc", 82 | "gen/SortSpecificationList.cc", 83 | "gen/Identifier.cc", 84 | "gen/GroupbyClause.cc", 85 | "gen/Like.cc", 86 | "gen/NullIf.cc", 87 | "gen/GroupingOperation.cc", 88 | "gen/NamedArgument.cc", 89 | "gen/ArrayElement.cc", 90 | "gen/GroupingSets.cc", 91 | "gen/LimitClause.cc", 92 | "gen/Unused.cc", 93 | "gen/QualifiedName.cc", 94 | "gen/Between.cc", 95 | "gen/ArgumentList.cc", 96 | "gen/Coalesce.cc", 97 | "gen/NullTreatment.cc", 98 | "gen/TableName.cc", 99 | "gen/IsExpression.cc", 100 | "gen/WindowFrameUnits.cc", 101 | "gen/UnsignedNumericLiteral.cc", 102 | "gen/CastExpression.cc", 103 | "gen/PartitionByClause.cc", 104 | "gen/CreateSchema.cc", 105 | "gen/SelectList.cc", 106 | "gen/ColumnNames.cc", 107 | "gen/Rollup.cc", 108 | "gen/BooleanLiteral.cc", 109 | "gen/SetQuantifier.cc", 110 | "gen/SearchedCaseOperand.cc", 111 | "gen/UnaryExpression.cc", 112 | "gen/NotExpression.cc", 113 | "gen/TableExpression.cc", 114 | "gen/AggregationFunction.cc", 115 | "gen/UnboundedPreceding.cc", 116 | "gen/IsDistinct.cc", 117 | "gen/Unnest.cc", 118 | "gen/Lambda.cc", 119 | "gen/SetOperation.cc", 120 | "gen/Unsuppoerted.cc", 121 | "gen/RowExpression.cc", 122 | "gen/TableSample.cc", 123 | "gen/LambdaBody.cc", 124 | "gen/Cte.cc", 125 | "gen/SortSpecification.cc", 126 | "gen/Alias.cc", 127 | "gen/Exists.cc", 128 | "gen/WindowFramePreceding.cc", 129 | "gen/IsNull.cc", 130 | "gen/SchemaName.cc", 131 | "gen/WithClause.cc", 132 | "gen/JJTSqlParserState.cc", 133 | "gen/SqlParser.cc", 134 | "gen/SqlParserTokenManager.cc", 135 | "gen/CharStream.cc", 136 | "gen/Token.cc", 137 | "gen/TokenMgrError.cc", 138 | "gen/ParseException.cc", 139 | ] 140 | 141 | custom_rule( 142 | name = "gen", 143 | srcs = glob([ "*.txt", "javacc/javacc.jar" ]), 144 | output_gen_files = GENERATED_FILES, 145 | build_script_dep = ":generate_parser", 146 | ) 147 | 148 | cpp_library( 149 | name = "sqlparser", 150 | deps = [ ":gen" ], 151 | srcs = 152 | glob(["ParseErrorHandler.cc"]) + 153 | #[":gen=" + x for x in GENERATED_FILES] 154 | [":gen"] 155 | ) 156 | 157 | #cpp_binary( 158 | #name = "parser_test", 159 | #srcs = [ 160 | #"main.cc" 161 | #], 162 | #deps = [ 163 | #":sqlparser" 164 | #], 165 | #) 166 | -------------------------------------------------------------------------------- /pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4.0.0 4 | 5 | 6 | com.facebook.airlift 7 | airbase 8 | 99 9 | 10 | 11 | com.facebook.presto 12 | presto-coresql 13 | 0.2-SNAPSHOT 14 | pom 15 | 16 | presto-coresql 17 | coreSql 18 | https://github.com/prestodb/sql 19 | 20 | 21 | ${project.parent.basedir} 22 | 0.198 23 | 24 | 25 | 26 | parser 27 | linter 28 | rewriter 29 | 30 | 31 | 32 | 33 | 34 | com.facebook.airlift 35 | configuration 36 | ${dep.airlift.version} 37 | 38 | 39 | 40 | com.facebook.airlift 41 | log 42 | ${dep.airlift.version} 43 | 44 | 45 | 46 | com.facebook.airlift 47 | bootstrap 48 | ${dep.airlift.version} 49 | 50 | 51 | 52 | javax.validation 53 | validation-api 54 | 2.0.1.Final 55 | 56 | 57 | 58 | 59 | 60 | 61 | 62 | com.facebook.presto 63 | presto-maven-plugin 64 | 0.3 65 | true 66 | 67 | 68 | 69 | org.apache.maven.plugins 70 | maven-release-plugin 71 | 72 | clean verify -DskipTests 73 | 74 | 75 | 76 | 77 | org.apache.maven.plugins 78 | maven-compiler-plugin 79 | 80 | false 81 | 82 | 83 | 84 | 85 | org.apache.maven.plugins 86 | maven-checkstyle-plugin 87 | 88 | **/generated-sources/**/* 89 | 90 | 91 | 92 | 93 | 94 | maven-pmd-plugin 95 | 96 | true 97 | 98 | 99 | 100 | 101 | com.github.spotbugs 102 | spotbugs-maven-plugin 103 | 104 | ${project.basedir}/../src/spotbugs/spotbugs-exclude.xml 105 | 106 | 107 | 108 | none 109 | 110 | 111 | 112 | 113 | 114 | org.apache.maven.plugins 115 | maven-checkstyle-plugin 116 | 117 | **/generated-sources/**/* 118 | 119 | 120 | 121 | 122 | 123 | org.apache.maven.plugins 124 | maven-jar-plugin 125 | 126 | 128 | true 129 | 130 | 131 | true 132 | true 133 | false 134 | 135 | 136 | 137 | 138 | 139 | 140 | 141 | -------------------------------------------------------------------------------- /parser/src/main/java/com/facebook/coresql/parser/AstNode.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed under the Apache License, Version 2.0 (the "License"); 3 | * you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at 5 | * 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * 8 | * Unless required by applicable law or agreed to in writing, software 9 | * distributed under the License is distributed on an "AS IS" BASIS, 10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the License for the specific language governing permissions and 12 | * limitations under the License. 13 | */ 14 | package com.facebook.coresql.parser; 15 | 16 | import static com.facebook.coresql.parser.SqlParserTreeConstants.JJTADDITIVEEXPRESSION; 17 | import static com.facebook.coresql.parser.SqlParserTreeConstants.JJTANDEXPRESSION; 18 | import static com.facebook.coresql.parser.SqlParserTreeConstants.JJTBETWEEN; 19 | import static com.facebook.coresql.parser.SqlParserTreeConstants.JJTBUILTINFUNCTIONCALL; 20 | import static com.facebook.coresql.parser.SqlParserTreeConstants.JJTCOMPARISON; 21 | import static com.facebook.coresql.parser.SqlParserTreeConstants.JJTFUNCTIONCALL; 22 | import static com.facebook.coresql.parser.SqlParserTreeConstants.JJTINPREDICATE; 23 | import static com.facebook.coresql.parser.SqlParserTreeConstants.JJTISNULL; 24 | import static com.facebook.coresql.parser.SqlParserTreeConstants.JJTLIKE; 25 | import static com.facebook.coresql.parser.SqlParserTreeConstants.JJTMULTIPLICATIVEEXPRESSION; 26 | import static com.facebook.coresql.parser.SqlParserTreeConstants.JJTNOTEXPRESSION; 27 | import static com.facebook.coresql.parser.SqlParserTreeConstants.JJTOREXPRESSION; 28 | 29 | public class AstNode 30 | extends com.facebook.coresql.parser.SimpleNode 31 | { 32 | public Token beginToken; 33 | public Token endToken; 34 | private int operator; 35 | private boolean negated; 36 | 37 | protected AstNode(int id) 38 | { 39 | super(id); 40 | } 41 | 42 | protected AstNode(com.facebook.coresql.parser.SqlParser parser, int id) 43 | { 44 | super(parser, id); 45 | } 46 | 47 | public int Kind() 48 | { 49 | return id; 50 | } 51 | 52 | public final int NumChildren() 53 | { 54 | return jjtGetNumChildren(); 55 | } 56 | 57 | public final AstNode GetChild(int i) 58 | { 59 | return (AstNode) jjtGetChild(i); 60 | } 61 | 62 | public String GetImage() 63 | { 64 | return NumChildren() == 0 ? beginToken.image : null; 65 | } 66 | 67 | public final void VisitNode(com.facebook.coresql.parser.SqlParserVisitor visitor) 68 | { 69 | jjtAccept(visitor, null); 70 | } 71 | 72 | public final AstNode FirstChild() 73 | { 74 | return NumChildren() > 0 ? GetChild(0) : null; 75 | } 76 | 77 | public final AstNode LastChild() 78 | { 79 | return NumChildren() > 0 ? GetChild(NumChildren() - 1) : null; 80 | } 81 | 82 | public final AstNode GetFirstChildOfKind(int kind) 83 | { 84 | for (int i = 0; i < NumChildren(); i++) { 85 | if (GetChild(i).Kind() == kind) { 86 | return GetChild(i); 87 | } 88 | } 89 | 90 | return null; 91 | } 92 | 93 | public final boolean hasChildOfKind(int kind) 94 | { 95 | return GetFirstChildOfKind(kind) != null; 96 | } 97 | 98 | public Location getLocation() 99 | { 100 | return new Location(beginToken.beginLine, beginToken.beginColumn, endToken.endLine, endToken.endColumn); 101 | } 102 | 103 | @Override 104 | public String toString(String prefix) 105 | { 106 | return super.toString(prefix) + " (" + getLocation().toString() + ")" + 107 | (NumChildren() == 0 ? " (" + beginToken.image + ")" : ""); 108 | } 109 | 110 | // Other utility methods. 111 | 112 | public boolean IsNegatableOperator() 113 | { 114 | switch (Kind()) { 115 | case JJTBETWEEN: 116 | case JJTLIKE: 117 | case JJTINPREDICATE: 118 | case JJTISNULL: 119 | return true; 120 | default: 121 | return false; 122 | } 123 | } 124 | 125 | public boolean IsOperator() 126 | { 127 | switch (Kind()) { 128 | case JJTADDITIVEEXPRESSION: 129 | case JJTMULTIPLICATIVEEXPRESSION: 130 | case JJTOREXPRESSION: 131 | case JJTANDEXPRESSION: 132 | case JJTNOTEXPRESSION: 133 | case JJTCOMPARISON: 134 | // Other binary negatable operators 135 | return true; 136 | default: 137 | return false; 138 | } 139 | } 140 | 141 | public void SetOperator(int operator) 142 | { 143 | this.operator = operator; 144 | } 145 | 146 | public int GetOperator() 147 | { 148 | if (!IsOperator()) { 149 | throw new IllegalArgumentException("GetOperator cannot be called on: " + this); 150 | } 151 | 152 | return operator; 153 | } 154 | 155 | public String GetFunctionName() 156 | { 157 | if (Kind() == JJTFUNCTIONCALL) { 158 | return GetChild(0).GetImage(); 159 | } 160 | 161 | if (Kind() == JJTBUILTINFUNCTIONCALL) { 162 | return beginToken.image; 163 | } 164 | 165 | return null; 166 | } 167 | 168 | public void SetNegated(boolean negated) 169 | { 170 | this.negated = negated; 171 | } 172 | 173 | public boolean IsNegated() 174 | { 175 | return negated; 176 | } 177 | } 178 | -------------------------------------------------------------------------------- /parser/cpp/javacc-options.txt: -------------------------------------------------------------------------------- 1 | options { 2 | STATIC = false; 3 | OUTPUT_LANGUAGE = "c++"; 4 | NAMESPACE = "commonsql::parser"; 5 | LOOKAHEAD=3; 6 | IGNORE_CASE=true; 7 | UNICODE_INPUT=true; 8 | PARSER_INCLUDE="parser.h"; 9 | ERROR_REPORTING=false; 10 | 11 | NODE_DEFAULT_VOID = true; 12 | NODE_SCOPE_HOOK = true; 13 | NODE_CLASS = "AstNode"; 14 | NODE_PREFIX = ""; 15 | MULTI = true; 16 | VISITOR = true; 17 | } 18 | 19 | PARSER_BEGIN(SqlParser) 20 | 21 | inline bool IsIdNonReservedWord() { 22 | auto kind = getToken(1)->kind; 23 | if (__builtin_expect(kind == regular_identifier, 1) || kind == delimited_identifier || kind == Unicode_delimited_identifier) return true; 24 | 25 | if (!(kind >= MIN_NON_RESERVED_WORD && kind <= MAX_NON_RESERVED_WORD)) return false; // Not a nonreserved word. 26 | 27 | // Some special cases. 28 | switch (kind) { 29 | // Some contextual keywords 30 | case GROUP: 31 | case ORDER: 32 | case PARTITION: 33 | return getToken(2)->kind != BY; 34 | 35 | case LIMIT: 36 | return getToken(2)->kind != unsigned_integer; 37 | 38 | case ROWS: 39 | return getToken(2)->kind != BETWEEN; 40 | 41 | // Some builtin functions 42 | case TRIM: 43 | case POSITION: 44 | case MOD: 45 | case POWER: 46 | case RANK: 47 | case ROW_NUMBER: 48 | case FLOOR: 49 | case MIN: 50 | case MAX: 51 | case UPPER: 52 | case LOWER: 53 | case CARDINALITY: 54 | case ABS: 55 | return getToken(2)->kind != lparen; 56 | 57 | default: 58 | return true; 59 | } 60 | 61 | // Should never come here. 62 | return true; 63 | } 64 | 65 | inline bool SyncToSemicolon() { 66 | if (hasError || getToken(0)->kind != semicolon) { 67 | while (getToken(1)->kind != _EOF && getToken(1)->kind != semicolon) { 68 | getNextToken(); 69 | } 70 | 71 | if (getToken(1)->kind == semicolon) { 72 | getNextToken(); 73 | } 74 | 75 | hasError = false; 76 | } 77 | 78 | return true; 79 | } 80 | 81 | inline bool NotEof() { 82 | return getToken(1)->kind != _EOF; 83 | } 84 | 85 | void PushNode(Node* node) { jjtree.pushNode(node); } 86 | Node* PopNode() { return jjtree.popNode(); } 87 | 88 | void jjtreeOpenNodeScope(Node* node) { 89 | static_cast(node)->beginToken = getToken(1); 90 | } 91 | 92 | void jjtreeCloseNodeScope(Node* node) { 93 | AstNode* astNode = static_cast(node); 94 | astNode->endToken = getToken(0); 95 | Token* t = astNode->beginToken; 96 | 97 | // For some nodes, the node is opened after some children are already created. Reset the begin for those to be 98 | // the begin of the left-most child. 99 | if (astNode->NumChildren() > 0) { 100 | Token* t0 = astNode->GetChild(0)->beginToken; 101 | if (t0->beginLine < t->beginLine || (t0->beginLine == t->beginLine && t0->beginColumn < t->beginColumn)) { 102 | astNode->beginToken = t0; 103 | } 104 | } 105 | 106 | if (astNode->getId() == JJTUNSUPPORTED) { 107 | cout << "Unsupported feature used at: " << t->beginLine << ":" << t->beginColumn << " " << t->image << "\n"; 108 | } 109 | 110 | 111 | if (astNode->IsNegatableOperator()) { 112 | Token* t1 = astNode->GetChild(0)->endToken; 113 | 114 | if (astNode->Kind() == JJTISNULL) { 115 | // IsNull -- see if the penultimate token is NOT 116 | while (t1 != null && t1->kind != IS) { 117 | t1 = t1->next; 118 | } 119 | 120 | if (t1->next->kind == NOT) { 121 | astNode->SetNegated(true); 122 | } 123 | } 124 | else if (astNode->NumChildren() > 1) { 125 | Token* t2 = astNode->GetChild(1)->beginToken; 126 | while (t1->next != null && t1->next != t2) { 127 | if (t1->kind == NOT) { 128 | astNode->SetNegated(true); 129 | break; 130 | } 131 | t1 = t1->next; 132 | } 133 | } 134 | } 135 | else if (astNode->NumChildren() == 2 && astNode->IsOperator()) { 136 | // Hack locate the token just before the first token of the second operator 137 | Token* t1 = astNode->GetChild(0)->endToken; 138 | Token* t2 = astNode->GetChild(1)->beginToken; 139 | while (t1->next != nullptr && t1->next != t2) { 140 | t1 = t1->next; 141 | } 142 | astNode->SetOperator(t1->kind); 143 | } 144 | 145 | if (astNode->NumChildren() == 1 && astNode->IsOperator()) { 146 | astNode->SetOperator(astNode->beginToken->kind); 147 | } 148 | } 149 | 150 | PARSER_END(SqlParser) 151 | 152 | TOKEN_MGR_DECLS: 153 | { 154 | void setKindToIdentifier(Token *t) { 155 | t->kind = regular_identifier; 156 | } 157 | 158 | void setUnicodeLiteralType(Token *t) { 159 | t->kind = unicode_literal; 160 | } 161 | 162 | void StoreImage(Token* matchedToken) { 163 | // TODO(sreeni): fix it. 164 | // matchedToken->image = image; 165 | } 166 | } 167 | 168 | // Temporary entry point 169 | Node* compilation_unit() #CompilationUnit: 170 | { 171 | Token *begin; 172 | } 173 | { 174 | ( )* 175 | ( 176 | LOOKAHEAD({NotEof()}) 177 | { begin = getToken(1); } 178 | statement_list() 179 | { if (hasError) cout << "Error parsing statement at: " << begin->beginLine; } 180 | { SyncToSemicolon(); } 181 | )* 182 | 183 | 184 | { if (jjtree.peekNode() != nullptr) return jjtree.peekNode(); return null; } 185 | } 186 | 187 | void statement_list(): 188 | {} 189 | { 190 | ( 191 | LOOKAHEAD({NotEof()}) 192 | direct_SQL_statement() 193 | ( )* 194 | )+ 195 | { SyncToSemicolon(); } 196 | } 197 | 198 | -------------------------------------------------------------------------------- /parser/grammar/reservedwords.txt: -------------------------------------------------------------------------------- 1 | SKIP: 2 | { 3 | // Dummy token to get a value range. Will never be mached. And it should be here positionally - before the first non reserved word 4 | 5 | } 6 | 7 | // reserved words 8 | TOKEN: 9 | { 10 | 11 | | 12 | | 13 | | 14 | | 15 | | 16 | | 17 | | 18 | | 19 | | 20 | | 21 | | 22 | | 23 | | 24 | | 25 | | 26 | | 27 | | 28 | | 29 | | 30 | | 31 | | 32 | | 33 | | 34 | | 35 | | 36 | | 37 | | 38 | | 39 | | 40 | | 41 | | 42 | | 43 | | 44 | | 45 | | 46 | | 47 | | 48 | | 49 | | 50 | | 51 | | 52 | | 53 | | 54 | | 55 | | 56 | | 57 | | 58 | | 59 | | 60 | | 61 | | 62 | | 63 | | 64 | | 65 | | 66 | | 67 | | 68 | | 69 | | 70 | | 71 | | 72 | | 73 | | 74 | | 75 | | 76 | | 77 | | 78 | | 79 | | 80 | | 81 | | 82 | | 83 | | 84 | | 85 | | 86 | | 87 | | 88 | | 89 | | 90 | | 91 | | 92 | | 93 | | 94 | | 95 | | 96 | | 97 | | 98 | | 99 | | 100 | | 101 | | 102 | | 103 | | 104 | | 105 | | 106 | | 107 | | 108 | | 109 | | 110 | | 111 | | 112 | | 113 | | 114 | | 115 | | 116 | | 117 | | 118 | | 119 | | 120 | | 121 | | 122 | | 123 | | 124 | | 125 | | 126 | | 127 | | 128 | | 129 | | 130 | | 131 | | 132 | | 133 | | 134 | | 135 | | 136 | | 137 | | 138 | | 139 | | 140 | | 141 | | 142 | | 143 | | 144 | | 145 | | 146 | | 147 | | 148 | | 149 | | 150 | | 151 | | 152 | | 153 | | 154 | | 155 | | 156 | | 157 | | 158 | | 159 | | 160 | | 161 | | 162 | | 163 | | 164 | | 165 | | 166 | | 167 | | 168 | | 169 | | 170 | | 171 | | 172 | | 173 | | 174 | | 175 | | 176 | | 177 | | 178 | | 179 | | 180 | | 181 | | 182 | | 183 | | 184 | | 185 | | 186 | | 187 | | 188 | | 189 | | 190 | | 191 | | 192 | | 193 | | 194 | | 195 | | 196 | | 197 | | 198 | | 199 | | 200 | | 201 | | 202 | | 203 | | 204 | | 205 | | 206 | | 207 | | 208 | | 209 | | 210 | | 211 | | 212 | | 213 | | 214 | | 215 | | 216 | | 217 | | 218 | | 219 | } 220 | 221 | SKIP: 222 | { 223 | // Dummy token to get a value range. Will never be mached: 224 | 225 | } 226 | 227 | //TODO(kaikalur): create a separate section for all special chars 228 | TOKEN: 229 | { 230 | 231 | | 232 | | 233 | | 234 | } 235 | -------------------------------------------------------------------------------- /rewriter/src/test/java/com/facebook/coresql/rewriter/TestOrderByRewriter.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed under the Apache License, Version 2.0 (the "License"); 3 | * you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at 5 | * 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * 8 | * Unless required by applicable law or agreed to in writing, software 9 | * distributed under the License is distributed on an "AS IS" BASIS, 10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the License for the specific language governing permissions and 12 | * limitations under the License. 13 | */ 14 | 15 | package com.facebook.coresql.rewriter; 16 | 17 | import com.facebook.coresql.parser.AstNode; 18 | import com.google.common.collect.ImmutableMap; 19 | import org.testng.annotations.Test; 20 | 21 | import java.util.Map; 22 | import java.util.Optional; 23 | 24 | import static com.facebook.coresql.parser.ParserHelper.parseStatement; 25 | import static org.testng.Assert.assertEquals; 26 | import static org.testng.Assert.assertFalse; 27 | import static org.testng.Assert.assertNotNull; 28 | import static org.testng.Assert.assertTrue; 29 | 30 | public class TestOrderByRewriter 31 | { 32 | private static final String[] STATEMENTS_THAT_DONT_NEED_REWRITE = new String[] { 33 | // False Positive 34 | "CREATE TABLE blah AS SELECT * FROM (SELECT * FROM (SELECT foo FROM T ORDER BY x LIMIT 10) ORDER BY y LIMIT 10) ORDER BY z LIMIT 10;", 35 | "SELECT dealer_id, sales OVER (PARTITION BY dealer_id ORDER BY sales);", 36 | "INSERT INTO blah SELECT * FROM (SELECT t.date, t.code, t.qty FROM sales AS t ORDER BY t.date LIMIT 100);", 37 | "SELECT (true or false) and false;", 38 | // True Negative 39 | "SELECT * FROM T ORDER BY y;", 40 | "SELECT * FROM T ORDER BY y LIMIT 10;", 41 | "use a.b;", 42 | " SELECT 1;", 43 | "SELECT a FROM T;", 44 | "SELECT a FROM T WHERE p1 > p2;", 45 | "SELECT a, b, c FROM T WHERE c1 < c2 and c3 < c4;", 46 | "SELECT CASE a WHEN IN ( 1 ) THEN b ELSE c END AS x, b, c FROM T WHERE c1 < c2 and c3 < c4;", 47 | "SELECT T.* FROM T JOIN W ON T.x = W.x;", 48 | "SELECT NULL;", 49 | "SELECT ARRAY[x] FROM T;", 50 | "SELECT TRANSFORM(ARRAY[x], x -> x + 2) AS arra FROM T;", 51 | "CREATE TABLE T AS SELECT TRANSFORM(ARRAY[x], x -> x + 2) AS arra FROM T;", 52 | "INSERT INTO T SELECT TRANSFORM(ARRAY[x], x -> x + 2) AS arra FROM T;", 53 | "SELECT ROW_NUMBER() OVER(PARTITION BY x) FROM T;", 54 | "SELECT x, SUM(y) OVER (PARTITION BY y ORDER BY 1) AS min\n" + 55 | "FROM (values ('b',10), ('a', 10)) AS T(x, y)\n;", 56 | "SELECT\n" + 57 | " CAST(MAP() AS map>) AS \"bool_tensor_features\";", 58 | "SELECT f(f(f(f(f(f(f(f(f(f(f(f(f(f(f(f(f(f(f(f(f(f(f(f(f(f(f(f(f(f())))))))))))))))))))))))))))));", 59 | "SELECT abs, 2 as abs;", 60 | }; 61 | 62 | private static final ImmutableMap STATEMENT_TO_REWRITTEN_STATEMENT = 63 | new ImmutableMap.Builder() 64 | .put("CREATE TABLE blah AS SELECT * FROM T ORDER BY y;", 65 | "CREATE TABLE blah AS SELECT * FROM T;") 66 | .put("INSERT INTO blah SELECT * FROM T ORDER BY y;", 67 | "INSERT INTO blah SELECT * FROM T;") 68 | .put("CREATE TABLE blah AS SELECT * FROM T ORDER BY SUM(payment);", 69 | "CREATE TABLE blah AS SELECT * FROM T;") 70 | .put("INSERT INTO blah SELECT * FROM (SELECT t.date, t.code, t.qty FROM sales AS t ORDER BY t.date) LIMIT 10;", 71 | "INSERT INTO blah SELECT * FROM (SELECT t.date, t.code, t.qty FROM sales AS t) LIMIT 10;") 72 | .put("CREATE TABLE blah AS SELECT * FROM (SELECT * FROM T) ORDER BY z;", 73 | "CREATE TABLE blah AS SELECT * FROM (SELECT * FROM T);") 74 | .put("CREATE TABLE blah AS SELECT * FROM (SELECT * FROM (SELECT foo FROM T ORDER BY x));", 75 | "CREATE TABLE blah AS SELECT * FROM (SELECT * FROM (SELECT foo FROM T));") 76 | .put("CREATE TABLE blah AS SELECT * FROM (SELECT * FROM (SELECT foo FROM T ORDER BY x LIMIT 10) ORDER BY y) ORDER BY z;", 77 | "CREATE TABLE blah AS SELECT * FROM (SELECT * FROM (SELECT foo FROM T ORDER BY x LIMIT 10));") 78 | .put("CREATE TABLE blah AS SELECT * FROM (SELECT * FROM (SELECT foo FROM T ORDER BY x) ORDER BY y LIMIT 10) ORDER BY z;", 79 | "CREATE TABLE blah AS SELECT * FROM (SELECT * FROM (SELECT foo FROM T) ORDER BY y LIMIT 10);") 80 | .put("CREATE TABLE blah AS SELECT * FROM (SELECT * FROM (SELECT foo FROM T ORDER BY x) ORDER BY y) ORDER BY z LIMIT 10;", 81 | "CREATE TABLE blah AS SELECT * FROM (SELECT * FROM (SELECT foo FROM T)) ORDER BY z LIMIT 10;") 82 | .put("CREATE TABLE blah AS SELECT * FROM (SELECT * FROM (SELECT foo FROM T ORDER BY x) ORDER BY y LIMIT 10) ORDER BY z LIMIT 10;", 83 | "CREATE TABLE blah AS SELECT * FROM (SELECT * FROM (SELECT foo FROM T) ORDER BY y LIMIT 10) ORDER BY z LIMIT 10;") 84 | .put("CREATE TABLE blah AS SELECT * FROM (SELECT * FROM (SELECT foo FROM T ORDER BY x LIMIT 10) ORDER BY y) ORDER BY z LIMIT 10;", 85 | "CREATE TABLE blah AS SELECT * FROM (SELECT * FROM (SELECT foo FROM T ORDER BY x LIMIT 10)) ORDER BY z LIMIT 10;") 86 | .put("CREATE TABLE blah AS SELECT * FROM (SELECT * FROM (SELECT foo FROM T ORDER BY x LIMIT 10) ORDER BY y LIMIT 10) ORDER BY z;", 87 | "CREATE TABLE blah AS SELECT * FROM (SELECT * FROM (SELECT foo FROM T ORDER BY x LIMIT 10) ORDER BY y LIMIT 10);") 88 | .build(); 89 | 90 | private void assertStatementUnchanged(String originalStatement) 91 | { 92 | assertFalse(getRewriteResult(originalStatement).isPresent()); 93 | } 94 | 95 | private void assertStatementRewritten(String originalStatement, String expectedStatement) 96 | { 97 | Optional result = getRewriteResult(originalStatement); 98 | assertTrue(result.isPresent()); 99 | assertEquals(result.get().getRewrittenSql(), expectedStatement); 100 | } 101 | 102 | private Optional getRewriteResult(String originalStatement) 103 | { 104 | AstNode ast = parseStatement(originalStatement); 105 | assertNotNull(ast); 106 | return new OrderByRewriter(ast).rewrite(); 107 | } 108 | 109 | @Test 110 | public void rewriteTest() 111 | { 112 | for (Map.Entry entry : STATEMENT_TO_REWRITTEN_STATEMENT.entrySet()) { 113 | assertStatementRewritten(entry.getKey(), entry.getValue()); 114 | } 115 | 116 | for (String sql : STATEMENTS_THAT_DONT_NEED_REWRITE) { 117 | assertStatementUnchanged(sql); 118 | } 119 | } 120 | } 121 | -------------------------------------------------------------------------------- /parser/grammar/lexical-elements.txt: -------------------------------------------------------------------------------- 1 | TOKEN: 2 | { 3 | /* 4 | <#SQL_terminal_character() ::= 5 | SQL_language_character() 6 | ; 7 | 8 | | <#SQL_language_character() ::= 9 | simple_Latin_letter() 10 | | digit() 11 | | SQL_special_character() 12 | ; 13 | 14 | | <#simple_Latin_letter() ::= 15 | simple_Latin_upper_case_letter() 16 | | simple_Latin_lower_case_letter() 17 | ; 18 | 19 | | <#simple_Latin_upper_case_letter() ::= 20 | ["A"-"Z"] 21 | ; 22 | 23 | | <#simple_Latin_lower_case_letter() ::= 24 | ["a"-"z"] 25 | ; 26 | 27 | | <#SQL_special_character() ::= 28 | space() 29 | | "\"" 30 | | "%" 31 | | "&" 32 | | "'" 33 | | left_paren() 34 | | right_paren() 35 | | asterisk() 36 | | plus_sign() 37 | | comma() 38 | | minus_sign() 39 | | period() 40 | | solidus() 41 | | colon() 42 | | semicolon() 43 | | less_than_operator() 44 | | equals_operator() 45 | | greater_than_operator() 46 | | question_mark() 47 | | "[" 48 | | "]" 49 | | "^" 50 | | "_" 51 | | "|" 52 | | "{" 53 | | "}" 54 | ; 55 | ; 56 | 57 | token() ::= 58 | nondelimiter_token() 59 | | delimiter_token() 60 | ; 61 | 62 | nondelimiter_token() ::= 63 | regular_identifier() 64 | | key_word() 65 | | unsigned_numeric_literal() 66 | | national_character_string_literal() 67 | | binary_string_literal() 68 | | large_object_length_token() 69 | | Unicode_delimited_identifier() 70 | | Unicode_character_string_literal() 71 | | SQL_language_identifier() 72 | ; 73 | 74 | */ 75 | 76 | 77 | > 78 | 79 | | <#identifier_body: ( )* > 80 | 81 | | <#identifier_part: | > 82 | 83 | | <#identifier_start: ["a"-"z"] // temp 84 | /*!! See the Syntax Rules.*/ 85 | > 86 | 87 | | <#identifier_extend: ["\u00B7", "0"-"9", "_"] // temp 88 | //!! See the Syntax Rules. 89 | > 90 | 91 | | )+ > 92 | 93 | | 94 | 95 | | )? "\"" > // Presto allows empty string as an id - yikes! 96 | 97 | | <#delimited_identifier_body: ( )+ > 98 | 99 | | <#delimited_identifier_part: | > 100 | 101 | | "\"" ( )? > 102 | 103 | | <#Unicode_escape_specifier: "UESCAPE" "'" "'" > 104 | 105 | | <#Unicode_delimiter_body: ( )+ > 106 | 107 | | <#Unicode_identifier_part: | > 108 | 109 | | <#Unicode_escape_value: | | > 110 | 111 | | <#Unicode_4_digit_escape_value: > 112 | 113 | | <#Unicode_6_digit_escape_value: "+" > 114 | 115 | | <#Unicode_character_escape_value: > 116 | 117 | | <#Unicode_escape_character: ~["a"-"z", "0"-"9", "+", "'", "\"", "\n", "\t", " "] // temp 118 | //17) If the source language character set contains , then let D 119 | //!! See the Syntax Rules.*/ 120 | > 121 | 122 | | <#nondoublequote_character: ~["\""] 123 | //!! See the Syntax Rules. 124 | > 125 | 126 | | <#doublequote_symbol: "\"\"" > 127 | 128 | /* 129 | delimiter_token: 130 | 131 | | 132 | | 133 | | 134 | | 135 | | 136 | | 137 | | <"> " 138 | | ">=" 139 | | "<=" 140 | | "||" 141 | | "->" 142 | | "??(" 143 | | "??)" 144 | | "::" 145 | | ".." 146 | | "=>" 147 | > 148 | */ 149 | } 150 | 151 | SPECIAL_TOKEN: 152 | { 153 | | [ " ", "\t" ] // temp 154 | //!! See the Syntax Rules. 155 | > 156 | 157 | | <#newline: (["\n", "\r"])+ 158 | //!! See the Syntax Rules. 159 | > 160 | 161 | //| | > 162 | | > 163 | | <#simple_comment: ( )* ()? > 164 | 165 | | <#simple_comment_introducer: "--" > 166 | 167 | //| <#bracketed_comment: > 168 | 169 | 170 | //| <#bracketed_comment_terminator: "*/" > 171 | 172 | //| <#bracketed_comment_contents: ( | )* 173 | ////!! See the Syntax Rules. 174 | //> 175 | 176 | //| <#comment_character: | "'" > 177 | | <#comment_character: (~["\n", "\r"]) | "'" > 178 | 179 | | )+ > 180 | 181 | //| <#key_word: | > > 182 | } 183 | 184 | 185 | MORE: 186 | { 187 | : comment_contents 188 | } 189 | 190 | MORE: 191 | { 192 | "*/" : match_comment 193 | | <~[]> 194 | } 195 | 196 | SPECIAL_TOKEN: 197 | { 198 | { StoreImage(matchedToken); } : DEFAULT 199 | } 200 | 201 | TOKEN: 202 | { 203 | <#separator: ( 204 | //TODO(kaikalur): fixit -- | )+ > 205 | )+ > 206 | | 207 | <#digit: ["0"-"9"] > 208 | | <#character_representation: | > 209 | 210 | | <#nonquote_character: ~["'"] 211 | //!! See the Syntax Rules. 212 | > 213 | 214 | | <#quote_symbol: "''" > 215 | 216 | | )* "'" > 217 | | )* "'" ( "'" ( )* "'" )* > 218 | 219 | | )* "'" ( "'" ( )* "'" )* > // TODO(kaikalur) - fixit 220 | 221 | | <#Unicode_representation: | > 222 | 223 | //TODO(kaikalur): fixit 224 | | <#space: " "> // temp 225 | | )* ( ( )* ( )* )* "'" ( "'" ( )* ( ( )* ( )* )* "'" )* > 226 | 227 | | <#hexit: ["a"-"f", "0"-"9"] > 228 | 229 | 230 | //| ( "." ( )? )? | "." > 231 | 232 | //| 233 | 234 | | )+ > 235 | | ( "." ( )? ) | "." > 236 | 237 | | "E" > 238 | 239 | | <#mantissa: | > 240 | 241 | | <#exponent: > 242 | 243 | | <#signed_integer: ( [ "+", "-" ] )? > 244 | 245 | 246 | | ( )* > 247 | 248 | // TODO(srenei): fixit 249 | | <#simple_Latin_letter: ["a"-"z"]> 250 | | <#SQL_language_identifier_start: > 251 | 252 | | <#SQL_language_identifier_part: | | "_" > 253 | } 254 | 255 | TOKEN: 256 | { 257 | : DEFAULT 258 | } 259 | -------------------------------------------------------------------------------- /linter/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4.0.0 4 | 5 | 6 | com.facebook.presto 7 | presto-coresql 8 | 0.2-SNAPSHOT 9 | 10 | 11 | presto-coresql-linter 12 | presto-coresql-linter 13 | 14 | 15 | ${project.parent.basedir} 16 | 1.6 17 | 1.6 18 | 19 | 20 | 21 | 22 | junit 23 | junit 24 | 4.12 25 | test 26 | 27 | 28 | 29 | org.testng 30 | testng 31 | test 32 | 33 | 34 | 35 | com.facebook.presto 36 | presto-coresql-parser 37 | 0.2-SNAPSHOT 38 | 39 | 40 | 41 | com.google.guava 42 | guava 43 | 44 | 45 | 46 | 47 | 48 | 49 | org.codehaus.mojo 50 | build-helper-maven-plugin 51 | 52 | 53 | generate-sources 54 | 55 | add-source 56 | 57 | 58 | 59 | ${project.build.directory}/generated-sources 60 | 61 | 62 | 63 | 64 | 65 | 66 | 67 | com.facebook.presto 68 | presto-maven-plugin 69 | 0.3 70 | true 71 | 72 | 73 | 74 | org.apache.maven.plugins 75 | maven-shade-plugin 76 | 3.1.1 77 | 78 | 79 | 80 | org.skife.maven 81 | really-executable-jar-maven-plugin 82 | 1.0.5 83 | 84 | 85 | 86 | org.apache.maven.plugins 87 | maven-antrun-plugin 88 | 1.8 89 | 90 | 91 | 92 | io.airlift.maven.plugins 93 | sphinx-maven-plugin 94 | 2.1 95 | 96 | 97 | 98 | org.apache.maven.plugins 99 | maven-enforcer-plugin 100 | 101 | 102 | 103 | 104 | 105 | org.alluxio:alluxio-shaded-client 106 | org.codehaus.plexus:plexus-utils 107 | com.google.guava:guava 108 | 109 | 110 | 111 | 112 | 113 | 114 | 115 | org.apache.maven.plugins 116 | maven-release-plugin 117 | 118 | clean verify -DskipTests 119 | 120 | 121 | 122 | 123 | org.apache.maven.plugins 124 | maven-compiler-plugin 125 | 126 | true 127 | 128 | -verbose 129 | -J-Xss100M 130 | 131 | 132 | 133 | 134 | 135 | org.apache.maven.plugins 136 | maven-surefire-plugin 137 | 138 | 139 | **/*.java 140 | target/**/*.java 141 | **/Benchmark*.java 142 | 143 | 144 | **/*jmhTest*.java 145 | **/*jmhType*.java 146 | 147 | 148 | 149 | 150 | 151 | 152 | org.apache.maven.plugins 153 | maven-jar-plugin 154 | 155 | 157 | true 158 | 159 | 160 | true 161 | true 162 | false 163 | 164 | 165 | 166 | 167 | 168 | 169 | 170 | 171 | 172 | org.gaul 173 | modernizer-maven-plugin 174 | 2.1.0 175 | 176 | 1.8 177 | com.facebook.coresql.parser 178 | 179 | 180 | 181 | 182 | 183 | 184 | -------------------------------------------------------------------------------- /rewriter/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4.0.0 4 | 5 | 6 | com.facebook.presto 7 | presto-coresql 8 | 0.2-SNAPSHOT 9 | 10 | 11 | presto-coresql-rewriter 12 | presto-coresql-rewriter 13 | 14 | 15 | ${project.parent.basedir} 16 | 1.6 17 | 1.6 18 | 19 | 20 | 21 | 22 | junit 23 | junit 24 | 4.12 25 | test 26 | 27 | 28 | 29 | org.testng 30 | testng 31 | test 32 | 33 | 34 | 35 | com.facebook.presto 36 | presto-coresql-parser 37 | 0.2-SNAPSHOT 38 | 39 | 40 | 41 | com.fasterxml.jackson.core 42 | jackson-annotations 43 | 44 | 45 | 46 | com.google.guava 47 | guava 48 | 49 | 50 | 51 | 52 | 53 | org.codehaus.mojo 54 | build-helper-maven-plugin 55 | 56 | 57 | generate-sources 58 | 59 | add-source 60 | 61 | 62 | 63 | ${project.build.directory}/generated-sources 64 | 65 | 66 | 67 | 68 | 69 | 70 | 71 | com.facebook.presto 72 | presto-maven-plugin 73 | 0.3 74 | true 75 | 76 | 77 | 78 | org.apache.maven.plugins 79 | maven-shade-plugin 80 | 3.1.1 81 | 82 | 83 | 84 | org.skife.maven 85 | really-executable-jar-maven-plugin 86 | 1.0.5 87 | 88 | 89 | 90 | org.apache.maven.plugins 91 | maven-antrun-plugin 92 | 1.8 93 | 94 | 95 | 96 | io.airlift.maven.plugins 97 | sphinx-maven-plugin 98 | 2.1 99 | 100 | 101 | 102 | org.apache.maven.plugins 103 | maven-enforcer-plugin 104 | 105 | 106 | 107 | 108 | 109 | org.alluxio:alluxio-shaded-client 110 | org.codehaus.plexus:plexus-utils 111 | com.google.guava:guava 112 | 113 | 114 | 115 | 116 | 117 | 118 | 119 | org.apache.maven.plugins 120 | maven-release-plugin 121 | 122 | clean verify -DskipTests 123 | 124 | 125 | 126 | 127 | org.apache.maven.plugins 128 | maven-compiler-plugin 129 | 130 | true 131 | 132 | -verbose 133 | -J-Xss100M 134 | 135 | 136 | 137 | 138 | 139 | org.apache.maven.plugins 140 | maven-surefire-plugin 141 | 142 | 143 | **/*.java 144 | target/**/*.java 145 | **/Benchmark*.java 146 | 147 | 148 | **/*jmhTest*.java 149 | **/*jmhType*.java 150 | 151 | 152 | 153 | 154 | 155 | 156 | org.apache.maven.plugins 157 | maven-jar-plugin 158 | 159 | 161 | true 162 | 163 | 164 | true 165 | true 166 | false 167 | 168 | 169 | 170 | 171 | 172 | 173 | 174 | 175 | org.gaul 176 | modernizer-maven-plugin 177 | 2.1.0 178 | 179 | 1.8 180 | 181 | 182 | 183 | 184 | 185 | 186 | -------------------------------------------------------------------------------- /parser/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4.0.0 4 | 5 | 6 | com.facebook.presto 7 | presto-coresql 8 | 0.2-SNAPSHOT 9 | 10 | 11 | presto-coresql-parser 12 | presto-coresql-parser 13 | 14 | 15 | ${project.parent.basedir} 16 | 1.6 17 | 1.6 18 | 19 | 20 | 21 | 22 | junit 23 | junit 24 | 4.12 25 | test 26 | 27 | 28 | 29 | org.testng 30 | testng 31 | test 32 | 33 | 34 | 35 | javax.validation 36 | validation-api 37 | 38 | 39 | 40 | com.facebook.airlift 41 | configuration 42 | 43 | 44 | 45 | com.facebook.airlift 46 | log 47 | 48 | 49 | 50 | com.facebook.airlift 51 | bootstrap 52 | 53 | 54 | 55 | com.google.inject 56 | guice 57 | 58 | 59 | 60 | com.google.guava 61 | guava 62 | 63 | 64 | 65 | 66 | 67 | 68 | org.codehaus.mojo 69 | exec-maven-plugin 70 | 1.6.0 71 | 72 | 73 | Build-parser 74 | generate-sources 75 | 76 | exec 77 | 78 | 79 | 80 | 81 | ./prepare-javacc-grammar.sh 82 | grammar 83 | grammar 84 | 85 | 86 | 87 | 88 | org.javacc.plugin 89 | javacc-maven-plugin 90 | 3.0.2 91 | 92 | 93 | jcc7jcc 94 | generate-sources 95 | 96 | jjtree-javacc 97 | 98 | 99 | target/generated-sources/javacc 100 | java 101 | 102 | 103 | 104 | 105 | 106 | 107 | org.codehaus.mojo 108 | build-helper-maven-plugin 109 | 110 | 111 | generate-sources 112 | 113 | add-source 114 | 115 | 116 | 117 | ${project.build.directory}/generated-sources 118 | 119 | 120 | 121 | 122 | 123 | 124 | 125 | com.facebook.presto 126 | presto-maven-plugin 127 | 0.3 128 | true 129 | 130 | 131 | 132 | org.apache.maven.plugins 133 | maven-shade-plugin 134 | 3.1.1 135 | 136 | 137 | 138 | org.skife.maven 139 | really-executable-jar-maven-plugin 140 | 1.0.5 141 | 142 | 143 | 144 | org.apache.maven.plugins 145 | maven-antrun-plugin 146 | 1.8 147 | 148 | 149 | 150 | io.airlift.maven.plugins 151 | sphinx-maven-plugin 152 | 2.1 153 | 154 | 155 | 156 | org.apache.maven.plugins 157 | maven-enforcer-plugin 158 | 159 | 160 | 161 | 162 | 163 | org.alluxio:alluxio-shaded-client 164 | org.codehaus.plexus:plexus-utils 165 | com.google.guava:guava 166 | 167 | 168 | 169 | 170 | 171 | 172 | 173 | org.apache.maven.plugins 174 | maven-release-plugin 175 | 176 | clean verify -DskipTests 177 | 178 | 179 | 180 | 181 | org.apache.maven.plugins 182 | maven-compiler-plugin 183 | 184 | true 185 | 186 | -verbose 187 | -J-Xss100M 188 | 189 | 190 | 191 | 192 | 193 | org.apache.maven.plugins 194 | maven-surefire-plugin 195 | 196 | 197 | **/*.java 198 | target/**/*.java 199 | **/Benchmark*.java 200 | 201 | 202 | **/*jmhTest*.java 203 | **/*jmhType*.java 204 | 205 | 206 | 207 | 208 | 209 | 210 | org.apache.maven.plugins 211 | maven-jar-plugin 212 | 213 | 215 | true 216 | 217 | 218 | true 219 | true 220 | false 221 | 222 | 223 | 224 | 225 | 226 | 227 | 228 | 229 | 230 | org.javacc.plugin 231 | javacc-maven-plugin 232 | 233 | 234 | net.java.dev.javacc 235 | javacc 236 | 7.0.10 237 | runtime 238 | 239 | 240 | 241 | 242 | 243 | org.gaul 244 | modernizer-maven-plugin 245 | 2.1.0 246 | 247 | 1.8 248 | com.facebook.coresql.parser 249 | 250 | 251 | 252 | 253 | 254 | 255 | -------------------------------------------------------------------------------- /parser/grammar/nonreservedwords.txt: -------------------------------------------------------------------------------- 1 | // non_reserved words 2 | 3 | SKIP: 4 | { 5 | // Dummy token to get a value range. Will never be mached. And it should be here positionally - before the first non reserved word 6 | 7 | } 8 | 9 | // This production should be here and moved out. See notes for details on handling non-reserved words. 10 | void non_reserved_word(): 11 | {} 12 | { 13 | 14 | | 15 | | 16 | | 17 | | 18 | | 19 | | 20 | | 21 | | 22 | | 23 | | 24 | | 25 | | 26 | | 27 | | 28 | | 29 | | 30 | | 31 | | 32 | | 33 | | 34 | | 35 | | 36 | | 37 | | 38 | | 39 | | 40 | | 41 | | 42 | | 43 | | 44 | | 45 | | 46 | | 47 | | 48 | | 49 | | 50 | | 51 | | 52 | | 53 | | 54 | | 55 | | 56 | | 57 | | 58 | | 59 | | 60 | | 61 | | 62 | | 63 | | 64 | | 65 | | 66 | | 67 | | 68 | | 69 | | 70 | | 71 | | 72 | | 73 | | 74 | | 75 | | 76 | | 77 | | 78 | | 79 | | 80 | | 81 | | 82 | | 83 | | 84 | | 85 | | 86 | | 87 | | 88 | | 89 | | 90 | | 91 | | 92 | | 93 | | 94 | | 95 | | 96 | | // Non-standard 97 | | 98 | | 99 | | 100 | | 101 | | 102 | | 103 | | 104 | | 105 | | 106 | | 107 | | 108 | | 109 | | 110 | | 111 | | 112 | | 113 | | 114 | | 115 | | 116 | | 117 | | 118 | | 119 | | 120 | | 121 | | 122 | | 123 | | 124 | | 125 | | 126 | | 127 | | 128 | | 129 | | 130 | | 131 | | 132 | | 133 | | 134 | | 135 | | 136 | | 137 | | 138 | | 139 | | 140 | | 141 | | 142 | | 143 | | 144 | | 145 | | 146 | | 147 | | 148 | | 149 | | 150 | | 151 | | 152 | | 153 | | 154 | | 155 | | 156 | | 157 | | 158 | | 159 | | 160 | | 161 | | 162 | | 163 | | 164 | | // Non-standard 165 | | 166 | | 167 | | 168 | | 169 | | 170 | | 171 | | 172 | | 173 | | 174 | | 175 | | 176 | | 177 | | 178 | | 179 | | 180 | | 181 | | 182 | | 183 | | 184 | | 185 | | 186 | | 187 | | 188 | | 189 | | 190 | | 191 | | 192 | | 193 | | 194 | | 195 | | 196 | | 197 | | 198 | | 199 | | 200 | | 201 | | 202 | | 203 | | 204 | | 205 | | 206 | | 207 | | 208 | | 209 | | 210 | | 211 | | 212 | | 213 | | 214 | | 215 | | 216 | | 217 | | 218 | | 219 | | 220 | | // Non-standard 221 | | 222 | | 223 | | 224 | | 225 | | 226 | | 227 | | 228 | | 229 | | 230 | | 231 | | 232 | | 233 | | 234 | | 235 | 236 | 237 | // Non-standard 238 | // Changed the following reserved words into non-reserved one as lot of users use them as identifiers. 239 | | 240 | | 241 | | 242 | | 243 | | 244 | | 245 | | 246 | | 247 | | 248 | | 249 | | 250 | | 251 | | 252 | | 253 | | 254 | | 255 | | 256 | | 257 | | 258 | | 259 | | 260 | | 261 | | 262 | | 263 | | 264 | | 265 | | 266 | | 267 | | 268 | | 269 | | 270 | | 271 | | 272 | | 273 | | 274 | | 275 | | 276 | | 277 | | 278 | | 279 | | 280 | | 281 | | 282 | | 283 | | 284 | | 285 | | 286 | | 287 | | 288 | | 289 | | 290 | | 291 | | 292 | | 293 | | 294 | | 295 | | 296 | | 297 | | 298 | | 299 | | 300 | | 301 | | 302 | | 303 | | 304 | | 305 | | 306 | | 307 | | 308 | | 309 | | 310 | | 311 | | 312 | | 313 | | 314 | | 315 | | 316 | | 317 | | 318 | | 319 | | 320 | | 321 | | 322 | | 323 | | 324 | | 325 | | 326 | | 327 | | 328 | | 329 | | 330 | | 331 | | 332 | | 333 | | 334 | | 335 | | 336 | | 337 | | 338 | | 339 | | 340 | | 341 | | 342 | | 343 | | 344 | | 345 | | 346 | | 347 | | 348 | | 349 | 350 | // Presto tokens 351 | | 352 | | 353 | | 354 | | 355 | | "NUMERIC_HISTOGRAM" 356 | | 357 | | "HISTOGRAM" 358 | | "APPROEX_PERCENTILE" 359 | | "MAP_AGG" 360 | | "SET_AGG" 361 | | "MAP_UNION" 362 | | 363 | } 364 | 365 | SKIP: 366 | { 367 | // Dummy token to get a value range. Will never be mached: 368 | 369 | } 370 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright [yyyy] [name of copyright owner] 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /parser/src/test/java/com/facebook/coresql/parser/sqllogictest/java/SqlLogicTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed under the Apache License, Version 2.0 (the "License"); 3 | * you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at 5 | * 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * 8 | * Unless required by applicable law or agreed to in writing, software 9 | * distributed under the License is distributed on an "AS IS" BASIS, 10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the License for the specific language governing permissions and 12 | * limitations under the License. 13 | */ 14 | 15 | package com.facebook.coresql.parser.sqllogictest.java; 16 | 17 | import com.facebook.airlift.bootstrap.Bootstrap; 18 | import com.facebook.airlift.log.Logger; 19 | import com.facebook.coresql.parser.AstNode; 20 | import com.google.inject.Inject; 21 | import com.google.inject.Injector; 22 | 23 | import java.io.FileNotFoundException; 24 | import java.io.IOException; 25 | import java.io.PrintWriter; 26 | import java.nio.file.FileAlreadyExistsException; 27 | import java.nio.file.FileVisitResult; 28 | import java.nio.file.FileVisitor; 29 | import java.nio.file.Path; 30 | import java.nio.file.attribute.BasicFileAttributes; 31 | import java.text.DecimalFormat; 32 | import java.util.ArrayList; 33 | import java.util.List; 34 | import java.util.Optional; 35 | 36 | import static com.facebook.coresql.parser.ParserHelper.parseStatement; 37 | import static com.facebook.coresql.parser.Unparser.unparse; 38 | import static com.facebook.coresql.parser.sqllogictest.java.SqlLogicTest.CoreSqlParsingError.PARSING_ERROR; 39 | import static com.facebook.coresql.parser.sqllogictest.java.SqlLogicTest.CoreSqlParsingError.UNPARSED_DOES_NOT_MATCH_ORIGINAL_ERROR; 40 | import static com.facebook.coresql.parser.sqllogictest.java.SqlLogicTest.CoreSqlParsingError.UNPARSING_ERROR; 41 | import static com.facebook.coresql.parser.sqllogictest.java.SqlLogicTest.DatabaseDialect.ALL; 42 | import static java.lang.Math.min; 43 | import static java.nio.file.FileVisitResult.CONTINUE; 44 | import static java.nio.file.FileVisitResult.TERMINATE; 45 | import static java.nio.file.Files.createFile; 46 | import static java.nio.file.Files.isHidden; 47 | import static java.nio.file.Files.readAllLines; 48 | import static java.nio.file.Files.walkFileTree; 49 | import static java.util.Objects.requireNonNull; 50 | import static java.util.stream.Collectors.toList; 51 | 52 | public final class SqlLogicTest 53 | { 54 | private static final String NAME_OF_RESULT_FILE = "sqllogictest_result.txt"; 55 | private static final Logger LOG = Logger.get(SqlLogicTest.class); 56 | 57 | private final SqlLogicTestResult result; 58 | private final FileVisitor fileVisitor; 59 | private final Path testFilesRootPath; 60 | private final Path resultPath; 61 | private final DatabaseDialect databaseDialect; 62 | private final int maxNumFilesToVisit; 63 | 64 | enum CoreSqlParsingError 65 | { 66 | PARSING_ERROR, 67 | UNPARSING_ERROR, 68 | UNPARSED_DOES_NOT_MATCH_ORIGINAL_ERROR 69 | } 70 | 71 | enum DatabaseDialect 72 | { 73 | ALL, 74 | ORACLE, 75 | MYSQL, 76 | MSSQL, 77 | SQLITE, 78 | POSTGRESQL 79 | } 80 | 81 | @Inject 82 | public SqlLogicTest(SqlLogicTestConfig config) 83 | { 84 | this.testFilesRootPath = requireNonNull(config.getTestFilesRootPath(), "test files root path is null"); 85 | this.resultPath = requireNonNull(config.getResultPath(), "result path is null"); 86 | this.databaseDialect = requireNonNull(config.getDatabaseDialect(), "database dialect is null"); 87 | this.result = new SqlLogicTestResult(config.getDatabaseDialect()); 88 | this.fileVisitor = new SqlLogicTestFileVisitor(); 89 | this.maxNumFilesToVisit = config.getMaxNumFilesToVisit(); 90 | } 91 | 92 | public static void execute() 93 | throws IOException 94 | { 95 | Bootstrap app = new Bootstrap(new SqlLogicTestModule()); 96 | 97 | Injector injector = app 98 | .doNotInitializeLogging() 99 | .quiet() 100 | .initialize(); 101 | 102 | injector.getInstance(SqlLogicTest.class).run(); 103 | } 104 | 105 | private void run() 106 | throws IOException 107 | { 108 | createFileForResult(resultPath); 109 | walkFileTree(testFilesRootPath, fileVisitor); 110 | writeTestResultToFile(result, resultPath); 111 | } 112 | 113 | private static void createFileForResult(Path resultPath) 114 | throws IOException 115 | { 116 | try { 117 | createFile(resultPath.resolve(NAME_OF_RESULT_FILE)); 118 | } 119 | catch (FileAlreadyExistsException e) { 120 | LOG.info("Result file already exists. We'll overwrite it with the new result."); 121 | } 122 | catch (IOException e) { 123 | throw new IOException("Could not create the sqllogictest result file.", e); 124 | } 125 | } 126 | 127 | private Optional checkStatementForParserError(String statement) 128 | { 129 | Optional ast = Optional.ofNullable(parseStatement(statement)); 130 | if (!ast.isPresent()) { 131 | return Optional.of(PARSING_ERROR); 132 | } 133 | String unparsed; 134 | try { 135 | unparsed = unparse(ast.get()); 136 | } 137 | catch (Exception e) { 138 | return Optional.of(UNPARSING_ERROR); 139 | } 140 | if (!statement.trim().equals(unparsed.trim())) { 141 | return Optional.of(UNPARSED_DOES_NOT_MATCH_ORIGINAL_ERROR); 142 | } 143 | return Optional.empty(); 144 | } 145 | 146 | private static void writeTestResultToFile(SqlLogicTestResult result, Path resultPath) 147 | { 148 | try (PrintWriter writer = new PrintWriter(resultPath.resolve(NAME_OF_RESULT_FILE).toString())) { 149 | for (SqlLogicTestStatement statement : result.getErroneousStatements()) { 150 | writer.print(statement.getStatement() + " ----> " + statement.getError().get().name()); 151 | writer.println(); 152 | writer.println(); 153 | } 154 | writer.println("==============================================="); 155 | writer.println("RESULT"); 156 | writer.println("==============================================="); 157 | writer.printf("Database dialect tested against: %s", result.getDatabaseDialect()); 158 | writer.println(); 159 | writer.printf("%d statements found, %d statements successfully parsed.", result.getNumStatementsFound(), result.getNumStatementsParsed()); 160 | writer.println(); 161 | writer.printf("%s%% of sqllogictest statements were successfully parsed.", 162 | new DecimalFormat("#.##").format(result.getNumStatementsParsed() / (double) result.getNumStatementsFound() * 100)); 163 | } 164 | catch (FileNotFoundException e) { 165 | LOG.error(e, "Could not find the result file at the given location"); 166 | } 167 | } 168 | 169 | private boolean isLineBeforeStatement(String line) 170 | { 171 | return line.startsWith("statement ok") || line.startsWith("query"); 172 | } 173 | 174 | private boolean statementIsAcceptableToCurrDialect(SqlLogicTestStatement sqlLogicTestStatement) 175 | { 176 | if (databaseDialect == ALL) { 177 | return sqlLogicTestStatement.getSkipIfSet().isEmpty() && !sqlLogicTestStatement.getCurrStatementsOnlyIfDialect().isPresent(); 178 | } 179 | if (sqlLogicTestStatement.getCurrStatementsOnlyIfDialect().isPresent() && sqlLogicTestStatement.getCurrStatementsOnlyIfDialect().get() != databaseDialect) { 180 | return false; 181 | } 182 | return !sqlLogicTestStatement.getSkipIfSet().contains(databaseDialect); 183 | } 184 | 185 | private boolean isConditionalRecordLine(String line) 186 | { 187 | String[] splitLine = line.split(" "); 188 | if (splitLine.length <= 1) { 189 | return false; 190 | } 191 | String conditionalFlag = splitLine[0]; 192 | if (!conditionalFlag.equals("skipif") && !conditionalFlag.equals("onlyif")) { 193 | return false; 194 | } 195 | String potentialDialect = splitLine[1]; 196 | try { 197 | DatabaseDialect.valueOf(potentialDialect.toUpperCase()); 198 | } 199 | catch (IllegalArgumentException e) { 200 | LOG.debug("This line had a skip-if or only-if dialect that isn't supported: %s", line); 201 | return false; 202 | } 203 | return true; 204 | } 205 | 206 | private void processConditionalRecordLine(String line, SqlLogicTestStatement.Builder builder) 207 | { 208 | String[] splitLine = line.split(" "); 209 | if (splitLine.length <= 1) { 210 | return; 211 | } 212 | String potentialDialect = splitLine[1]; 213 | DatabaseDialect dialect = DatabaseDialect.valueOf(potentialDialect.toUpperCase()); 214 | if (line.startsWith("onlyif")) { 215 | builder.setOnlyIfDialect(dialect); 216 | } 217 | else if (line.startsWith("skipif")) { 218 | builder.addDialectToSkipIfSet(dialect); 219 | } 220 | } 221 | 222 | private static String getCleanedUpStatement(StringBuilder statement) 223 | { 224 | statement = new StringBuilder(statement.toString().trim()); 225 | if (statement.length() > 0 && !statement.toString().endsWith(";")) { 226 | statement.append(";"); // This semi-colon is necessary for parsing. 227 | } 228 | return statement.toString(); 229 | } 230 | 231 | private int getStartingIndexOfNextStatement(int currIdx, List statements) 232 | { 233 | while (currIdx < statements.size() && !statements.get(currIdx).equals("")) { 234 | ++currIdx; 235 | } 236 | return currIdx; 237 | } 238 | 239 | private int getIndexOfNextDelimiter(int currIdx, List statements) 240 | { 241 | while (currIdx < statements.size() && !statements.get(currIdx).equals("----")) { 242 | ++currIdx; 243 | } 244 | return currIdx; 245 | } 246 | 247 | private List> getListOfStatementsAsLines(List linesFromFile) 248 | { 249 | List> statementsAsListsOfLines = new ArrayList<>(); 250 | int currIdx = 0; 251 | while (currIdx < linesFromFile.size()) { 252 | if (isConditionalRecordLine(linesFromFile.get(currIdx)) || isLineBeforeStatement(linesFromFile.get(currIdx))) { 253 | int startingIdxOfNextStatement = getStartingIndexOfNextStatement(currIdx, linesFromFile); 254 | int endOfCurrStatement = min(startingIdxOfNextStatement, getIndexOfNextDelimiter(currIdx, linesFromFile)); 255 | statementsAsListsOfLines.add(linesFromFile.subList(currIdx, endOfCurrStatement)); 256 | currIdx = startingIdxOfNextStatement; 257 | } 258 | else { 259 | ++currIdx; 260 | } 261 | } 262 | return statementsAsListsOfLines; 263 | } 264 | 265 | private SqlLogicTestStatement buildSqlLogicTestStatementFromLines(List statementAsListsOfLines) 266 | { 267 | SqlLogicTestStatement.Builder builder = SqlLogicTestStatement.builder(); 268 | StringBuilder statement = new StringBuilder(); 269 | for (String line : statementAsListsOfLines) { 270 | if (isConditionalRecordLine(line)) { 271 | processConditionalRecordLine(line, builder); 272 | } 273 | else if (!isLineBeforeStatement(line)) { 274 | statement.append(line); 275 | } 276 | } 277 | String cleanStatement = getCleanedUpStatement(statement); 278 | return builder.setStatement(cleanStatement).setError(checkStatementForParserError(cleanStatement)).build(); 279 | } 280 | 281 | private List parseSqlLogicTestStatementsFromFile(Path file) 282 | throws IOException 283 | { 284 | List lines = readAllLines(file); 285 | List> statementsAsListsOfLines = getListOfStatementsAsLines(lines); 286 | return statementsAsListsOfLines.stream() 287 | .map(this::buildSqlLogicTestStatementFromLines) 288 | .filter(this::statementIsAcceptableToCurrDialect) 289 | .collect(toList()); 290 | } 291 | 292 | private List getErroneousStatements(List testStatements) 293 | { 294 | return testStatements.stream() 295 | .filter(statement -> checkStatementForParserError(statement.getStatement()).isPresent()) 296 | .collect(toList()); 297 | } 298 | 299 | private class SqlLogicTestFileVisitor 300 | implements FileVisitor 301 | { 302 | @Override 303 | public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) 304 | { 305 | return CONTINUE; 306 | } 307 | 308 | @Override 309 | public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) 310 | throws IOException 311 | { 312 | if (isHidden(file)) { 313 | return CONTINUE; 314 | } 315 | if (result.getNumFilesVisited() >= maxNumFilesToVisit) { 316 | return TERMINATE; 317 | } 318 | result.incrementNumFilesVisited(); 319 | List testStatements = parseSqlLogicTestStatementsFromFile(file); 320 | result.incrementNumStatementsFound(testStatements.size()); 321 | List erroneousStatements = getErroneousStatements(testStatements); 322 | result.addErroneousStatements(erroneousStatements); 323 | return CONTINUE; 324 | } 325 | 326 | @Override 327 | public FileVisitResult visitFileFailed(Path file, IOException exception) 328 | { 329 | LOG.warn(exception, "Failed to access file: %s", file.toString()); 330 | return CONTINUE; 331 | } 332 | 333 | @Override 334 | public FileVisitResult postVisitDirectory(Path dir, IOException exception) 335 | { 336 | return CONTINUE; 337 | } 338 | } 339 | } 340 | -------------------------------------------------------------------------------- /parser/src/main/java/com/facebook/coresql/parser/Unparser.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed under the Apache License, Version 2.0 (the "License"); 3 | * you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at 5 | * 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * 8 | * Unless required by applicable law or agreed to in writing, software 9 | * distributed under the License is distributed on an "AS IS" BASIS, 10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the License for the specific language governing permissions and 12 | * limitations under the License. 13 | */ 14 | package com.facebook.coresql.parser; 15 | 16 | import static com.facebook.coresql.parser.SqlParserConstants.EOF; 17 | 18 | public class Unparser 19 | extends com.facebook.coresql.parser.SqlParserDefaultVisitor 20 | { 21 | protected StringBuilder stringBuilder = new StringBuilder(); 22 | private Token lastToken = new Token(); 23 | 24 | public static String unparse(AstNode node, Unparser unparser) 25 | { 26 | unparser.stringBuilder.setLength(0); 27 | unparser.lastToken.next = node.beginToken; 28 | node.jjtAccept(unparser, null); 29 | unparser.printTrailingComments(); 30 | return unparser.stringBuilder.toString(); 31 | } 32 | 33 | public static String unparse(AstNode node) 34 | { 35 | return unparse(node, new Unparser()); 36 | } 37 | 38 | private void printSpecialTokens(Token t) 39 | { 40 | // special tokens are chained in reverse. 41 | Token specialToken = t.specialToken; 42 | if (specialToken != null) { 43 | while (specialToken.specialToken != null) { 44 | specialToken = specialToken.specialToken; 45 | } 46 | while (specialToken != null) { 47 | stringBuilder.append(specialToken.image); 48 | specialToken = specialToken.next; 49 | } 50 | } 51 | } 52 | 53 | private void printTrailingComments() 54 | { 55 | if (lastToken != null && lastToken.kind == EOF) { 56 | printSpecialTokens(lastToken); 57 | } 58 | } 59 | 60 | public final void printToken(String s) 61 | { 62 | stringBuilder.append(" " + s + " "); 63 | } 64 | 65 | private void printToken(Token t) 66 | { 67 | while (lastToken != t) { 68 | lastToken = lastToken.next; 69 | printSpecialTokens(lastToken); 70 | stringBuilder.append(lastToken.image); 71 | } 72 | } 73 | 74 | protected void printUptoToken(Token t) 75 | { 76 | while (lastToken.next != t) { 77 | printToken(lastToken.next); 78 | } 79 | } 80 | 81 | protected void moveToEndOfNode(AstNode node) 82 | { 83 | lastToken = node.endToken; 84 | } 85 | 86 | protected void unparseUpto(AstNode node) 87 | { 88 | printUptoToken(node.beginToken); 89 | } 90 | 91 | protected void unparseRestOfTheNode(AstNode node) 92 | { 93 | if (lastToken != node.endToken) { 94 | printUptoToken(node.endToken); 95 | } 96 | } 97 | 98 | @Override 99 | public void defaultVisit(SimpleNode node, Void data) 100 | { 101 | AstNode astNode = (AstNode) node; 102 | AstNode firstChild = astNode.FirstChild(); 103 | AstNode lastChild = astNode.LastChild(); 104 | 105 | // Print the tokens. 106 | if (firstChild == null || astNode.beginToken != firstChild.beginToken) { 107 | printToken(astNode.beginToken); 108 | } 109 | 110 | node.childrenAccept(this, data); 111 | 112 | if (lastChild != null && astNode.endToken != lastChild.endToken) { 113 | printToken(astNode.endToken); 114 | } 115 | } 116 | 117 | @Override 118 | public void visit(CompilationUnit node, Void data) 119 | { 120 | defaultVisit(node, data); 121 | } 122 | 123 | @Override 124 | public void visit(UnaryExpression node, Void data) 125 | { 126 | defaultVisit(node, data); 127 | } 128 | 129 | @Override 130 | public void visit(UnsignedNumericLiteral node, Void data) 131 | { 132 | defaultVisit(node, data); 133 | } 134 | 135 | @Override 136 | public void visit(CharStringLiteral node, Void data) 137 | { 138 | defaultVisit(node, data); 139 | } 140 | 141 | @Override 142 | public void visit(DateLiteral node, Void data) 143 | { 144 | defaultVisit(node, data); 145 | } 146 | 147 | @Override 148 | public void visit(TimeLiteral node, Void data) 149 | { 150 | defaultVisit(node, data); 151 | } 152 | 153 | @Override 154 | public void visit(TimestampLiteral node, Void data) 155 | { 156 | defaultVisit(node, data); 157 | } 158 | 159 | @Override 160 | public void visit(IntervalLiteral node, Void data) 161 | { 162 | defaultVisit(node, data); 163 | } 164 | 165 | @Override 166 | public void visit(BooleanLiteral node, Void data) 167 | { 168 | defaultVisit(node, data); 169 | } 170 | 171 | @Override 172 | public void visit(Unsupported node, Void data) 173 | { 174 | defaultVisit(node, data); 175 | } 176 | 177 | @Override 178 | public void visit(Identifier node, Void data) 179 | { 180 | defaultVisit(node, data); 181 | } 182 | 183 | @Override 184 | public void visit(TableName node, Void data) 185 | { 186 | defaultVisit(node, data); 187 | } 188 | 189 | @Override 190 | public void visit(SchemaName node, Void data) 191 | { 192 | defaultVisit(node, data); 193 | } 194 | 195 | @Override 196 | public void visit(CatalogName node, Void data) 197 | { 198 | defaultVisit(node, data); 199 | } 200 | 201 | @Override 202 | public void visit(SchemaQualifiedName node, Void data) 203 | { 204 | defaultVisit(node, data); 205 | } 206 | 207 | @Override 208 | public void visit(ArrayType node, Void data) 209 | { 210 | defaultVisit(node, data); 211 | } 212 | 213 | @Override 214 | public void visit(FieldDefinition node, Void data) 215 | { 216 | defaultVisit(node, data); 217 | } 218 | 219 | @Override 220 | public void visit(ParenthesizedExpression node, Void data) 221 | { 222 | defaultVisit(node, data); 223 | } 224 | 225 | @Override 226 | public void visit(BuiltinValue node, Void data) 227 | { 228 | defaultVisit(node, data); 229 | } 230 | 231 | @Override 232 | public void visit(NullLiteral node, Void data) 233 | { 234 | defaultVisit(node, data); 235 | } 236 | 237 | @Override 238 | public void visit(ArrayLiteral node, Void data) 239 | { 240 | defaultVisit(node, data); 241 | } 242 | 243 | @Override 244 | public void visit(QualifiedName node, Void data) 245 | { 246 | defaultVisit(node, data); 247 | } 248 | 249 | @Override 250 | public void visit(GroupingOperation node, Void data) 251 | { 252 | defaultVisit(node, data); 253 | } 254 | 255 | @Override 256 | public void visit(WindowFunction node, Void data) 257 | { 258 | defaultVisit(node, data); 259 | } 260 | 261 | @Override 262 | public void visit(NullTreatment node, Void data) 263 | { 264 | defaultVisit(node, data); 265 | } 266 | 267 | @Override 268 | public void visit(NullIf node, Void data) 269 | { 270 | defaultVisit(node, data); 271 | } 272 | 273 | @Override 274 | public void visit(Coalesce node, Void data) 275 | { 276 | defaultVisit(node, data); 277 | } 278 | 279 | @Override 280 | public void visit(CaseExpression node, Void data) 281 | { 282 | defaultVisit(node, data); 283 | } 284 | 285 | @Override 286 | public void visit(WhenClause node, Void data) 287 | { 288 | defaultVisit(node, data); 289 | } 290 | 291 | @Override 292 | public void visit(ElseClause node, Void data) 293 | { 294 | defaultVisit(node, data); 295 | } 296 | 297 | @Override 298 | public void visit(WhenOperand node, Void data) 299 | { 300 | defaultVisit(node, data); 301 | } 302 | 303 | @Override 304 | public void visit(SearchedCaseOperand node, Void data) 305 | { 306 | defaultVisit(node, data); 307 | } 308 | 309 | @Override 310 | public void visit(CastExpression node, Void data) 311 | { 312 | defaultVisit(node, data); 313 | } 314 | 315 | @Override 316 | public void visit(FieldReference node, Void data) 317 | { 318 | defaultVisit(node, data); 319 | } 320 | 321 | @Override 322 | public void visit(AggregationFunction node, Void data) 323 | { 324 | defaultVisit(node, data); 325 | } 326 | 327 | @Override 328 | public void visit(FunctionCall node, Void data) 329 | { 330 | defaultVisit(node, data); 331 | } 332 | 333 | @Override 334 | public void visit(BuiltinFunctionCall node, Void data) 335 | { 336 | defaultVisit(node, data); 337 | } 338 | 339 | @Override 340 | public void visit(Unused node, Void data) 341 | { 342 | defaultVisit(node, data); 343 | } 344 | 345 | @Override 346 | public void visit(Lambda node, Void data) 347 | { 348 | defaultVisit(node, data); 349 | } 350 | 351 | @Override 352 | public void visit(ArrayElement node, Void data) 353 | { 354 | defaultVisit(node, data); 355 | } 356 | 357 | @Override 358 | public void visit(TimeZoneField node, Void data) 359 | { 360 | defaultVisit(node, data); 361 | } 362 | 363 | @Override 364 | public void visit(Concatenation node, Void data) 365 | { 366 | defaultVisit(node, data); 367 | } 368 | 369 | @Override 370 | public void visit(AdditiveExpression node, Void data) 371 | { 372 | defaultVisit(node, data); 373 | } 374 | 375 | @Override 376 | public void visit(MultiplicativeExpression node, Void data) 377 | { 378 | defaultVisit(node, data); 379 | } 380 | 381 | @Override 382 | public void visit(OrExpression node, Void data) 383 | { 384 | defaultVisit(node, data); 385 | } 386 | 387 | @Override 388 | public void visit(AndExpression node, Void data) 389 | { 390 | defaultVisit(node, data); 391 | } 392 | 393 | @Override 394 | public void visit(NotExpression node, Void data) 395 | { 396 | defaultVisit(node, data); 397 | } 398 | 399 | @Override 400 | public void visit(IsExpression node, Void data) 401 | { 402 | defaultVisit(node, data); 403 | } 404 | 405 | @Override 406 | public void visit(RowExpression node, Void data) 407 | { 408 | defaultVisit(node, data); 409 | } 410 | 411 | @Override 412 | public void visit(RowExression node, Void data) 413 | { 414 | defaultVisit(node, data); 415 | } 416 | 417 | @Override 418 | public void visit(Values node, Void data) 419 | { 420 | defaultVisit(node, data); 421 | } 422 | 423 | @Override 424 | public void visit(TableExpression node, Void data) 425 | { 426 | defaultVisit(node, data); 427 | } 428 | 429 | @Override 430 | public void visit(FromClause node, Void data) 431 | { 432 | defaultVisit(node, data); 433 | } 434 | 435 | @Override 436 | public void visit(CommaJoin node, Void data) 437 | { 438 | defaultVisit(node, data); 439 | } 440 | 441 | @Override 442 | public void visit(Join node, Void data) 443 | { 444 | defaultVisit(node, data); 445 | } 446 | 447 | @Override 448 | public void visit(TableSample node, Void data) 449 | { 450 | defaultVisit(node, data); 451 | } 452 | 453 | @Override 454 | public void visit(AliasedTable node, Void data) 455 | { 456 | defaultVisit(node, data); 457 | } 458 | 459 | @Override 460 | public void visit(Alias node, Void data) 461 | { 462 | defaultVisit(node, data); 463 | } 464 | 465 | @Override 466 | public void visit(Unnest node, Void data) 467 | { 468 | defaultVisit(node, data); 469 | } 470 | 471 | @Override 472 | public void visit(ColumnNames node, Void data) 473 | { 474 | defaultVisit(node, data); 475 | } 476 | 477 | @Override 478 | public void visit(OnClause node, Void data) 479 | { 480 | defaultVisit(node, data); 481 | } 482 | 483 | @Override 484 | public void visit(UsingClause node, Void data) 485 | { 486 | defaultVisit(node, data); 487 | } 488 | 489 | @Override 490 | public void visit(WhereClause node, Void data) 491 | { 492 | defaultVisit(node, data); 493 | } 494 | 495 | @Override 496 | public void visit(GroupbyClause node, Void data) 497 | { 498 | defaultVisit(node, data); 499 | } 500 | 501 | @Override 502 | public void visit(Rollup node, Void data) 503 | { 504 | defaultVisit(node, data); 505 | } 506 | 507 | @Override 508 | public void visit(Cube node, Void data) 509 | { 510 | defaultVisit(node, data); 511 | } 512 | 513 | @Override 514 | public void visit(GroupingSets node, Void data) 515 | { 516 | defaultVisit(node, data); 517 | } 518 | 519 | @Override 520 | public void visit(HavingClause node, Void data) 521 | { 522 | defaultVisit(node, data); 523 | } 524 | 525 | @Override 526 | public void visit(PartitionByClause node, Void data) 527 | { 528 | defaultVisit(node, data); 529 | } 530 | 531 | @Override 532 | public void visit(OrderByClause node, Void data) 533 | { 534 | defaultVisit(node, data); 535 | } 536 | 537 | @Override 538 | public void visit(WindowFrameUnits node, Void data) 539 | { 540 | defaultVisit(node, data); 541 | } 542 | 543 | @Override 544 | public void visit(WindowFrameExtent node, Void data) 545 | { 546 | defaultVisit(node, data); 547 | } 548 | 549 | @Override 550 | public void visit(UnboundedPreceding node, Void data) 551 | { 552 | defaultVisit(node, data); 553 | } 554 | 555 | @Override 556 | public void visit(CurrentRow node, Void data) 557 | { 558 | defaultVisit(node, data); 559 | } 560 | 561 | @Override 562 | public void visit(WindowFramePreceding node, Void data) 563 | { 564 | defaultVisit(node, data); 565 | } 566 | 567 | @Override 568 | public void visit(WindowFrameBetween node, Void data) 569 | { 570 | defaultVisit(node, data); 571 | } 572 | 573 | @Override 574 | public void visit(UnboundedFollowing node, Void data) 575 | { 576 | defaultVisit(node, data); 577 | } 578 | 579 | @Override 580 | public void visit(WindowFrameFollowing node, Void data) 581 | { 582 | defaultVisit(node, data); 583 | } 584 | 585 | @Override 586 | public void visit(Select node, Void data) 587 | { 588 | defaultVisit(node, data); 589 | } 590 | 591 | @Override 592 | public void visit(SelectList node, Void data) 593 | { 594 | defaultVisit(node, data); 595 | } 596 | 597 | @Override 598 | public void visit(SelectItem node, Void data) 599 | { 600 | defaultVisit(node, data); 601 | } 602 | 603 | @Override 604 | public void visit(Star node, Void data) 605 | { 606 | defaultVisit(node, data); 607 | } 608 | 609 | @Override 610 | public void visit(QuerySpecification node, Void data) 611 | { 612 | defaultVisit(node, data); 613 | } 614 | 615 | @Override 616 | public void visit(WithClause node, Void data) 617 | { 618 | defaultVisit(node, data); 619 | } 620 | 621 | @Override 622 | public void visit(Cte node, Void data) 623 | { 624 | defaultVisit(node, data); 625 | } 626 | 627 | @Override 628 | public void visit(SetOperation node, Void data) 629 | { 630 | defaultVisit(node, data); 631 | } 632 | 633 | @Override 634 | public void visit(Subquery node, Void data) 635 | { 636 | defaultVisit(node, data); 637 | } 638 | 639 | @Override 640 | public void visit(Comparison node, Void data) 641 | { 642 | defaultVisit(node, data); 643 | } 644 | 645 | @Override 646 | public void visit(Between node, Void data) 647 | { 648 | defaultVisit(node, data); 649 | } 650 | 651 | @Override 652 | public void visit(InPredicate node, Void data) 653 | { 654 | defaultVisit(node, data); 655 | } 656 | 657 | @Override 658 | public void visit(InvalueList node, Void data) 659 | { 660 | defaultVisit(node, data); 661 | } 662 | 663 | @Override 664 | public void visit(Like node, Void data) 665 | { 666 | defaultVisit(node, data); 667 | } 668 | 669 | @Override 670 | public void visit(IsNull node, Void data) 671 | { 672 | defaultVisit(node, data); 673 | } 674 | 675 | @Override 676 | public void visit(QuantifiedComparison node, Void data) 677 | { 678 | defaultVisit(node, data); 679 | } 680 | 681 | @Override 682 | public void visit(Exists node, Void data) 683 | { 684 | defaultVisit(node, data); 685 | } 686 | 687 | @Override 688 | public void visit(IsDistinct node, Void data) 689 | { 690 | defaultVisit(node, data); 691 | } 692 | 693 | @Override 694 | public void visit(InvervalQualifier node, Void data) 695 | { 696 | defaultVisit(node, data); 697 | } 698 | 699 | @Override 700 | public void visit(NonSecondField node, Void data) 701 | { 702 | defaultVisit(node, data); 703 | } 704 | 705 | @Override 706 | public void visit(SecondField node, Void data) 707 | { 708 | defaultVisit(node, data); 709 | } 710 | 711 | @Override 712 | public void visit(NonSecondDateTimeField node, Void data) 713 | { 714 | defaultVisit(node, data); 715 | } 716 | 717 | @Override 718 | public void visit(LanguageClause node, Void data) 719 | { 720 | defaultVisit(node, data); 721 | } 722 | 723 | @Override 724 | public void visit(ArgumentList node, Void data) 725 | { 726 | defaultVisit(node, data); 727 | } 728 | 729 | @Override 730 | public void visit(NamedArgument node, Void data) 731 | { 732 | defaultVisit(node, data); 733 | } 734 | 735 | @Override 736 | public void visit(SetQuantifier node, Void data) 737 | { 738 | defaultVisit(node, data); 739 | } 740 | 741 | @Override 742 | public void visit(FilterClause node, Void data) 743 | { 744 | defaultVisit(node, data); 745 | } 746 | 747 | @Override 748 | public void visit(SortSpecificationList node, Void data) 749 | { 750 | defaultVisit(node, data); 751 | } 752 | 753 | @Override 754 | public void visit(SortSpecification node, Void data) 755 | { 756 | defaultVisit(node, data); 757 | } 758 | 759 | @Override 760 | public void visit(OrderingDirection node, Void data) 761 | { 762 | defaultVisit(node, data); 763 | } 764 | 765 | @Override 766 | public void visit(NullOrdering node, Void data) 767 | { 768 | defaultVisit(node, data); 769 | } 770 | 771 | @Override 772 | public void visit(CreateSchema node, Void data) 773 | { 774 | defaultVisit(node, data); 775 | } 776 | 777 | @Override 778 | public void visit(Unsuppoerted node, Void data) 779 | { 780 | defaultVisit(node, data); 781 | } 782 | 783 | @Override 784 | public void visit(UseStatement node, Void data) 785 | { 786 | defaultVisit(node, data); 787 | } 788 | 789 | @Override 790 | public void visit(LambdaBody node, Void data) 791 | { 792 | defaultVisit(node, data); 793 | } 794 | 795 | @Override 796 | public void visit(LambdaParams node, Void data) 797 | { 798 | defaultVisit(node, data); 799 | } 800 | 801 | @Override 802 | public void visit(LambdaParam node, Void data) 803 | { 804 | defaultVisit(node, data); 805 | } 806 | 807 | @Override 808 | public void visit(LimitClause node, Void data) 809 | { 810 | defaultVisit(node, data); 811 | } 812 | 813 | @Override 814 | public void visit(MapType node, Void data) 815 | { 816 | defaultVisit(node, data); 817 | } 818 | 819 | @Override 820 | public void visit(TryExpression node, Void data) 821 | { 822 | defaultVisit(node, data); 823 | } 824 | } 825 | --------------------------------------------------------------------------------